Merge "Fix app tests to work without javaMockFS()"
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index bbec389..0595d68 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -717,7 +717,7 @@
// Add ninja file dependencies for files which all bazel invocations require.
bazelBuildList := absolutePath(filepath.Join(
- filepath.Dir(bootstrap.ModuleListFile), "bazel.list"))
+ filepath.Dir(bootstrap.CmdlineModuleListFile()), "bazel.list"))
ctx.AddNinjaFileDeps(bazelBuildList)
data, err := ioutil.ReadFile(bazelBuildList)
diff --git a/android/config.go b/android/config.go
index cae419b..2f8cb57 100644
--- a/android/config.go
+++ b/android/config.go
@@ -19,6 +19,7 @@
import (
"encoding/json"
+ "errors"
"fmt"
"io/ioutil"
"os"
@@ -506,6 +507,10 @@
c.stopBefore = stopBefore
}
+func (c *config) SetAllowMissingDependencies() {
+ c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true)
+}
+
var _ bootstrap.ConfigStopBefore = (*config)(nil)
// BlueprintToolLocation returns the directory containing build system tools
@@ -1631,6 +1636,20 @@
return nil
}
+func (l *ConfiguredJarList) MarshalJSON() ([]byte, error) {
+ if len(l.apexes) != len(l.jars) {
+ return nil, errors.New(fmt.Sprintf("Inconsistent ConfiguredJarList: apexes: %q, jars: %q", l.apexes, l.jars))
+ }
+
+ list := make([]string, 0, len(l.apexes))
+
+ for i := 0; i < len(l.apexes); i++ {
+ list = append(list, l.apexes[i]+":"+l.jars[i])
+ }
+
+ return json.Marshal(list)
+}
+
// ModuleStem hardcodes the stem of framework-minus-apex to return "framework".
//
// TODO(b/139391334): hard coded until we find a good way to query the stem of a
diff --git a/android/config_test.go b/android/config_test.go
index a11115d..9df5288 100644
--- a/android/config_test.go
+++ b/android/config_test.go
@@ -16,6 +16,7 @@
import (
"fmt"
+ "path/filepath"
"reflect"
"strings"
"testing"
@@ -87,6 +88,37 @@
}
}
+func verifyProductVariableMarshaling(t *testing.T, v productVariables) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "test.variables")
+ err := saveToConfigFile(&v, path)
+ if err != nil {
+ t.Errorf("Couldn't save default product config: %q", err)
+ }
+
+ var v2 productVariables
+ err = loadFromConfigFile(&v2, path)
+ if err != nil {
+ t.Errorf("Couldn't load default product config: %q", err)
+ }
+}
+func TestDefaultProductVariableMarshaling(t *testing.T) {
+ v := productVariables{}
+ v.SetDefaultConfig()
+ verifyProductVariableMarshaling(t, v)
+}
+
+func TestBootJarsMarshaling(t *testing.T) {
+ v := productVariables{}
+ v.SetDefaultConfig()
+ v.BootJars = ConfiguredJarList{
+ apexes: []string{"apex"},
+ jars: []string{"jar"},
+ }
+
+ verifyProductVariableMarshaling(t, v)
+}
+
func assertStringEquals(t *testing.T, expected, actual string) {
if actual != expected {
t.Errorf("expected %q found %q", expected, actual)
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 84501fe..17f211b 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -385,6 +385,21 @@
return rspFileInputs
}
+// RspFile returns the path to the rspfile that was passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
+func (r *RuleBuilder) RspFile() WritablePath {
+ var rspFile WritablePath
+ for _, c := range r.commands {
+ if c.rspFile != nil {
+ if rspFile != nil {
+ panic("Multiple commands in a rule may not have rsp file inputs")
+ }
+ rspFile = c.rspFile
+ }
+ }
+
+ return rspFile
+}
+
// Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
func (r *RuleBuilder) Commands() []string {
var commands []string
@@ -394,16 +409,6 @@
return commands
}
-// NinjaEscapedCommands returns a slice containing the built command line after ninja escaping for each call to
-// RuleBuilder.Command.
-func (r *RuleBuilder) NinjaEscapedCommands() []string {
- var commands []string
- for _, c := range r.commands {
- commands = append(commands, c.NinjaEscapedString())
- }
- return commands
-}
-
// BuilderContext is a subset of ModuleContext and SingletonContext.
type BuilderContext interface {
PathContext
@@ -458,9 +463,11 @@
}
tools := r.Tools()
- commands := r.NinjaEscapedCommands()
+ commands := r.Commands()
outputs := r.Outputs()
inputs := r.Inputs()
+ rspFileInputs := r.RspFileInputs()
+ rspFilePath := r.RspFile()
if len(commands) == 0 {
return
@@ -530,7 +537,7 @@
}
// Create a rule to write the manifest as a the textproto.
- WriteFileRule(r.ctx, r.sboxManifestPath, proto.MarshalTextString(&manifest))
+ WriteFileRule(r.ctx, r.sboxManifestPath, proptools.NinjaEscape(proto.MarshalTextString(&manifest)))
// Generate a new string to use as the command line of the sbox rule. This uses
// a RuleBuilderCommand as a convenience method of building the command line, then
@@ -559,15 +566,14 @@
}
// Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
- // ImplicitOutputs. RuleBuilder only uses "$out" for the rsp file location, so the distinction between Outputs and
+ // ImplicitOutputs. RuleBuilder doesn't use "$out", so the distinction between Outputs and
// ImplicitOutputs doesn't matter.
output := outputs[0]
implicitOutputs := outputs[1:]
var rspFile, rspFileContent string
- rspFileInputs := r.RspFileInputs()
- if rspFileInputs != nil {
- rspFile = "$out.rsp"
+ if rspFilePath != nil {
+ rspFile = rspFilePath.String()
rspFileContent = "$in"
}
@@ -585,10 +591,10 @@
r.ctx.Build(r.pctx, BuildParams{
Rule: r.ctx.Rule(pctx, name, blueprint.RuleParams{
- Command: commandString,
- CommandDeps: tools.Strings(),
+ Command: proptools.NinjaEscape(commandString),
+ CommandDeps: proptools.NinjaEscapeList(tools.Strings()),
Restat: r.restat,
- Rspfile: rspFile,
+ Rspfile: proptools.NinjaEscape(rspFile),
RspfileContent: rspFileContent,
Pool: pool,
}),
@@ -620,9 +626,7 @@
tools Paths
packagedTools []PackagingSpec
rspFileInputs Paths
-
- // spans [start,end) of the command that should not be ninja escaped
- unescapedSpans [][2]int
+ rspFile WritablePath
}
func (c *RuleBuilderCommand) addInput(path Path) string {
@@ -1020,8 +1024,9 @@
}
// FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with no separator
-// between them. The paths will be written to the rspfile.
-func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, paths Paths) *RuleBuilderCommand {
+// between them. The paths will be written to the rspfile. If sbox is enabled, the rspfile must
+// be outside the sbox directory.
+func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
if c.rspFileInputs != nil {
panic("FlagWithRspFileInputList cannot be called if rsp file inputs have already been provided")
}
@@ -1033,10 +1038,16 @@
}
c.rspFileInputs = paths
+ c.rspFile = rspFile
- rspFile := "$out.rsp"
- c.FlagWithArg(flag, rspFile)
- c.unescapedSpans = append(c.unescapedSpans, [2]int{c.buf.Len() - len(rspFile), c.buf.Len()})
+ if c.rule.sbox {
+ if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
+ panic(fmt.Errorf("FlagWithRspFileInputList rspfile %q must not be inside out dir %q",
+ rspFile.String(), c.rule.outDir.String()))
+ }
+ }
+
+ c.FlagWithArg(flag, rspFile.String())
return c
}
@@ -1045,11 +1056,6 @@
return c.buf.String()
}
-// String returns the command line.
-func (c *RuleBuilderCommand) NinjaEscapedString() string {
- return ninjaEscapeExceptForSpans(c.String(), c.unescapedSpans)
-}
-
// RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
// and returns sbox testproto generated by the RuleBuilder.
func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
@@ -1063,25 +1069,6 @@
return &manifest
}
-func ninjaEscapeExceptForSpans(s string, spans [][2]int) string {
- if len(spans) == 0 {
- return proptools.NinjaEscape(s)
- }
-
- sb := strings.Builder{}
- sb.Grow(len(s) * 11 / 10)
-
- i := 0
- for _, span := range spans {
- sb.WriteString(proptools.NinjaEscape(s[i:span[0]]))
- sb.WriteString(s[span[0]:span[1]])
- i = span[1]
- }
- sb.WriteString(proptools.NinjaEscape(s[i:]))
-
- return sb.String()
-}
-
func ninjaNameEscape(s string) string {
b := []byte(s)
escaped := false
diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go
index 06ea124..080e236 100644
--- a/android/rule_builder_test.go
+++ b/android/rule_builder_test.go
@@ -267,10 +267,10 @@
ctx := builderContext()
fmt.Println(NewRuleBuilder(pctx, ctx).Command().
Tool(PathForSource(ctx, "javac")).
- FlagWithRspFileInputList("@", PathsForTesting("a.java", "b.java")).
- NinjaEscapedString())
+ FlagWithRspFileInputList("@", PathForOutput(ctx, "foo.rsp"), PathsForTesting("a.java", "b.java")).
+ String())
// Output:
- // javac @$out.rsp
+ // javac @out/foo.rsp
}
func ExampleRuleBuilderCommand_String() {
@@ -283,16 +283,6 @@
// FOO=foo echo $FOO
}
-func ExampleRuleBuilderCommand_NinjaEscapedString() {
- ctx := builderContext()
- fmt.Println(NewRuleBuilder(pctx, ctx).Command().
- Text("FOO=foo").
- Text("echo $FOO").
- NinjaEscapedString())
- // Output:
- // FOO=foo echo $$FOO
-}
-
func TestRuleBuilder(t *testing.T) {
fs := map[string][]byte{
"dep_fixer": nil,
@@ -631,80 +621,6 @@
})
}
-func Test_ninjaEscapeExceptForSpans(t *testing.T) {
- type args struct {
- s string
- spans [][2]int
- }
- tests := []struct {
- name string
- args args
- want string
- }{
- {
- name: "empty",
- args: args{
- s: "",
- },
- want: "",
- },
- {
- name: "unescape none",
- args: args{
- s: "$abc",
- },
- want: "$$abc",
- },
- {
- name: "unescape all",
- args: args{
- s: "$abc",
- spans: [][2]int{{0, 4}},
- },
- want: "$abc",
- },
- {
- name: "unescape first",
- args: args{
- s: "$abc$",
- spans: [][2]int{{0, 1}},
- },
- want: "$abc$$",
- },
- {
- name: "unescape last",
- args: args{
- s: "$abc$",
- spans: [][2]int{{4, 5}},
- },
- want: "$$abc$",
- },
- {
- name: "unescape middle",
- args: args{
- s: "$a$b$c$",
- spans: [][2]int{{2, 5}},
- },
- want: "$$a$b$c$$",
- },
- {
- name: "unescape multiple",
- args: args{
- s: "$a$b$c$",
- spans: [][2]int{{2, 3}, {4, 5}},
- },
- want: "$$a$b$c$$",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if got := ninjaEscapeExceptForSpans(tt.args.s, tt.args.spans); got != tt.want {
- t.Errorf("ninjaEscapeExceptForSpans() = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
func TestRuleBuilderHashInputs(t *testing.T) {
// The basic idea here is to verify that the command (in the case of a
// non-sbox rule) or the sbox textproto manifest contain a hash of the
diff --git a/android/test_suites.go b/android/test_suites.go
index 7ecb8d2..6b7b909 100644
--- a/android/test_suites.go
+++ b/android/test_suites.go
@@ -68,7 +68,7 @@
FlagWithOutput("-o ", outputFile).
FlagWithArg("-P ", "host/testcases").
FlagWithArg("-C ", testCasesDir.String()).
- FlagWithRspFileInputList("-r ", installedPaths.Paths())
+ FlagWithRspFileInputList("-r ", outputFile.ReplaceExtension(ctx, "rsp"), installedPaths.Paths())
rule.Build("robolectric_tests_zip", "robolectric-tests.zip")
return outputFile
diff --git a/android/variable.go b/android/variable.go
index a5e9ab4..08fa12c 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -426,6 +426,9 @@
Malloc_zero_contents: boolPtr(true),
Malloc_pattern_fill_contents: boolPtr(false),
Safestack: boolPtr(false),
+
+ BootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}},
+ UpdatableBootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}},
}
if runtime.GOOS == "linux" {
diff --git a/bootstrap_test.sh b/bootstrap_test.sh
new file mode 100755
index 0000000..87f5e31
--- /dev/null
+++ b/bootstrap_test.sh
@@ -0,0 +1,319 @@
+#!/bin/bash -eu
+
+# This test exercises the bootstrapping process of the build system
+# in a source tree that only contains enough files for Bazel and Soong to work.
+
+HARDWIRED_MOCK_TOP=
+# Uncomment this to be able to view the source tree after a test is run
+# HARDWIRED_MOCK_TOP=/tmp/td
+
+REAL_TOP="$(readlink -f "$(dirname "$0")"/../..)"
+
+function fail {
+ echo ERROR: $1
+ exit 1
+}
+
+function copy_directory() {
+ local dir="$1"
+ local parent="$(dirname "$dir")"
+
+ mkdir -p "$MOCK_TOP/$parent"
+ cp -R "$REAL_TOP/$dir" "$MOCK_TOP/$parent"
+}
+
+function symlink_file() {
+ local file="$1"
+
+ mkdir -p "$MOCK_TOP/$(dirname "$file")"
+ ln -s "$REAL_TOP/$file" "$MOCK_TOP/$file"
+}
+
+function symlink_directory() {
+ local dir="$1"
+
+ mkdir -p "$MOCK_TOP/$dir"
+ # We need to symlink the contents of the directory individually instead of
+ # using one symlink for the whole directory because finder.go doesn't follow
+ # symlinks when looking for Android.bp files
+ for i in $(ls "$REAL_TOP/$dir"); do
+ local target="$MOCK_TOP/$dir/$i"
+ local source="$REAL_TOP/$dir/$i"
+
+ if [[ -e "$target" ]]; then
+ if [[ ! -d "$source" || ! -d "$target" ]]; then
+ fail "Trying to symlink $dir twice"
+ fi
+ else
+ ln -s "$REAL_TOP/$dir/$i" "$MOCK_TOP/$dir/$i";
+ fi
+ done
+}
+
+function setup_bazel() {
+ copy_directory build/bazel
+
+ symlink_directory prebuilts/bazel
+ symlink_directory prebuilts/jdk
+
+ symlink_file WORKSPACE
+ symlink_file tools/bazel
+}
+
+function setup() {
+ if [[ ! -z "$HARDWIRED_MOCK_TOP" ]]; then
+ MOCK_TOP="$HARDWIRED_MOCK_TOP"
+ rm -fr "$MOCK_TOP"
+ mkdir -p "$MOCK_TOP"
+ else
+ MOCK_TOP=$(mktemp -t -d st.XXXXX)
+ trap 'echo cd / && echo rm -fr "$MOCK_TOP"' EXIT
+ fi
+
+ echo "Test case: ${FUNCNAME[1]}, mock top path: $MOCK_TOP"
+ cd "$MOCK_TOP"
+
+ copy_directory build/blueprint
+ copy_directory build/soong
+
+ symlink_directory prebuilts/go
+ symlink_directory prebuilts/build-tools
+ symlink_directory external/golang-protobuf
+
+ touch "$MOCK_TOP/Android.bp"
+
+ export ALLOW_MISSING_DEPENDENCIES=true
+
+ mkdir -p out/soong
+}
+
+function run_soong() {
+ build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests
+}
+
+function test_smoke {
+ setup
+ run_soong
+}
+
+function test_bazel_smoke {
+ setup
+ setup_bazel
+
+ tools/bazel info
+
+}
+function test_null_build() {
+ setup
+ run_soong
+ local bootstrap_mtime1=$(stat -c "%y" out/soong/.bootstrap/build.ninja)
+ local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
+ run_soong
+ local bootstrap_mtime2=$(stat -c "%y" out/soong/.bootstrap/build.ninja)
+ local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
+
+ if [[ "$bootstrap_mtime1" == "$bootstrap_mtime2" ]]; then
+ # Bootstrapping is always done. It doesn't take a measurable amount of time.
+ fail "Bootstrap Ninja file did not change on null build"
+ fi
+
+ if [[ "$output_mtime1" != "$output_mtime2" ]]; then
+ fail "Output Ninja file changed on null build"
+ fi
+}
+
+function test_soong_build_rebuilt_if_blueprint_changes() {
+ setup
+ run_soong
+ local mtime1=$(stat -c "%y" out/soong/.bootstrap/build.ninja)
+
+ sed -i 's/pluginGenSrcCmd/pluginGenSrcCmd2/g' build/blueprint/bootstrap/bootstrap.go
+
+ run_soong
+ local mtime2=$(stat -c "%y" out/soong/.bootstrap/build.ninja)
+
+ if [[ "$mtime1" == "$mtime2" ]]; then
+ fail "Bootstrap Ninja file did not change"
+ fi
+}
+
+function test_change_android_bp() {
+ setup
+ mkdir -p a
+ cat > a/Android.bp <<'EOF'
+python_binary_host {
+ name: "my_little_binary_host",
+ srcs: ["my_little_binary_host.py"]
+}
+EOF
+ touch a/my_little_binary_host.py
+ run_soong
+
+ grep -q "^# Module:.*my_little_binary_host" out/soong/build.ninja || fail "module not found"
+
+ cat > a/Android.bp <<'EOF'
+python_binary_host {
+ name: "my_great_binary_host",
+ srcs: ["my_great_binary_host.py"]
+}
+EOF
+ touch a/my_great_binary_host.py
+ run_soong
+
+ grep -q "^# Module:.*my_little_binary_host" out/soong/build.ninja && fail "old module found"
+ grep -q "^# Module:.*my_great_binary_host" out/soong/build.ninja || fail "new module not found"
+}
+
+
+function test_add_android_bp() {
+ setup
+ run_soong
+ local mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+ mkdir -p a
+ cat > a/Android.bp <<'EOF'
+python_binary_host {
+ name: "my_little_binary_host",
+ srcs: ["my_little_binary_host.py"]
+}
+EOF
+ touch a/my_little_binary_host.py
+ run_soong
+
+ local mtime2=$(stat -c "%y" out/soong/build.ninja)
+ if [[ "$mtime1" == "$mtime2" ]]; then
+ fail "Output Ninja file did not change"
+ fi
+
+ grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja || fail "New module not in output"
+
+ run_soong
+}
+
+function test_delete_android_bp() {
+ setup
+ mkdir -p a
+ cat > a/Android.bp <<'EOF'
+python_binary_host {
+ name: "my_little_binary_host",
+ srcs: ["my_little_binary_host.py"]
+}
+EOF
+ touch a/my_little_binary_host.py
+ run_soong
+
+ grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja || fail "Module not in output"
+
+ rm a/Android.bp
+ run_soong
+
+ grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output"
+}
+
+function test_add_file_to_glob() {
+ setup
+
+ mkdir -p a
+ cat > a/Android.bp <<'EOF'
+python_binary_host {
+ name: "my_little_binary_host",
+ srcs: ["*.py"],
+}
+EOF
+ touch a/my_little_binary_host.py
+ run_soong
+ local mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+ touch a/my_little_library.py
+ run_soong
+
+ local mtime2=$(stat -c "%y" out/soong/build.ninja)
+ if [[ "$mtime1" == "$mtime2" ]]; then
+ fail "Output Ninja file did not change"
+ fi
+
+ grep -q my_little_library.py out/soong/build.ninja || fail "new file is not in output"
+}
+
+function test_add_file_to_soong_build() {
+ setup
+ run_soong
+ local mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+ mkdir -p a
+ cat > a/Android.bp <<'EOF'
+bootstrap_go_package {
+ name: "picard-soong-rules",
+ pkgPath: "android/soong/picard",
+ deps: [
+ "blueprint",
+ "soong",
+ "soong-android",
+ ],
+ srcs: [
+ "picard.go",
+ ],
+ pluginFor: ["soong_build"],
+}
+EOF
+
+ cat > a/picard.go <<'EOF'
+package picard
+
+import (
+ "android/soong/android"
+ "github.com/google/blueprint"
+)
+
+var (
+ pctx = android.NewPackageContext("picard")
+)
+
+func init() {
+ android.RegisterSingletonType("picard", PicardSingleton)
+}
+
+func PicardSingleton() android.Singleton {
+ return &picardSingleton{}
+}
+
+type picardSingleton struct{}
+
+func (p *picardSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ picardRule := ctx.Rule(pctx, "picard",
+ blueprint.RuleParams{
+ Command: "echo Make it so. > ${out}",
+ CommandDeps: []string{},
+ Description: "Something quotable",
+ })
+
+ outputFile := android.PathForOutput(ctx, "picard", "picard.txt")
+ var deps android.Paths
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: picardRule,
+ Output: outputFile,
+ Inputs: deps,
+ })
+}
+
+EOF
+
+ run_soong
+ local mtime2=$(stat -c "%y" out/soong/build.ninja)
+ if [[ "$mtime1" == "$mtime2" ]]; then
+ fail "Output Ninja file did not change"
+ fi
+
+ grep -q "Make it so" out/soong/build.ninja || fail "New action not present"
+}
+
+test_bazel_smoke
+test_smoke
+test_null_build
+test_soong_build_rebuilt_if_blueprint_changes
+test_add_file_to_glob
+test_add_android_bp
+test_change_android_bp
+test_delete_android_bp
+test_add_file_to_soong_build
diff --git a/cc/fuzz.go b/cc/fuzz.go
index 9fe5b17..5219ebc 100644
--- a/cc/fuzz.go
+++ b/cc/fuzz.go
@@ -440,7 +440,8 @@
command := builder.Command().BuiltTool("soong_zip").
Flag("-j").
FlagWithOutput("-o ", corpusZip)
- command.FlagWithRspFileInputList("-r ", fuzzModule.corpus)
+ rspFile := corpusZip.ReplaceExtension(ctx, "rsp")
+ command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus)
files = append(files, fileToZip{corpusZip, ""})
}
diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go
index fdd1fec..4014fe0 100644
--- a/cc/vendor_snapshot.go
+++ b/cc/vendor_snapshot.go
@@ -528,10 +528,11 @@
ctx,
snapshotDir,
c.name+"-"+ctx.Config().DeviceName()+"_list")
+ rspFile := snapshotOutputList.ReplaceExtension(ctx, "rsp")
zipRule.Command().
Text("tr").
FlagWithArg("-d ", "\\'").
- FlagWithRspFileInputList("< ", snapshotOutputs).
+ FlagWithRspFileInputList("< ", rspFile, snapshotOutputs).
FlagWithOutput("> ", snapshotOutputList)
zipRule.Temporary(snapshotOutputList)
diff --git a/cc/vndk.go b/cc/vndk.go
index 85028d0..b7047e9 100644
--- a/cc/vndk.go
+++ b/cc/vndk.go
@@ -827,10 +827,11 @@
// filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with tr
snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+"_list")
+ rspFile := snapshotOutputList.ReplaceExtension(ctx, "rsp")
zipRule.Command().
Text("tr").
FlagWithArg("-d ", "\\'").
- FlagWithRspFileInputList("< ", snapshotOutputs).
+ FlagWithRspFileInputList("< ", rspFile, snapshotOutputs).
FlagWithOutput("> ", snapshotOutputList)
zipRule.Temporary(snapshotOutputList)
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index d96abd9..11d3620 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -80,7 +80,7 @@
}
func newConfig(srcDir string) android.Config {
- configuration, err := android.NewConfig(srcDir, bootstrap.BuildDir, bootstrap.ModuleListFile)
+ configuration, err := android.NewConfig(srcDir, bootstrap.CmdlineBuildDir(), bootstrap.CmdlineModuleListFile())
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@@ -101,6 +101,10 @@
configuration := newConfig(srcDir)
extraNinjaDeps := []string{configuration.ProductVariablesFileName}
+ if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
+ configuration.SetAllowMissingDependencies()
+ }
+
// These two are here so that we restart a non-debugged soong_build when the
// user sets SOONG_DELVE the first time.
configuration.Getenv("SOONG_DELVE")
@@ -127,7 +131,7 @@
// the incorrect results from the first pass, and file I/O is expensive.
firstCtx := newContext(configuration)
configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja)
- bootstrap.Main(firstCtx.Context, configuration, extraNinjaDeps...)
+ bootstrap.Main(firstCtx.Context, configuration, false, extraNinjaDeps...)
// Invoke bazel commands and save results for second pass.
if err := configuration.BazelContext.InvokeBazel(); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
@@ -140,10 +144,10 @@
os.Exit(1)
}
ctx = newContext(secondPassConfig)
- bootstrap.Main(ctx.Context, secondPassConfig, extraNinjaDeps...)
+ bootstrap.Main(ctx.Context, secondPassConfig, false, extraNinjaDeps...)
} else {
ctx = newContext(configuration)
- bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...)
+ bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...)
}
// Convert the Soong module graph into Bazel BUILD files.
@@ -167,7 +171,7 @@
// TODO(ccross): make this a command line argument. Requires plumbing through blueprint
// to affect the command line of the primary builder.
if shouldPrepareBuildActions(configuration) {
- metricsFile := filepath.Join(bootstrap.BuildDir, "soong_build_metrics.pb")
+ metricsFile := filepath.Join(bootstrap.CmdlineBuildDir(), "soong_build_metrics.pb")
err := android.WriteMetrics(configuration, metricsFile)
if err != nil {
fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err)
@@ -193,7 +197,7 @@
// Android.bp files. It must not depend on the values of per-build product
// configurations or variables, since those will generate different BUILD
// files based on how the user has configured their tree.
- bp2buildCtx.SetModuleListFile(bootstrap.ModuleListFile)
+ bp2buildCtx.SetModuleListFile(bootstrap.CmdlineModuleListFile())
extraNinjaDeps, err := bp2buildCtx.ListModulePaths(srcDir)
if err != nil {
panic(err)
@@ -202,7 +206,7 @@
// Run the loading and analysis pipeline to prepare the graph of regular
// Modules parsed from Android.bp files, and the BazelTargetModules mapped
// from the regular Modules.
- bootstrap.Main(bp2buildCtx.Context, configuration, extraNinjaDeps...)
+ bootstrap.Main(bp2buildCtx.Context, configuration, false, extraNinjaDeps...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 1c5e78a..390a9ec 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -70,7 +70,7 @@
return build.NewConfig(ctx, args...)
},
stdio: stdio,
- run: make,
+ run: runMake,
}, {
flag: "--dumpvar-mode",
description: "print the value of the legacy make variable VAR to stdout",
@@ -92,7 +92,7 @@
description: "build modules based on the specified build action",
config: buildActionConfig,
stdio: stdio,
- run: make,
+ run: runMake,
},
}
@@ -478,7 +478,7 @@
return build.NewBuildActionConfig(buildAction, *dir, ctx, args...)
}
-func make(ctx build.Context, config build.Config, _ []string, logsDir string) {
+func runMake(ctx build.Context, config build.Config, _ []string, logsDir string) {
if config.IsVerbose() {
writer := ctx.Writer
fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.")
diff --git a/filesystem/Android.bp b/filesystem/Android.bp
index dcdbdcf..791019d 100644
--- a/filesystem/Android.bp
+++ b/filesystem/Android.bp
@@ -14,6 +14,7 @@
"bootimg.go",
"filesystem.go",
"logical_partition.go",
+ "vbmeta.go",
],
testSrcs: [
],
diff --git a/filesystem/bootimg.go b/filesystem/bootimg.go
index 372a610..3dcc416 100644
--- a/filesystem/bootimg.go
+++ b/filesystem/bootimg.go
@@ -17,6 +17,7 @@
import (
"fmt"
"strconv"
+ "strings"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
@@ -217,22 +218,46 @@
}
func (b *bootimg) signImage(ctx android.ModuleContext, unsignedImage android.OutputPath) android.OutputPath {
- output := android.PathForModuleOut(ctx, b.installFileName()).OutputPath
- key := android.PathForModuleSrc(ctx, proptools.String(b.properties.Avb_private_key))
+ propFile, toolDeps := b.buildPropFile(ctx)
+ output := android.PathForModuleOut(ctx, b.installFileName()).OutputPath
builder := android.NewRuleBuilder(pctx, ctx)
builder.Command().Text("cp").Input(unsignedImage).Output(output)
- builder.Command().
- BuiltTool("avbtool").
- Flag("add_hash_footer").
- FlagWithArg("--partition_name ", b.partitionName()).
- FlagWithInput("--key ", key).
- FlagWithOutput("--image ", output)
+ builder.Command().BuiltTool("verity_utils").
+ Input(propFile).
+ Implicits(toolDeps).
+ Output(output)
builder.Build("sign_bootimg", fmt.Sprintf("Signing %s", b.BaseModuleName()))
return output
}
+func (b *bootimg) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) {
+ var sb strings.Builder
+ var deps android.Paths
+ addStr := func(name string, value string) {
+ fmt.Fprintf(&sb, "%s=%s\n", name, value)
+ }
+ addPath := func(name string, path android.Path) {
+ addStr(name, path.String())
+ deps = append(deps, path)
+ }
+
+ addStr("avb_hash_enable", "true")
+ addPath("avb_avbtool", ctx.Config().HostToolPath(ctx, "avbtool"))
+ algorithm := proptools.StringDefault(b.properties.Avb_algorithm, "SHA256_RSA4096")
+ addStr("avb_algorithm", algorithm)
+ key := android.PathForModuleSrc(ctx, proptools.String(b.properties.Avb_private_key))
+ addPath("avb_key_path", key)
+ addStr("avb_add_hash_footer_args", "") // TODO(jiyong): add --rollback_index
+ partitionName := proptools.StringDefault(b.properties.Partition_name, b.Name())
+ addStr("partition_name", partitionName)
+
+ propFile = android.PathForModuleOut(ctx, "prop").OutputPath
+ android.WriteFileRule(ctx, propFile, sb.String())
+ return propFile, deps
+}
+
var _ android.AndroidMkEntriesProvider = (*bootimg)(nil)
// Implements android.AndroidMkEntriesProvider
@@ -255,6 +280,13 @@
return b.output
}
+func (b *bootimg) SignedOutputPath() android.Path {
+ if proptools.Bool(b.properties.Use_avb) {
+ return b.OutputPath()
+ }
+ return nil
+}
+
var _ android.OutputFileProducer = (*bootimg)(nil)
// Implements android.OutputFileProducer
diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go
index 3b0a7ae..8974eba 100644
--- a/filesystem/filesystem.go
+++ b/filesystem/filesystem.go
@@ -55,6 +55,9 @@
// Hash and signing algorithm for avbtool. Default is SHA256_RSA4096.
Avb_algorithm *string
+ // Name of the partition stored in vbmeta desc. Defaults to the name of this module.
+ Partition_name *string
+
// Type of the filesystem. Currently, ext4, cpio, and compressed_cpio are supported. Default
// is ext4.
Type *string
@@ -279,7 +282,8 @@
key := android.PathForModuleSrc(ctx, proptools.String(f.properties.Avb_private_key))
addPath("avb_key_path", key)
addStr("avb_add_hashtree_footer_args", "--do_not_generate_fec")
- addStr("partition_name", f.Name())
+ partitionName := proptools.StringDefault(f.properties.Partition_name, f.Name())
+ addStr("partition_name", partitionName)
}
if proptools.String(f.properties.File_contexts) != "" {
@@ -381,6 +385,10 @@
type Filesystem interface {
android.Module
OutputPath() android.Path
+
+ // Returns the output file that is signed by avbtool. If this module is not signed, returns
+ // nil.
+ SignedOutputPath() android.Path
}
var _ Filesystem = (*filesystem)(nil)
@@ -388,3 +396,10 @@
func (f *filesystem) OutputPath() android.Path {
return f.output
}
+
+func (f *filesystem) SignedOutputPath() android.Path {
+ if proptools.Bool(f.properties.Use_avb) {
+ return f.OutputPath()
+ }
+ return nil
+}
diff --git a/filesystem/logical_partition.go b/filesystem/logical_partition.go
index 16b6037..20d9622 100644
--- a/filesystem/logical_partition.go
+++ b/filesystem/logical_partition.go
@@ -209,6 +209,10 @@
return l.output
}
+func (l *logicalPartition) SignedOutputPath() android.Path {
+ return nil // logical partition is not signed by itself
+}
+
var _ android.OutputFileProducer = (*logicalPartition)(nil)
// Implements android.OutputFileProducer
diff --git a/filesystem/vbmeta.go b/filesystem/vbmeta.go
new file mode 100644
index 0000000..f823387
--- /dev/null
+++ b/filesystem/vbmeta.go
@@ -0,0 +1,265 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filesystem
+
+import (
+ "fmt"
+ "strconv"
+
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
+
+ "android/soong/android"
+)
+
+func init() {
+ android.RegisterModuleType("vbmeta", vbmetaFactory)
+}
+
+type vbmeta struct {
+ android.ModuleBase
+
+ properties vbmetaProperties
+
+ output android.OutputPath
+ installDir android.InstallPath
+}
+
+type vbmetaProperties struct {
+ // Name of the partition stored in vbmeta desc. Defaults to the name of this module.
+ Partition_name *string
+
+ // Set the name of the output. Defaults to <module_name>.img.
+ Stem *string
+
+ // Path to the private key that avbtool will use to sign this vbmeta image.
+ Private_key *string `android:"path"`
+
+ // Algorithm that avbtool will use to sign this vbmeta image. Default is SHA256_RSA4096.
+ Algorithm *string
+
+ // File whose content will provide the rollback index. If unspecified, the rollback index
+ // is from PLATFORM_SECURITY_PATCH
+ Rollback_index_file *string `android:"path"`
+
+ // Rollback index location of this vbmeta image. Must be 0, 1, 2, etc. Default is 0.
+ Rollback_index_location *int64
+
+ // List of filesystem modules that this vbmeta has descriptors for. The filesystem modules
+ // have to be signed (use_avb: true).
+ Partitions []string
+
+ // List of chained partitions that this vbmeta deletages the verification.
+ Chained_partitions []chainedPartitionProperties
+}
+
+type chainedPartitionProperties struct {
+ // Name of the chained partition
+ Name *string
+
+ // Rollback index location of the chained partition. Must be 0, 1, 2, etc. Default is the
+ // index of this partition in the list + 1.
+ Rollback_index_location *int64
+
+ // Path to the public key that the chained partition is signed with. If this is specified,
+ // private_key is ignored.
+ Public_key *string `android:"path"`
+
+ // Path to the private key that the chained partition is signed with. If this is specified,
+ // and public_key is not specified, a public key is extracted from this private key and
+ // the extracted public key is embedded in the vbmeta image.
+ Private_key *string `android:"path"`
+}
+
+// vbmeta is the partition image that has the verification information for other partitions.
+func vbmetaFactory() android.Module {
+ module := &vbmeta{}
+ module.AddProperties(&module.properties)
+ android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst)
+ return module
+}
+
+type vbmetaDep struct {
+ blueprint.BaseDependencyTag
+ kind string
+}
+
+var vbmetaPartitionDep = vbmetaDep{kind: "partition"}
+
+func (v *vbmeta) DepsMutator(ctx android.BottomUpMutatorContext) {
+ ctx.AddDependency(ctx.Module(), vbmetaPartitionDep, v.properties.Partitions...)
+}
+
+func (v *vbmeta) installFileName() string {
+ return proptools.StringDefault(v.properties.Stem, v.BaseModuleName()+".img")
+}
+
+func (v *vbmeta) partitionName() string {
+ return proptools.StringDefault(v.properties.Partition_name, v.BaseModuleName())
+}
+
+func (v *vbmeta) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ extractedPublicKeys := v.extractPublicKeys(ctx)
+
+ v.output = android.PathForModuleOut(ctx, v.installFileName()).OutputPath
+
+ builder := android.NewRuleBuilder(pctx, ctx)
+ cmd := builder.Command().BuiltTool("avbtool").Text("make_vbmeta_image")
+
+ key := android.PathForModuleSrc(ctx, proptools.String(v.properties.Private_key))
+ cmd.FlagWithInput("--key ", key)
+
+ algorithm := proptools.StringDefault(v.properties.Algorithm, "SHA256_RSA4096")
+ cmd.FlagWithArg("--algorithm ", algorithm)
+
+ cmd.FlagWithArg("--rollback_index ", v.rollbackIndexCommand(ctx))
+ ril := proptools.IntDefault(v.properties.Rollback_index_location, 0)
+ if ril < 0 {
+ ctx.PropertyErrorf("rollback_index_location", "must be 0, 1, 2, ...")
+ return
+ }
+ cmd.FlagWithArg("--rollback_index_location ", strconv.Itoa(ril))
+
+ for _, p := range ctx.GetDirectDepsWithTag(vbmetaPartitionDep) {
+ f, ok := p.(Filesystem)
+ if !ok {
+ ctx.PropertyErrorf("partitions", "%q(type: %s) is not supported",
+ p.Name(), ctx.OtherModuleType(p))
+ continue
+ }
+ signedImage := f.SignedOutputPath()
+ if signedImage == nil {
+ ctx.PropertyErrorf("partitions", "%q(type: %s) is not signed. Use `use_avb: true`",
+ p.Name(), ctx.OtherModuleType(p))
+ continue
+ }
+ cmd.FlagWithInput("--include_descriptors_from_image ", signedImage)
+ }
+
+ for i, cp := range v.properties.Chained_partitions {
+ name := proptools.String(cp.Name)
+ if name == "" {
+ ctx.PropertyErrorf("chained_partitions", "name must be specified")
+ continue
+ }
+
+ ril := proptools.IntDefault(cp.Rollback_index_location, i+1)
+ if ril < 0 {
+ ctx.PropertyErrorf("chained_partitions", "must be 0, 1, 2, ...")
+ continue
+ }
+
+ var publicKey android.Path
+ if cp.Public_key != nil {
+ publicKey = android.PathForModuleSrc(ctx, proptools.String(cp.Public_key))
+ } else {
+ publicKey = extractedPublicKeys[name]
+ }
+ cmd.FlagWithArg("--chain_partition ", fmt.Sprintf("%s:%d:%s", name, ril, publicKey.String()))
+ cmd.Implicit(publicKey)
+ }
+
+ cmd.FlagWithOutput("--output ", v.output)
+ builder.Build("vbmeta", fmt.Sprintf("vbmeta %s", ctx.ModuleName()))
+
+ v.installDir = android.PathForModuleInstall(ctx, "etc")
+ ctx.InstallFile(v.installDir, v.installFileName(), v.output)
+}
+
+// Returns the embedded shell command that prints the rollback index
+func (v *vbmeta) rollbackIndexCommand(ctx android.ModuleContext) string {
+ var cmd string
+ if v.properties.Rollback_index_file != nil {
+ f := android.PathForModuleSrc(ctx, proptools.String(v.properties.Rollback_index_file))
+ cmd = "cat " + f.String()
+ } else {
+ cmd = "date -d 'TZ=\"GMT\" " + ctx.Config().PlatformSecurityPatch() + "' +%s"
+ }
+ // Take the first line and remove the newline char
+ return "$(" + cmd + " | head -1 | tr -d '\n'" + ")"
+}
+
+// Extract public keys from chained_partitions.private_key. The keys are indexed with the partition
+// name.
+func (v *vbmeta) extractPublicKeys(ctx android.ModuleContext) map[string]android.OutputPath {
+ result := make(map[string]android.OutputPath)
+
+ builder := android.NewRuleBuilder(pctx, ctx)
+ for _, cp := range v.properties.Chained_partitions {
+ if cp.Private_key == nil {
+ continue
+ }
+
+ name := proptools.String(cp.Name)
+ if name == "" {
+ ctx.PropertyErrorf("chained_partitions", "name must be specified")
+ continue
+ }
+
+ if _, ok := result[name]; ok {
+ ctx.PropertyErrorf("chained_partitions", "name %q is duplicated", name)
+ continue
+ }
+
+ privateKeyFile := android.PathForModuleSrc(ctx, proptools.String(cp.Private_key))
+ publicKeyFile := android.PathForModuleOut(ctx, name+".avbpubkey").OutputPath
+
+ builder.Command().
+ BuiltTool("avbtool").
+ Text("extract_public_key").
+ FlagWithInput("--key ", privateKeyFile).
+ FlagWithOutput("--output ", publicKeyFile)
+
+ result[name] = publicKeyFile
+ }
+ builder.Build("vbmeta_extract_public_key", fmt.Sprintf("Extract public keys for %s", ctx.ModuleName()))
+ return result
+}
+
+var _ android.AndroidMkEntriesProvider = (*vbmeta)(nil)
+
+// Implements android.AndroidMkEntriesProvider
+func (v *vbmeta) AndroidMkEntries() []android.AndroidMkEntries {
+ return []android.AndroidMkEntries{android.AndroidMkEntries{
+ Class: "ETC",
+ OutputFile: android.OptionalPathForPath(v.output),
+ ExtraEntries: []android.AndroidMkExtraEntriesFunc{
+ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
+ entries.SetString("LOCAL_MODULE_PATH", v.installDir.ToMakePath().String())
+ entries.SetString("LOCAL_INSTALLED_MODULE_STEM", v.installFileName())
+ },
+ },
+ }}
+}
+
+var _ Filesystem = (*vbmeta)(nil)
+
+func (v *vbmeta) OutputPath() android.Path {
+ return v.output
+}
+
+func (v *vbmeta) SignedOutputPath() android.Path {
+ return v.OutputPath() // vbmeta is always signed
+}
+
+var _ android.OutputFileProducer = (*vbmeta)(nil)
+
+// Implements android.OutputFileProducer
+func (v *vbmeta) OutputFiles(tag string) (android.Paths, error) {
+ if tag == "" {
+ return []android.Path{v.output}, nil
+ }
+ return nil, fmt.Errorf("unsupported module reference tag %q", tag)
+}
diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go
index d131e94..199a7df 100644
--- a/genrule/genrule_test.go
+++ b/genrule/genrule_test.go
@@ -298,6 +298,14 @@
`,
expect: "echo foo > __SBOX_SANDBOX_DIR__/out/foo && cp __SBOX_SANDBOX_DIR__/out/foo __SBOX_SANDBOX_DIR__/out/out",
},
+ {
+ name: "$",
+ prop: `
+ out: ["out"],
+ cmd: "echo $$ > $(out)",
+ `,
+ expect: "echo $ > __SBOX_SANDBOX_DIR__/out/out",
+ },
{
name: "error empty location",
diff --git a/java/boot_image.go b/java/boot_image.go
index 12e2874..25a4f17 100644
--- a/java/boot_image.go
+++ b/java/boot_image.go
@@ -20,6 +20,7 @@
"android/soong/android"
"android/soong/dexpreopt"
+
"github.com/google/blueprint"
)
@@ -56,9 +57,9 @@
func bootImageFactory() android.Module {
m := &BootImageModule{}
m.AddProperties(&m.properties)
- android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibCommon)
android.InitApexModule(m)
android.InitSdkAwareModule(m)
+ android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibCommon)
return m
}
@@ -210,11 +211,11 @@
func prebuiltBootImageFactory() android.Module {
m := &prebuiltBootImageModule{}
m.AddProperties(&m.properties)
- android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibCommon)
// This doesn't actually have any prebuilt files of its own so pass a placeholder for the srcs
// array.
android.InitPrebuiltModule(m, &[]string{"placeholder"})
android.InitApexModule(m)
android.InitSdkAwareModule(m)
+ android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibCommon)
return m
}
diff --git a/java/droiddoc.go b/java/droiddoc.go
index f0decec..da13c62 100644
--- a/java/droiddoc.go
+++ b/java/droiddoc.go
@@ -812,7 +812,7 @@
BuiltTool("soong_javac_wrapper").Tool(config.JavadocCmd(ctx)).
Flag(config.JavacVmFlags).
FlagWithArg("-encoding ", "UTF-8").
- FlagWithRspFileInputList("@", srcs).
+ FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "javadoc.rsp"), srcs).
FlagWithInput("@", srcJarList)
// TODO(ccross): Remove this if- statement once we finish migration for all Doclava
@@ -1243,7 +1243,7 @@
Flag("-J--add-opens=java.base/java.util=ALL-UNNAMED").
FlagWithArg("-encoding ", "UTF-8").
FlagWithArg("-source ", javaVersion.String()).
- FlagWithRspFileInputList("@", srcs).
+ FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "metalava.rsp"), srcs).
FlagWithInput("@", srcJarList)
if javaHome := ctx.Config().Getenv("ANDROID_JAVA_HOME"); javaHome != "" {
@@ -1446,7 +1446,9 @@
// add a large number of inputs to a file without exceeding bash command length limits (which
// would happen if we use the WriteFile rule). The cp is needed because RuleBuilder sets the
// rsp file to be ${output}.rsp.
- impCmd.Text("cp").FlagWithRspFileInputList("", cmd.GetImplicits()).Output(implicitsRsp)
+ impCmd.Text("cp").
+ FlagWithRspFileInputList("", android.PathForModuleOut(ctx, "metalava-implicits.rsp"), cmd.GetImplicits()).
+ Output(implicitsRsp)
impRule.Build("implicitsGen", "implicits generation")
cmd.Implicit(implicitsRsp)
@@ -1750,7 +1752,7 @@
Flag("-jar").
FlagWithOutput("-o ", p.stubsSrcJar).
FlagWithArg("-C ", srcDir.String()).
- FlagWithRspFileInputList("-r ", srcPaths)
+ FlagWithRspFileInputList("-r ", p.stubsSrcJar.ReplaceExtension(ctx, "rsp"), srcPaths)
rule.Restat()
diff --git a/java/kotlin.go b/java/kotlin.go
index 8067ad5..2960f81 100644
--- a/java/kotlin.go
+++ b/java/kotlin.go
@@ -64,7 +64,9 @@
// Insert a second rule to write out the list of resources to a file.
commonSrcsList := android.PathForModuleOut(ctx, "kotlinc_common_srcs.list")
rule := android.NewRuleBuilder(pctx, ctx)
- rule.Command().Text("cp").FlagWithRspFileInputList("", commonSrcFiles).Output(commonSrcsList)
+ rule.Command().Text("cp").
+ FlagWithRspFileInputList("", commonSrcsList.ReplaceExtension(ctx, "rsp"), commonSrcFiles).
+ Output(commonSrcsList)
rule.Build("kotlin_common_srcs_list", "kotlin common_srcs list")
return android.OptionalPathForPath(commonSrcsList)
}
diff --git a/java/lint.go b/java/lint.go
index 9f677db..fccd1a5 100644
--- a/java/lint.go
+++ b/java/lint.go
@@ -220,7 +220,9 @@
// Insert a second rule to write out the list of resources to a file.
resourcesList = android.PathForModuleOut(ctx, "lint", "resources.list")
resListRule := android.NewRuleBuilder(pctx, ctx)
- resListRule.Command().Text("cp").FlagWithRspFileInputList("", l.resources).Output(resourcesList)
+ resListRule.Command().Text("cp").
+ FlagWithRspFileInputList("", resourcesList.ReplaceExtension(ctx, "rsp"), l.resources).
+ Output(resourcesList)
resListRule.Build("lint_resources_list", "lint resources list")
trackRSPDependency(l.resources, resourcesList)
}
@@ -241,7 +243,10 @@
// TODO(ccross): some of the files in l.srcs are generated sources and should be passed to
// lint separately.
srcsList := android.PathForModuleOut(ctx, "lint", "srcs.list")
- rule.Command().Text("cp").FlagWithRspFileInputList("", l.srcs).Output(srcsList)
+ srcsListRsp := android.PathForModuleOut(ctx, "lint-srcs.list.rsp")
+ rule.Command().Text("cp").
+ FlagWithRspFileInputList("", srcsListRsp, l.srcs).
+ Output(srcsList)
trackRSPDependency(l.srcs, srcsList)
cmd := rule.Command().
@@ -635,7 +640,7 @@
rule.Command().BuiltTool("soong_zip").
FlagWithOutput("-o ", outputPath).
FlagWithArg("-C ", android.PathForIntermediates(ctx).String()).
- FlagWithRspFileInputList("-r ", paths)
+ FlagWithRspFileInputList("-r ", outputPath.ReplaceExtension(ctx, "rsp"), paths)
rule.Build(outputPath.Base(), outputPath.Base())
}
diff --git a/rust/bindgen.go b/rust/bindgen.go
index 56d660e..db69e23 100644
--- a/rust/bindgen.go
+++ b/rust/bindgen.go
@@ -29,7 +29,7 @@
defaultBindgenFlags = []string{""}
// bindgen should specify its own Clang revision so updating Clang isn't potentially blocked on bindgen failures.
- bindgenClangVersion = "clang-r399163b"
+ bindgenClangVersion = "clang-r412851"
_ = pctx.VariableFunc("bindgenClangVersion", func(ctx android.PackageVarContext) string {
if override := ctx.Config().Getenv("LLVM_BINDGEN_PREBUILTS_VERSION"); override != "" {
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 32b6eda..d17b464 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -32,6 +32,8 @@
name: "soong-ui-build",
pkgPath: "android/soong/ui/build",
deps: [
+ "blueprint",
+ "blueprint-bootstrap",
"soong-ui-build-paths",
"soong-ui-logger",
"soong-ui-metrics",
diff --git a/ui/build/build.go b/ui/build/build.go
index 215a6c8..3692f4f 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -218,6 +218,11 @@
what = what &^ BuildKati
}
+ if config.SkipNinja() {
+ ctx.Verboseln("Skipping Ninja as requested")
+ what = what &^ BuildNinja
+ }
+
if config.StartGoma() {
// Ensure start Goma compiler_proxy
startGoma(ctx, config)
@@ -290,7 +295,7 @@
}
// Run ninja
- runNinja(ctx, config)
+ runNinjaForBuild(ctx, config)
}
// Currently, using Bazel requires Kati and Soong to run first, so check whether to run Bazel last.
diff --git a/ui/build/config.go b/ui/build/config.go
index 1152cd7..4816d1f 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -48,6 +48,7 @@
dist bool
skipConfig bool
skipKati bool
+ skipNinja bool
skipSoongTests bool
// From the product config
@@ -552,6 +553,8 @@
if arg == "--make-mode" {
} else if arg == "showcommands" {
c.verbose = true
+ } else if arg == "--skip-ninja" {
+ c.skipNinja = true
} else if arg == "--skip-make" {
c.skipConfig = true
c.skipKati = true
@@ -772,6 +775,10 @@
return c.skipKati
}
+func (c *configImpl) SkipNinja() bool {
+ return c.skipNinja
+}
+
func (c *configImpl) SkipConfig() bool {
return c.skipConfig
}
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index 7799766..893fd6d 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -30,7 +30,7 @@
// Constructs and runs the Ninja command line with a restricted set of
// environment variables. It's important to restrict the environment Ninja runs
// for hermeticity reasons, and to avoid spurious rebuilds.
-func runNinja(ctx Context, config Config) {
+func runNinjaForBuild(ctx Context, config Config) {
ctx.BeginTrace(metrics.PrimaryNinja, "ninja")
defer ctx.EndTrace()
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 884e957..fee5723 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -23,6 +23,8 @@
"android/soong/shared"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/bootstrap"
"github.com/golang/protobuf/proto"
"github.com/google/blueprint/microfactory"
@@ -42,17 +44,72 @@
// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
//
-// However, the execution of <builddir>/build.ninja happens later in build/soong/ui/build/build.go#Build()
+// However, the execution of <builddir>/build.ninja happens later in
+// build/soong/ui/build/build.go#Build()
//
-// We want to rely on as few prebuilts as possible, so there is some bootstrapping here.
+// We want to rely on as few prebuilts as possible, so we need to bootstrap
+// Soong. The process is as follows:
//
-// "Microfactory" is a tool for compiling Go code. We use it to build two other tools:
-// - minibp, used to generate build.ninja files. This is really build/blueprint/bootstrap/command.go#Main()
-// - bpglob, used during incremental builds to identify files in a glob that have changed
+// 1. We use "Microfactory", a simple tool to compile Go code, to build
+// first itself, then soong_ui from soong_ui.bash. This binary contains
+// parts of soong_build that are needed to build itself.
+// 2. This simplified version of soong_build then reads the Blueprint files
+// that describe itself and emits .bootstrap/build.ninja that describes
+// how to build its full version and use that to produce the final Ninja
+// file Soong emits.
+// 3. soong_ui executes .bootstrap/build.ninja
//
-// In reality, several build.ninja files are generated and/or used during the bootstrapping and build process.
-// See build/blueprint/bootstrap/doc.go for more information.
-//
+// (After this, Kati is executed to parse the Makefiles, but that's not part of
+// bootstrapping Soong)
+
+// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
+// probably be nicer to use a flag in bootstrap.Args instead.
+type BlueprintConfig struct {
+ srcDir string
+ buildDir string
+ ninjaBuildDir string
+}
+
+func (c BlueprintConfig) SrcDir() string {
+ return "."
+}
+
+func (c BlueprintConfig) BuildDir() string {
+ return c.buildDir
+}
+
+func (c BlueprintConfig) NinjaBuildDir() string {
+ return c.ninjaBuildDir
+}
+
+func bootstrapBlueprint(ctx Context, config Config) {
+ ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
+ defer ctx.EndTrace()
+
+ var args bootstrap.Args
+
+ args.RunGoTests = !config.skipSoongTests
+ args.UseValidations = true // Use validations to depend on tests
+ args.BuildDir = config.SoongOutDir()
+ args.NinjaBuildDir = config.OutDir()
+ args.TopFile = "Android.bp"
+ args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
+ args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
+ args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
+ args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+ args.GeneratingPrimaryBuilder = true
+
+ blueprintCtx := blueprint.NewContext()
+ blueprintCtx.SetIgnoreUnknownModuleTypes(true)
+ blueprintConfig := BlueprintConfig{
+ srcDir: os.Getenv("TOP"),
+ buildDir: config.SoongOutDir(),
+ ninjaBuildDir: config.OutDir(),
+ }
+
+ bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
+}
+
func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "soong")
defer ctx.EndTrace()
@@ -63,33 +120,15 @@
// unused variables were changed?
envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
- // Use an anonymous inline function for tracing purposes (this pattern is used several times below).
- func() {
- ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
- defer ctx.EndTrace()
-
- // Use validations to depend on tests.
- args := []string{"-n"}
-
- if !config.skipSoongTests {
- // Run tests.
- args = append(args, "-t")
+ for _, n := range []string{".bootstrap", ".minibootstrap"} {
+ dir := filepath.Join(config.SoongOutDir(), n)
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ ctx.Fatalf("Cannot mkdir " + dir)
}
+ }
- cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", args...)
-
- cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint")
- cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash")
- cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
- cmd.Environment.Set("GOROOT", "./"+filepath.Join("prebuilts/go", config.HostPrebuiltTag()))
- cmd.Environment.Set("BLUEPRINT_LIST_FILE", filepath.Join(config.FileListDir(), "Android.bp.list"))
- cmd.Environment.Set("NINJA_BUILDDIR", config.OutDir())
- cmd.Environment.Set("SRCDIR", ".")
- cmd.Environment.Set("TOPNAME", "Android.bp")
- cmd.Sandbox = soongSandbox
-
- cmd.RunAndPrintOrFatal()
- }()
+ // This is done unconditionally, but does not take a measurable amount of time
+ bootstrapBlueprint(ctx, config)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -105,6 +144,11 @@
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
+ // For Soong bootstrapping tests
+ if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
+ soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
+ }
+
err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
if err != nil {
ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
@@ -130,16 +174,6 @@
cfg.TrimPath = absPath(ctx, ".")
func() {
- ctx.BeginTrace(metrics.RunSoong, "minibp")
- defer ctx.EndTrace()
-
- minibp := filepath.Join(config.SoongOutDir(), ".minibootstrap/minibp")
- if _, err := microfactory.Build(&cfg, minibp, "github.com/google/blueprint/bootstrap/minibp"); err != nil {
- ctx.Fatalln("Failed to build minibp:", err)
- }
- }()
-
- func() {
ctx.BeginTrace(metrics.RunSoong, "bpglob")
defer ctx.EndTrace()
@@ -187,10 +221,6 @@
cmd.Sandbox = soongSandbox
cmd.RunAndStreamOrFatal()
}
-
- // This build generates .bootstrap/build.ninja, which is used in the next step.
- ninja("minibootstrap", ".minibootstrap/build.ninja")
-
// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
ninja("bootstrap", ".bootstrap/build.ninja")