Add a Go replacement for our top-level Make wrapper
Right now this mostly just copies what Make is doing in
build/core/ninja.mk and build/core/soong.mk. The only major feature it
adds is a rotating log file with some verbose logging.
There is one major functional difference -- you cannot override random
Make variables during the Make phase anymore. The environment variable
is set, and if Make uses ?= or the equivalent, it can still use those
variables. We already made this change for Kati, which also loads all of
the same code and actually does the build, so it has been half-removed
for a while.
The only "UI" this implements is what I'll call "Make Emulation" mode --
it's expected that current command lines will continue working, and
we'll explore alternate user interfaces later.
We're still using Make as a wrapper, but all it does is call into this
single Go program, it won't even load the product configuration. Once
this is default, we can start moving individual users over to using this
directly (still in Make emulation mode), skipping the Make wrapper.
Ideas for the future:
* Generating trace files showing time spent in Make/Kati/Soong/Ninja
(also importing ninja traces into the same stream). I had this working
in a previous version of this patch, but removed it to keep the size
down and focus on the current features.
* More intelligent SIGALRM handling, once we fully remove the Make
wrapper (which hides the SIGALRM)
* Reading the experimental binary output stream from Ninja, so that we
can always save the verbose log even if we're not printing it out to
the console
Test: USE_SOONG_UI=true m -j blueprint_tools
Change-Id: I884327b9a8ae24499eb6c56f6e1ad26df1cfa4e4
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
new file mode 100644
index 0000000..d6da950
--- /dev/null
+++ b/ui/build/Android.bp
@@ -0,0 +1,36 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+bootstrap_go_package {
+ name: "soong-ui-build",
+ pkgPath: "android/soong/ui/build",
+ deps: [
+ "soong-ui-logger",
+ ],
+ srcs: [
+ "build.go",
+ "config.go",
+ "context.go",
+ "environment.go",
+ "kati.go",
+ "make.go",
+ "ninja.go",
+ "signal.go",
+ "soong.go",
+ "util.go",
+ ],
+ testSrcs: [
+ "environment_test.go",
+ ],
+}
diff --git a/ui/build/build.go b/ui/build/build.go
new file mode 100644
index 0000000..506ff51
--- /dev/null
+++ b/ui/build/build.go
@@ -0,0 +1,105 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "text/template"
+)
+
+// Ensures the out directory exists, and has the proper files to prevent kati
+// from recursing into it.
+func SetupOutDir(ctx Context, config Config) {
+ ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "Android.mk"))
+ ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "CleanSpec.mk"))
+ ensureEmptyFileExists(ctx, filepath.Join(config.SoongOutDir(), ".soong.in_make"))
+ // The ninja_build file is used by our buildbots to understand that the output
+ // can be parsed as ninja output.
+ ensureEmptyFileExists(ctx, filepath.Join(config.OutDir(), "ninja_build"))
+}
+
+var combinedBuildNinjaTemplate = template.Must(template.New("combined").Parse(`
+builddir = {{.OutDir}}
+include {{.KatiNinjaFile}}
+include {{.SoongNinjaFile}}
+build {{.CombinedNinjaFile}}: phony {{.SoongNinjaFile}}
+`))
+
+func createCombinedBuildNinjaFile(ctx Context, config Config) {
+ file, err := os.Create(config.CombinedNinjaFile())
+ if err != nil {
+ ctx.Fatalln("Failed to create combined ninja file:", err)
+ }
+ defer file.Close()
+
+ if err := combinedBuildNinjaTemplate.Execute(file, config); err != nil {
+ ctx.Fatalln("Failed to write combined ninja file:", err)
+ }
+}
+
+const (
+ BuildNone = iota
+ BuildProductConfig = 1 << iota
+ BuildSoong = 1 << iota
+ BuildKati = 1 << iota
+ BuildNinja = 1 << iota
+ BuildAll = BuildProductConfig | BuildSoong | BuildKati | BuildNinja
+)
+
+// Build the tree. The 'what' argument can be used to chose which components of
+// the build to run.
+func Build(ctx Context, config Config, what int) {
+ ctx.Verboseln("Starting build with args:", config.Arguments())
+ ctx.Verboseln("Environment:", config.Environment().Environ())
+
+ if inList("help", config.Arguments()) {
+ cmd := exec.CommandContext(ctx.Context, "make", "-f", "build/core/help.mk")
+ cmd.Env = config.Environment().Environ()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ if err := cmd.Run(); err != nil {
+ ctx.Fatalln("Failed to run make:", err)
+ }
+ return
+ }
+
+ SetupOutDir(ctx, config)
+
+ if what&BuildProductConfig != 0 {
+ // Run make for product config
+ runMakeProductConfig(ctx, config)
+ }
+
+ if what&BuildSoong != 0 {
+ // Run Soong
+ runSoongBootstrap(ctx, config)
+ runSoong(ctx, config)
+ }
+
+ if what&BuildKati != 0 {
+ // Run ckati
+ runKati(ctx, config)
+ }
+
+ if what&BuildNinja != 0 {
+ // Write combined ninja file
+ createCombinedBuildNinjaFile(ctx, config)
+
+ // Run ninja
+ runNinja(ctx, config)
+ }
+}
diff --git a/ui/build/config.go b/ui/build/config.go
new file mode 100644
index 0000000..b0a7d7a
--- /dev/null
+++ b/ui/build/config.go
@@ -0,0 +1,274 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+)
+
+type Config struct{ *configImpl }
+
+type configImpl struct {
+ // From the environment
+ arguments []string
+ goma bool
+ environ *Environment
+
+ // From the arguments
+ parallel int
+ keepGoing int
+ verbose bool
+
+ // From the product config
+ katiArgs []string
+ ninjaArgs []string
+ katiSuffix string
+}
+
+func NewConfig(ctx Context, args ...string) Config {
+ ret := &configImpl{
+ environ: OsEnvironment(),
+ }
+
+ ret.environ.Unset(
+ // We're already using it
+ "USE_SOONG_UI",
+
+ // We should never use GOROOT/GOPATH from the shell environment
+ "GOROOT",
+ "GOPATH",
+
+ // These should only come from Soong, not the environment.
+ "CLANG",
+ "CLANG_CXX",
+ "CCC_CC",
+ "CCC_CXX",
+
+ // Used by the goma compiler wrapper, but should only be set by
+ // gomacc
+ "GOMACC_PATH",
+ )
+
+ // Tell python not to spam the source tree with .pyc files.
+ ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
+
+ // Sane default matching ninja
+ ret.parallel = runtime.NumCPU() + 2
+ ret.keepGoing = 1
+
+ for _, arg := range args {
+ arg = strings.TrimSpace(arg)
+ if arg == "--make-mode" {
+ continue
+ } else if arg == "showcommands" {
+ ret.verbose = true
+ continue
+ }
+ if arg[0] == '-' {
+ var err error
+ if arg[1] == 'j' {
+ // TODO: handle space between j and number
+ // Unnecessary if used with makeparallel
+ ret.parallel, err = strconv.Atoi(arg[2:])
+ } else if arg[1] == 'k' {
+ // TODO: handle space between k and number
+ // Unnecessary if used with makeparallel
+ ret.keepGoing, err = strconv.Atoi(arg[2:])
+ } else {
+ ctx.Fatalln("Unknown option:", arg)
+ }
+ if err != nil {
+ ctx.Fatalln("Argument error:", err, arg)
+ }
+ } else {
+ ret.arguments = append(ret.arguments, arg)
+ }
+ }
+
+ return Config{ret}
+}
+
+// Lunch configures the environment for a specific product similarly to the
+// `lunch` bash function.
+func (c *configImpl) Lunch(ctx Context, product, variant string) {
+ if variant != "eng" && variant != "userdebug" && variant != "user" {
+ ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
+ }
+
+ c.environ.Set("TARGET_PRODUCT", product)
+ c.environ.Set("TARGET_BUILD_VARIANT", variant)
+ c.environ.Set("TARGET_BUILD_TYPE", "release")
+ c.environ.Unset("TARGET_BUILD_APPS")
+}
+
+// Tapas configures the environment to build one or more unbundled apps,
+// similarly to the `tapas` bash function.
+func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) {
+ if len(apps) == 0 {
+ apps = []string{"all"}
+ }
+ if variant == "" {
+ variant = "eng"
+ }
+
+ if variant != "eng" && variant != "userdebug" && variant != "user" {
+ ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
+ }
+
+ var product string
+ switch arch {
+ case "armv5":
+ product = "generic_armv5"
+ case "arm", "":
+ product = "aosp_arm"
+ case "arm64":
+ product = "aosm_arm64"
+ case "mips":
+ product = "aosp_mips"
+ case "mips64":
+ product = "aosp_mips64"
+ case "x86":
+ product = "aosp_x86"
+ case "x86_64":
+ product = "aosp_x86_64"
+ default:
+ ctx.Fatalf("Invalid architecture: %q", arch)
+ }
+
+ c.environ.Set("TARGET_PRODUCT", product)
+ c.environ.Set("TARGET_BUILD_VARIANT", variant)
+ c.environ.Set("TARGET_BUILD_TYPE", "release")
+ c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " "))
+}
+
+func (c *configImpl) Environment() *Environment {
+ return c.environ
+}
+
+func (c *configImpl) Arguments() []string {
+ return c.arguments
+}
+
+func (c *configImpl) OutDir() string {
+ if outDir, ok := c.environ.Get("OUT_DIR"); ok {
+ return outDir
+ }
+ return "out"
+}
+
+func (c *configImpl) NinjaArgs() []string {
+ return c.ninjaArgs
+}
+
+func (c *configImpl) SoongOutDir() string {
+ return filepath.Join(c.OutDir(), "soong")
+}
+
+func (c *configImpl) KatiSuffix() string {
+ if c.katiSuffix != "" {
+ return c.katiSuffix
+ }
+ panic("SetKatiSuffix has not been called")
+}
+
+func (c *configImpl) IsVerbose() bool {
+ return c.verbose
+}
+
+func (c *configImpl) TargetProduct() string {
+ if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
+ return v
+ }
+ panic("TARGET_PRODUCT is not defined")
+}
+
+func (c *configImpl) KatiArgs() []string {
+ return c.katiArgs
+}
+
+func (c *configImpl) Parallel() int {
+ return c.parallel
+}
+
+func (c *configImpl) UseGoma() bool {
+ if v, ok := c.environ.Get("USE_GOMA"); ok {
+ v = strings.TrimSpace(v)
+ if v != "" && v != "false" {
+ return true
+ }
+ }
+ return false
+}
+
+// RemoteParallel controls how many remote jobs (i.e., commands which contain
+// gomacc) are run in parallel. Note the paralleism of all other jobs is
+// still limited by Parallel()
+func (c *configImpl) RemoteParallel() int {
+ if v, ok := c.environ.Get("NINJA_REMOTE_NUM_JOBS"); ok {
+ if i, err := strconv.Atoi(v); err == nil {
+ return i
+ }
+ }
+ return 500
+}
+
+func (c *configImpl) SetKatiArgs(args []string) {
+ c.katiArgs = args
+}
+
+func (c *configImpl) SetNinjaArgs(args []string) {
+ c.ninjaArgs = args
+}
+
+func (c *configImpl) SetKatiSuffix(suffix string) {
+ c.katiSuffix = suffix
+}
+
+func (c *configImpl) KatiEnvFile() string {
+ return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
+}
+
+func (c *configImpl) KatiNinjaFile() string {
+ return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+".ninja")
+}
+
+func (c *configImpl) SoongNinjaFile() string {
+ return filepath.Join(c.SoongOutDir(), "build.ninja")
+}
+
+func (c *configImpl) CombinedNinjaFile() string {
+ return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
+}
+
+func (c *configImpl) SoongAndroidMk() string {
+ return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
+}
+
+func (c *configImpl) SoongMakeVarsMk() string {
+ return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
+}
+
+func (c *configImpl) HostPrebuiltTag() string {
+ if runtime.GOOS == "linux" {
+ return "linux-x86"
+ } else if runtime.GOOS == "darwin" {
+ return "darwin-x86"
+ } else {
+ panic("Unsupported OS")
+ }
+}
diff --git a/ui/build/context.go b/ui/build/context.go
new file mode 100644
index 0000000..59474f5
--- /dev/null
+++ b/ui/build/context.go
@@ -0,0 +1,64 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "context"
+ "io"
+ "os"
+
+ "android/soong/ui/logger"
+)
+
+type StdioInterface interface {
+ Stdin() io.Reader
+ Stdout() io.Writer
+ Stderr() io.Writer
+}
+
+type StdioImpl struct{}
+
+func (StdioImpl) Stdin() io.Reader { return os.Stdin }
+func (StdioImpl) Stdout() io.Writer { return os.Stdout }
+func (StdioImpl) Stderr() io.Writer { return os.Stderr }
+
+var _ StdioInterface = StdioImpl{}
+
+type customStdio struct {
+ stdin io.Reader
+ stdout io.Writer
+ stderr io.Writer
+}
+
+func NewCustomStdio(stdin io.Reader, stdout, stderr io.Writer) StdioInterface {
+ return customStdio{stdin, stdout, stderr}
+}
+
+func (c customStdio) Stdin() io.Reader { return c.stdin }
+func (c customStdio) Stdout() io.Writer { return c.stdout }
+func (c customStdio) Stderr() io.Writer { return c.stderr }
+
+var _ StdioInterface = customStdio{}
+
+// Context combines a context.Context, logger.Logger, and StdIO redirection.
+// These all are agnostic of the current build, and may be used for multiple
+// builds, while the Config objects contain per-build information.
+type Context *ContextImpl
+type ContextImpl struct {
+ context.Context
+ logger.Logger
+
+ StdioInterface
+}
diff --git a/ui/build/environment.go b/ui/build/environment.go
new file mode 100644
index 0000000..baab101
--- /dev/null
+++ b/ui/build/environment.go
@@ -0,0 +1,152 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+)
+
+// Environment adds a number of useful manipulation functions to the list of
+// strings returned by os.Environ() and used in exec.Cmd.Env.
+type Environment []string
+
+// OsEnvironment wraps the current environment returned by os.Environ()
+func OsEnvironment() *Environment {
+ env := Environment(os.Environ())
+ return &env
+}
+
+// Get returns the value associated with the key, and whether it exists.
+// It's equivalent to the os.LookupEnv function, but with this copy of the
+// Environment.
+func (e *Environment) Get(key string) (string, bool) {
+ for _, env := range *e {
+ if k, v, ok := decodeKeyValue(env); ok && k == key {
+ return v, true
+ }
+ }
+ return "", false
+}
+
+// Set sets the value associated with the key, overwriting the current value
+// if it exists.
+func (e *Environment) Set(key, value string) {
+ e.Unset(key)
+ *e = append(*e, key+"="+value)
+}
+
+// Unset removes the specified keys from the Environment.
+func (e *Environment) Unset(keys ...string) {
+ out := (*e)[:0]
+ for _, env := range *e {
+ if key, _, ok := decodeKeyValue(env); ok && inList(key, keys) {
+ continue
+ }
+ out = append(out, env)
+ }
+ *e = out
+}
+
+// Environ returns the []string required for exec.Cmd.Env
+func (e *Environment) Environ() []string {
+ return []string(*e)
+}
+
+// Copy returns a copy of the Environment so that independent changes may be made.
+func (e *Environment) Copy() *Environment {
+ ret := Environment(make([]string, len(*e)))
+ for i, v := range *e {
+ ret[i] = v
+ }
+ return &ret
+}
+
+// IsTrue returns whether an environment variable is set to a positive value (1,y,yes,on,true)
+func (e *Environment) IsEnvTrue(key string) bool {
+ if value, ok := e.Get(key); ok {
+ return value == "1" || value == "y" || value == "yes" || value == "on" || value == "true"
+ }
+ return false
+}
+
+// IsFalse returns whether an environment variable is set to a negative value (0,n,no,off,false)
+func (e *Environment) IsFalse(key string) bool {
+ if value, ok := e.Get(key); ok {
+ return value == "0" || value == "n" || value == "no" || value == "off" || value == "false"
+ }
+ return false
+}
+
+// AppendFromKati reads a shell script written by Kati that exports or unsets
+// environment variables, and applies those to the local Environment.
+func (e *Environment) AppendFromKati(filename string) error {
+ file, err := os.Open(filename)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ return e.appendFromKati(file)
+}
+
+func (e *Environment) appendFromKati(reader io.Reader) error {
+ scanner := bufio.NewScanner(reader)
+ for scanner.Scan() {
+ text := strings.TrimSpace(scanner.Text())
+
+ if len(text) == 0 || text[0] == '#' {
+ continue
+ }
+
+ cmd := strings.SplitN(text, " ", 2)
+ if len(cmd) != 2 {
+ return fmt.Errorf("Unknown kati environment line: %q", text)
+ }
+
+ if cmd[0] == "unset" {
+ str, ok := singleUnquote(cmd[1])
+ if !ok {
+ fmt.Errorf("Failed to unquote kati line: %q", text)
+ }
+ e.Unset(str)
+ } else if cmd[0] == "export" {
+ key, value, ok := decodeKeyValue(cmd[1])
+ if !ok {
+ return fmt.Errorf("Failed to parse export: %v", cmd)
+ }
+
+ key, ok = singleUnquote(key)
+ if !ok {
+ return fmt.Errorf("Failed to unquote kati line: %q", text)
+ }
+ value, ok = singleUnquote(value)
+ if !ok {
+ return fmt.Errorf("Failed to unquote kati line: %q", text)
+ }
+
+ e.Set(key, value)
+ } else {
+ return fmt.Errorf("Unknown kati environment command: %q", text)
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/ui/build/environment_test.go b/ui/build/environment_test.go
new file mode 100644
index 0000000..0294dac
--- /dev/null
+++ b/ui/build/environment_test.go
@@ -0,0 +1,80 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestEnvUnset(t *testing.T) {
+ initial := &Environment{"TEST=1", "TEST2=0"}
+ initial.Unset("TEST")
+ got := initial.Environ()
+ if len(got) != 1 || got[0] != "TEST2=0" {
+ t.Errorf("Expected [TEST2=0], got: %v", got)
+ }
+}
+
+func TestEnvUnsetMissing(t *testing.T) {
+ initial := &Environment{"TEST2=0"}
+ initial.Unset("TEST")
+ got := initial.Environ()
+ if len(got) != 1 || got[0] != "TEST2=0" {
+ t.Errorf("Expected [TEST2=0], got: %v", got)
+ }
+}
+
+func TestEnvSet(t *testing.T) {
+ initial := &Environment{}
+ initial.Set("TEST", "0")
+ got := initial.Environ()
+ if len(got) != 1 || got[0] != "TEST=0" {
+ t.Errorf("Expected [TEST=0], got: %v", got)
+ }
+}
+
+func TestEnvSetDup(t *testing.T) {
+ initial := &Environment{"TEST=1"}
+ initial.Set("TEST", "0")
+ got := initial.Environ()
+ if len(got) != 1 || got[0] != "TEST=0" {
+ t.Errorf("Expected [TEST=0], got: %v", got)
+ }
+}
+
+const testKatiEnvFileContents = `#!/bin/sh
+# Generated by kati unknown
+
+unset 'CLANG'
+export 'BUILD_ID'='NYC'
+`
+
+func TestEnvAppendFromKati(t *testing.T) {
+ initial := &Environment{"CLANG=/usr/bin/clang", "TEST=0"}
+ err := initial.appendFromKati(strings.NewReader(testKatiEnvFileContents))
+ if err != nil {
+ t.Fatalf("Unexpected error from %v", err)
+ }
+
+ got := initial.Environ()
+ expected := []string{"TEST=0", "BUILD_ID=NYC"}
+ if !reflect.DeepEqual(got, expected) {
+ t.Errorf("Environment list does not match")
+ t.Errorf("expected: %v", expected)
+ t.Errorf(" got: %v", got)
+ }
+}
diff --git a/ui/build/kati.go b/ui/build/kati.go
new file mode 100644
index 0000000..6997fbe
--- /dev/null
+++ b/ui/build/kati.go
@@ -0,0 +1,104 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "crypto/md5"
+ "fmt"
+ "io/ioutil"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+)
+
+var spaceSlashReplacer = strings.NewReplacer("/", "_", " ", "_")
+
+// genKatiSuffix creates a suffix for kati-generated files so that we can cache
+// them based on their inputs. So this should encode all common changes to Kati
+// inputs. Currently that includes the TARGET_PRODUCT, kati-processed command
+// line arguments, and the directories specified by mm/mmm.
+func genKatiSuffix(ctx Context, config Config) {
+ katiSuffix := "-" + config.TargetProduct()
+ if args := config.KatiArgs(); len(args) > 0 {
+ katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_"))
+ }
+ if oneShot, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok {
+ katiSuffix += "-" + spaceSlashReplacer.Replace(oneShot)
+ }
+
+ // If the suffix is too long, replace it with a md5 hash and write a
+ // file that contains the original suffix.
+ if len(katiSuffix) > 64 {
+ shortSuffix := "-" + fmt.Sprintf("%x", md5.Sum([]byte(katiSuffix)))
+ config.SetKatiSuffix(shortSuffix)
+
+ ctx.Verbosef("Kati ninja suffix too long: %q", katiSuffix)
+ ctx.Verbosef("Replacing with: %q", shortSuffix)
+
+ if err := ioutil.WriteFile(strings.TrimSuffix(config.KatiNinjaFile(), "ninja")+"suf", []byte(katiSuffix), 0777); err != nil {
+ ctx.Println("Error writing suffix file:", err)
+ }
+ } else {
+ config.SetKatiSuffix(katiSuffix)
+ }
+}
+
+func runKati(ctx Context, config Config) {
+ genKatiSuffix(ctx, config)
+
+ executable := "prebuilts/build-tools/" + config.HostPrebuiltTag() + "/bin/ckati"
+ args := []string{
+ "--ninja",
+ "--ninja_dir=" + config.OutDir(),
+ "--ninja_suffix=" + config.KatiSuffix(),
+ "--regen",
+ "--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"),
+ "--detect_android_echo",
+ }
+
+ if !config.Environment().IsFalse("KATI_EMULATE_FIND") {
+ args = append(args, "--use_find_emulator")
+ }
+
+ // The argument order could be simplified, but currently this matches
+ // the ordering in Make
+ args = append(args, "-f", "build/core/main.mk")
+
+ args = append(args, config.KatiArgs()...)
+
+ args = append(args,
+ "--gen_all_targets",
+ "BUILDING_WITH_NINJA=true",
+ "SOONG_ANDROID_MK="+config.SoongAndroidMk(),
+ "SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk())
+
+ if config.UseGoma() {
+ args = append(args, "-j"+strconv.Itoa(config.Parallel()))
+ }
+
+ cmd := exec.CommandContext(ctx.Context, executable, args...)
+ cmd.Env = config.Environment().Environ()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ ctx.Verboseln(cmd.Path, cmd.Args)
+ if err := cmd.Run(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ ctx.Fatalln("ckati failed with:", e.ProcessState.String())
+ } else {
+ ctx.Fatalln("Failed to run ckati:", err)
+ }
+ }
+}
diff --git a/ui/build/make.go b/ui/build/make.go
new file mode 100644
index 0000000..5880509
--- /dev/null
+++ b/ui/build/make.go
@@ -0,0 +1,160 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "fmt"
+ "os/exec"
+ "path/filepath"
+ "strings"
+)
+
+// DumpMakeVars can be used to extract the values of Make variables after the
+// product configurations are loaded. This is roughly equivalent to the
+// `get_build_var` bash function.
+//
+// goals can be used to set MAKECMDGOALS, which emulates passing arguments to
+// Make without actually building them. So all the variables based on
+// MAKECMDGOALS can be read.
+//
+// extra_targets adds real arguments to the make command, in case other targets
+// actually need to be run (like the Soong config generator).
+//
+// vars is the list of variables to read. The values will be put in the
+// returned map.
+func DumpMakeVars(ctx Context, config Config, goals, extra_targets, vars []string) (map[string]string, error) {
+ cmd := exec.CommandContext(ctx.Context,
+ "make",
+ "--no-print-directory",
+ "-f", "build/core/config.mk",
+ "dump-many-vars",
+ "CALLED_FROM_SETUP=true",
+ "BUILD_SYSTEM=build/core",
+ "MAKECMDGOALS="+strings.Join(goals, " "),
+ "DUMP_MANY_VARS="+strings.Join(vars, " "),
+ "OUT_DIR="+config.OutDir())
+ cmd.Env = config.Environment().Environ()
+ cmd.Args = append(cmd.Args, extra_targets...)
+ // TODO: error out when Stderr contains any content
+ cmd.Stderr = ctx.Stderr()
+ ctx.Verboseln(cmd.Path, cmd.Args)
+ output, err := cmd.Output()
+ if err != nil {
+ return nil, err
+ }
+
+ ret := make(map[string]string, len(vars))
+ for _, line := range strings.Split(string(output), "\n") {
+ if len(line) == 0 {
+ continue
+ }
+
+ if key, value, ok := decodeKeyValue(line); ok {
+ if value, ok = singleUnquote(value); ok {
+ ret[key] = value
+ ctx.Verboseln(key, value)
+ } else {
+ return nil, fmt.Errorf("Failed to parse make line: %q", line)
+ }
+ } else {
+ return nil, fmt.Errorf("Failed to parse make line: %q", line)
+ }
+ }
+
+ return ret, nil
+}
+
+func runMakeProductConfig(ctx Context, config Config) {
+ // Variables to export into the environment of Kati/Ninja
+ exportEnvVars := []string{
+ // So that we can use the correct TARGET_PRODUCT if it's been
+ // modified by PRODUCT-* arguments
+ "TARGET_PRODUCT",
+
+ // compiler wrappers set up by make
+ "CC_WRAPPER",
+ "CXX_WRAPPER",
+
+ // ccache settings
+ "CCACHE_COMPILERCHECK",
+ "CCACHE_SLOPPINESS",
+ "CCACHE_BASEDIR",
+ "CCACHE_CPP2",
+ }
+
+ // Variables to print out in the top banner
+ bannerVars := []string{
+ "PLATFORM_VERSION_CODENAME",
+ "PLATFORM_VERSION",
+ "TARGET_PRODUCT",
+ "TARGET_BUILD_VARIANT",
+ "TARGET_BUILD_TYPE",
+ "TARGET_BUILD_APPS",
+ "TARGET_ARCH",
+ "TARGET_ARCH_VARIANT",
+ "TARGET_CPU_VARIANT",
+ "TARGET_2ND_ARCH",
+ "TARGET_2ND_ARCH_VARIANT",
+ "TARGET_2ND_CPU_VARIANT",
+ "HOST_ARCH",
+ "HOST_2ND_ARCH",
+ "HOST_OS",
+ "HOST_OS_EXTRA",
+ "HOST_CROSS_OS",
+ "HOST_CROSS_ARCH",
+ "HOST_CROSS_2ND_ARCH",
+ "HOST_BUILD_TYPE",
+ "BUILD_ID",
+ "OUT_DIR",
+ "AUX_OS_VARIANT_LIST",
+ "TARGET_BUILD_PDK",
+ "PDK_FUSION_PLATFORM_ZIP",
+ }
+
+ allVars := append(append([]string{
+ // Used to execute Kati and Ninja
+ "NINJA_GOALS",
+ "KATI_GOALS",
+ }, exportEnvVars...), bannerVars...)
+
+ make_vars, err := DumpMakeVars(ctx, config, config.Arguments(), []string{
+ filepath.Join(config.SoongOutDir(), "soong.variables"),
+ }, allVars)
+ if err != nil {
+ ctx.Fatalln("Error dumping make vars:", err)
+ }
+
+ // Print the banner like make does
+ fmt.Fprintln(ctx.Stdout(), "============================================")
+ for _, name := range bannerVars {
+ if make_vars[name] != "" {
+ fmt.Fprintf(ctx.Stdout(), "%s=%s\n", name, make_vars[name])
+ }
+ }
+ fmt.Fprintln(ctx.Stdout(), "============================================")
+
+ // Populate the environment
+ env := config.Environment()
+ for _, name := range exportEnvVars {
+ if make_vars[name] == "" {
+ env.Unset(name)
+ } else {
+ env.Set(name, make_vars[name])
+ }
+ }
+
+ config.SetKatiArgs(strings.Fields(make_vars["KATI_GOALS"]))
+ config.SetNinjaArgs(strings.Fields(make_vars["NINJA_GOALS"]))
+}
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
new file mode 100644
index 0000000..13e1834
--- /dev/null
+++ b/ui/build/ninja.go
@@ -0,0 +1,78 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os/exec"
+ "strconv"
+ "strings"
+)
+
+func runNinja(ctx Context, config Config) {
+ executable := "prebuilts/build-tools/" + config.HostPrebuiltTag() + "/bin/ninja"
+ args := []string{
+ "-d", "keepdepfile",
+ }
+
+ args = append(args, config.NinjaArgs()...)
+
+ var parallel int
+ if config.UseGoma() {
+ parallel = config.RemoteParallel()
+ } else {
+ parallel = config.Parallel()
+ }
+ args = append(args, "-j", strconv.Itoa(parallel))
+ if config.keepGoing != 1 {
+ args = append(args, "-k", strconv.Itoa(config.keepGoing))
+ }
+
+ args = append(args, "-f", config.CombinedNinjaFile())
+
+ if config.IsVerbose() {
+ args = append(args, "-v")
+ }
+ args = append(args, "-w", "dupbuild=err")
+
+ env := config.Environment().Copy()
+ env.AppendFromKati(config.KatiEnvFile())
+
+ // Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been
+ // used in the past to specify extra ninja arguments.
+ if extra, ok := env.Get("NINJA_ARGS"); ok {
+ args = append(args, strings.Fields(extra)...)
+ }
+ if extra, ok := env.Get("NINJA_EXTRA_ARGS"); ok {
+ args = append(args, strings.Fields(extra)...)
+ }
+
+ if _, ok := env.Get("NINJA_STATUS"); !ok {
+ env.Set("NINJA_STATUS", "[%p %f/%t] ")
+ }
+
+ cmd := exec.CommandContext(ctx.Context, executable, args...)
+ cmd.Env = env.Environ()
+ cmd.Stdin = ctx.Stdin()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ ctx.Verboseln(cmd.Path, cmd.Args)
+ if err := cmd.Run(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ ctx.Fatalln("ninja failed with:", e.ProcessState.String())
+ } else {
+ ctx.Fatalln("Failed to run ninja:", err)
+ }
+ }
+}
diff --git a/ui/build/signal.go b/ui/build/signal.go
new file mode 100644
index 0000000..3c8c8e1
--- /dev/null
+++ b/ui/build/signal.go
@@ -0,0 +1,60 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os"
+ "os/signal"
+ "runtime/debug"
+ "syscall"
+
+ "android/soong/ui/logger"
+)
+
+// SetupSignals sets up signal handling to kill our children and allow us to cleanly finish
+// writing our log/trace files.
+//
+// Currently, on the first SIGINT|SIGALARM we call the cancel() function, which is usually
+// the CancelFunc returned by context.WithCancel, which will kill all the commands running
+// within that Context. Usually that's enough, and you'll run through your normal error paths.
+//
+// If another signal comes in after the first one, we'll trigger a panic with full stacktraces
+// from every goroutine so that it's possible to debug what is stuck. Just before the process
+// exits, we'll call the cleanup() function so that you can flush your log files.
+func SetupSignals(log logger.Logger, cancel, cleanup func()) {
+ signals := make(chan os.Signal, 5)
+ // TODO: Handle other signals
+ signal.Notify(signals, os.Interrupt, syscall.SIGALRM)
+ go handleSignals(signals, log, cancel, cleanup)
+}
+
+func handleSignals(signals chan os.Signal, log logger.Logger, cancel, cleanup func()) {
+ defer cleanup()
+
+ var force bool
+
+ for {
+ s := <-signals
+ if force {
+ // So that we can better see what was stuck
+ debug.SetTraceback("all")
+ log.Panicln("Second signal received:", s)
+ } else {
+ log.Println("Got signal:", s)
+ cancel()
+ force = true
+ }
+ }
+}
diff --git a/ui/build/soong.go b/ui/build/soong.go
new file mode 100644
index 0000000..88e4161
--- /dev/null
+++ b/ui/build/soong.go
@@ -0,0 +1,57 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os/exec"
+ "path/filepath"
+)
+
+func runSoongBootstrap(ctx Context, config Config) {
+ cmd := exec.CommandContext(ctx.Context, "./bootstrap.bash")
+ env := config.Environment().Copy()
+ env.Set("BUILDDIR", config.SoongOutDir())
+ cmd.Env = env.Environ()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ ctx.Verboseln(cmd.Path, cmd.Args)
+ if err := cmd.Run(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
+ } else {
+ ctx.Fatalln("Failed to run soong bootstrap:", err)
+ }
+ }
+}
+
+func runSoong(ctx Context, config Config) {
+ cmd := exec.CommandContext(ctx.Context, filepath.Join(config.SoongOutDir(), "soong"), "-w", "dupbuild=err")
+ if config.IsVerbose() {
+ cmd.Args = append(cmd.Args, "-v")
+ }
+ env := config.Environment().Copy()
+ env.Set("SKIP_NINJA", "true")
+ cmd.Env = env.Environ()
+ cmd.Stdout = ctx.Stdout()
+ cmd.Stderr = ctx.Stderr()
+ ctx.Verboseln(cmd.Path, cmd.Args)
+ if err := cmd.Run(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
+ } else {
+ ctx.Fatalln("Failed to run soong bootstrap:", err)
+ }
+ }
+}
diff --git a/ui/build/util.go b/ui/build/util.go
new file mode 100644
index 0000000..ad084da
--- /dev/null
+++ b/ui/build/util.go
@@ -0,0 +1,79 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// indexList finds the index of a string in a []string
+func indexList(s string, list []string) int {
+ for i, l := range list {
+ if l == s {
+ return i
+ }
+ }
+
+ return -1
+}
+
+// inList determines whether a string is in a []string
+func inList(s string, list []string) bool {
+ return indexList(s, list) != -1
+}
+
+// ensureDirectoriesExist is a shortcut to os.MkdirAll, sending errors to the ctx logger.
+func ensureDirectoriesExist(ctx Context, dirs ...string) {
+ for _, dir := range dirs {
+ err := os.MkdirAll(dir, 0777)
+ if err != nil {
+ ctx.Fatalf("Error creating %s: %q\n", dir, err)
+ }
+ }
+}
+
+// ensureEmptyFileExists ensures that the containing directory exists, and the
+// specified file exists. If it doesn't exist, it will write an empty file.
+func ensureEmptyFileExists(ctx Context, file string) {
+ ensureDirectoriesExist(ctx, filepath.Dir(file))
+ if _, err := os.Stat(file); os.IsNotExist(err) {
+ f, err := os.Create(file)
+ if err != nil {
+ ctx.Fatalf("Error creating %s: %q\n", file, err)
+ }
+ f.Close()
+ } else if err != nil {
+ ctx.Fatalf("Error checking %s: %q\n", file, err)
+ }
+}
+
+// singleUnquote is similar to strconv.Unquote, but can handle multi-character strings inside single quotes.
+func singleUnquote(str string) (string, bool) {
+ if len(str) < 2 || str[0] != '\'' || str[len(str)-1] != '\'' {
+ return "", false
+ }
+ return str[1 : len(str)-1], true
+}
+
+// decodeKeyValue decodes a key=value string
+func decodeKeyValue(str string) (string, string, bool) {
+ idx := strings.IndexRune(str, '=')
+ if idx == -1 {
+ return "", "", false
+ }
+ return str[:idx], str[idx+1:], true
+}