Merge changes I9512642d,I6548889c,I8db5198f
* changes:
Add build_test.sh, split common parts of soong_ui.bash
Allow specifying a build variant
Improve multiproduct_kati output
diff --git a/build_test.bash b/build_test.bash
new file mode 100755
index 0000000..f833366
--- /dev/null
+++ b/build_test.bash
@@ -0,0 +1,34 @@
+#!/bin/bash -eu
+#
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file is used in our continous build infrastructure to run a variety of
+# tests related to the build system.
+#
+# Currently, it's used to build and run multiproduct_kati, so it'll attempt
+# to build ninja files for every product in the tree. I expect this to
+# evolve as we find interesting things to test or track performance for.
+#
+
+# To track how long we took to startup. %N isn't supported on Darwin, but
+# that's detected in the Go code, which skips calculating the startup time.
+export TRACE_BEGIN_SOONG=$(date +%s%N)
+
+export TOP=$(cd $(dirname ${BASH_SOURCE[0]})/../..; PWD= /bin/pwd)
+source "${TOP}/build/soong/cmd/microfactory/microfactory.bash"
+
+build_go multiproduct_kati android/soong/cmd/multiproduct_kati
+exec "$(getoutdir)/multiproduct_kati" "$@"
diff --git a/cmd/microfactory/microfactory.bash b/cmd/microfactory/microfactory.bash
new file mode 100644
index 0000000..7489fe3
--- /dev/null
+++ b/cmd/microfactory/microfactory.bash
@@ -0,0 +1,88 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set of utility functions to build and run go code with microfactory
+#
+# Inputs:
+# ${TOP}: The top of the android source tree
+# ${OUT_DIR}: The output directory location (defaults to ${TOP}/out)
+# ${OUT_DIR_COMMON_BASE}: Change the default out directory to
+# ${OUT_DIR_COMMON_BASE}/$(basename ${TOP})
+
+# Ensure GOROOT is set to the in-tree version.
+case $(uname) in
+ Linux)
+ export GOROOT="${TOP}/prebuilts/go/linux-x86/"
+ ;;
+ Darwin)
+ export GOROOT="${TOP}/prebuilts/go/darwin-x86/"
+ ;;
+ *) echo "unknown OS:" $(uname) >&2 && exit 1;;
+esac
+
+# Find the output directory
+function getoutdir
+{
+ local out_dir="${OUT_DIR-}"
+ if [ -z "${out_dir}" ]; then
+ if [ "${OUT_DIR_COMMON_BASE-}" ]; then
+ out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})"
+ else
+ out_dir="${TOP}/out"
+ fi
+ fi
+ echo "${out_dir}"
+}
+
+# Bootstrap microfactory from source if necessary and use it to build the
+# requested binary.
+#
+# Arguments:
+# $1: name of the requested binary
+# $2: package name
+function build_go
+{
+ # Increment when microfactory changes enough that it cannot rebuild itself.
+ # For example, if we use a new command line argument that doesn't work on older versions.
+ local mf_version=2
+
+ local mf_src="${TOP}/build/soong/cmd/microfactory"
+
+ local out_dir=$(getoutdir)
+ local mf_bin="${out_dir}/microfactory_$(uname)"
+ local mf_version_file="${out_dir}/.microfactory_$(uname)_version"
+ local built_bin="${out_dir}/$1"
+ local from_src=1
+
+ if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then
+ if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then
+ from_src=0
+ fi
+ fi
+
+ local mf_cmd
+ if [ $from_src -eq 1 ]; then
+ mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go"
+ else
+ mf_cmd="${mf_bin}"
+ fi
+
+ ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \
+ -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \
+ -o "${built_bin}" $2
+
+ if [ $from_src -eq 1 ]; then
+ echo "${mf_version}" >"${mf_version_file}"
+ fi
+}
diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go
index 3aa5a87..b12628e 100644
--- a/cmd/multiproduct_kati/main.go
+++ b/cmd/multiproduct_kati/main.go
@@ -51,11 +51,86 @@
var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
+var buildVariant = flag.String("variant", "eng", "build variant to use")
+
type Product struct {
ctx build.Context
config build.Config
}
+type Status struct {
+ cur int
+ total int
+ failed int
+
+ ctx build.Context
+ haveBlankLine bool
+ smartTerminal bool
+
+ lock sync.Mutex
+}
+
+func NewStatus(ctx build.Context) *Status {
+ return &Status{
+ ctx: ctx,
+ haveBlankLine: true,
+ smartTerminal: ctx.IsTerminal(),
+ }
+}
+
+func (s *Status) SetTotal(total int) {
+ s.total = total
+}
+
+func (s *Status) Fail(product string, err error) {
+ s.Finish(product)
+
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ if s.smartTerminal && !s.haveBlankLine {
+ fmt.Fprintln(s.ctx.Stdout())
+ s.haveBlankLine = true
+ }
+
+ s.failed++
+ fmt.Fprintln(s.ctx.Stderr(), "FAILED:", product)
+ s.ctx.Verboseln("FAILED:", product)
+ s.ctx.Println(err)
+}
+
+func (s *Status) Finish(product string) {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ s.cur++
+ line := fmt.Sprintf("[%d/%d] %s", s.cur, s.total, product)
+
+ if s.smartTerminal {
+ if max, ok := s.ctx.TermWidth(); ok {
+ if len(line) > max {
+ line = line[:max]
+ }
+ }
+
+ fmt.Fprint(s.ctx.Stdout(), "\r", line, "\x1b[K")
+ s.haveBlankLine = false
+ } else {
+ s.ctx.Println(line)
+ }
+}
+
+func (s *Status) Finished() int {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ if !s.haveBlankLine {
+ fmt.Fprintln(s.ctx.Stdout())
+ s.haveBlankLine = true
+ }
+ return s.failed
+}
+
func main() {
log := logger.New(os.Stderr)
defer log.Cleanup()
@@ -80,7 +155,7 @@
StdioInterface: build.StdioImpl{},
}}
- failed := false
+ status := NewStatus(buildCtx)
config := build.NewConfig(buildCtx)
if *outDir == "" {
@@ -94,7 +169,7 @@
if !*keep {
defer func() {
- if !failed {
+ if status.Finished() == 0 {
os.RemoveAll(*outDir)
}
}()
@@ -114,8 +189,9 @@
products := strings.Fields(vars["all_named_products"])
log.Verbose("Got product list:", products)
+ status.SetTotal(len(products))
+
var wg sync.WaitGroup
- errs := make(chan error, len(products))
productConfigs := make(chan Product, len(products))
// Run the product config for every product in parallel
@@ -124,7 +200,7 @@
go func(product string) {
defer wg.Done()
defer logger.Recover(func(err error) {
- errs <- fmt.Errorf("Error building %s: %v", product, err)
+ status.Fail(product, err)
})
productOutDir := filepath.Join(config.OutDir(), product)
@@ -151,7 +227,7 @@
productConfig := build.NewConfig(productCtx)
productConfig.Environment().Set("OUT_DIR", productOutDir)
- productConfig.Lunch(productCtx, product, "eng")
+ productConfig.Lunch(productCtx, product, *buildVariant)
build.Build(productCtx, productConfig, build.BuildProductConfig)
productConfigs <- Product{productCtx, productConfig}
@@ -171,7 +247,7 @@
for product := range productConfigs {
func() {
defer logger.Recover(func(err error) {
- errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err)
+ status.Fail(product.config.TargetProduct(), err)
})
buildWhat := 0
@@ -185,22 +261,14 @@
if !*keep {
os.RemoveAll(product.config.OutDir())
}
- log.Println("Finished running for", product.config.TargetProduct())
+ status.Finish(product.config.TargetProduct())
}()
}
}()
}
- go func() {
- wg2.Wait()
- close(errs)
- }()
+ wg2.Wait()
- for err := range errs {
- failed = true
- log.Print(err)
- }
-
- if failed {
- log.Fatalln("Failed")
+ if count := status.Finished(); count > 0 {
+ log.Fatalln(count, "products failed")
}
}
diff --git a/soong_ui.bash b/soong_ui.bash
index e3997cf..105af9f 100755
--- a/soong_ui.bash
+++ b/soong_ui.bash
@@ -15,7 +15,7 @@
# limitations under the License.
# To track how long we took to startup. %N isn't supported on Darwin, but
-# that's detected in the Go code, and skip calculating the startup time.
+# that's detected in the Go code, which skips calculating the startup time.
export TRACE_BEGIN_SOONG=$(date +%s%N)
# Function to find top of the source tree (if $TOP isn't set) by walking up the
@@ -47,63 +47,8 @@
fi
}
-# Bootstrap microfactory from source if necessary and use it to build the
-# soong_ui binary, then run soong_ui.
-function run_go
-{
- # Increment when microfactory changes enough that it cannot rebuild itself.
- # For example, if we use a new command line argument that doesn't work on older versions.
- local mf_version=2
-
- local mf_src="${TOP}/build/soong/cmd/microfactory"
-
- local out_dir="${OUT_DIR-}"
- if [ -z "${out_dir}" ]; then
- if [ "${OUT_DIR_COMMON_BASE-}" ]; then
- out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${TOP})"
- else
- out_dir="${TOP}/out"
- fi
- fi
-
- local mf_bin="${out_dir}/microfactory_$(uname)"
- local mf_version_file="${out_dir}/.microfactory_$(uname)_version"
- local soong_ui_bin="${out_dir}/soong_ui"
- local from_src=1
-
- if [ -f "${mf_bin}" ] && [ -f "${mf_version_file}" ]; then
- if [ "${mf_version}" -eq "$(cat "${mf_version_file}")" ]; then
- from_src=0
- fi
- fi
-
- local mf_cmd
- if [ $from_src -eq 1 ]; then
- mf_cmd="${GOROOT}/bin/go run ${mf_src}/microfactory.go"
- else
- mf_cmd="${mf_bin}"
- fi
-
- ${mf_cmd} -s "${mf_src}" -b "${mf_bin}" \
- -pkg-path "android/soong=${TOP}/build/soong" -trimpath "${TOP}/build/soong" \
- -o "${soong_ui_bin}" android/soong/cmd/soong_ui
-
- if [ $from_src -eq 1 ]; then
- echo "${mf_version}" >"${mf_version_file}"
- fi
-
- exec "${out_dir}/soong_ui" "$@"
-}
-
export TOP=$(gettop)
-case $(uname) in
- Linux)
- export GOROOT="${TOP}/prebuilts/go/linux-x86/"
- ;;
- Darwin)
- export GOROOT="${TOP}/prebuilts/go/darwin-x86/"
- ;;
- *) echo "unknown OS:" $(uname) >&2 && exit 1;;
-esac
+source build/soong/cmd/microfactory/microfactory.bash
-run_go "$@"
+build_go soong_ui android/soong/cmd/soong_ui
+exec "$(getoutdir)/soong_ui" "$@"
diff --git a/ui/build/context.go b/ui/build/context.go
index f85bb6c..52a337d 100644
--- a/ui/build/context.go
+++ b/ui/build/context.go
@@ -102,3 +102,7 @@
}
return false
}
+
+func (c ContextImpl) TermWidth() (int, bool) {
+ return termWidth(c.Stdout())
+}