Add path interposer
This will allow us to track (and eventually limit) the commands that the
build references via $PATH. These are mostly implicit dependencies on
the host system -- for Linux, we assume something similar to Ubuntu
14.04 with a few extra packages, but this will let us better define
that.
This will not catch uses of tools with absolute paths (/bin/bash, etc),
but most uses shouldn't be relying on absolute path names anyways.
Adds ~400ms on the first startup, ~140ms on subsequent runs, and
overhead of a few ms for every forwarded execution.
Test: m
Test: build/soong/build_test.bash
Test: Add `gcc --version`, TEMPORARY_DISABLE_PATH_RESTRICTIONS=true m
Change-Id: Id68cbb1c8ceef65bbbb10751e83722c7662d2351
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 5809894..1fe5b6f 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -13,9 +13,22 @@
// limitations under the License.
bootstrap_go_package {
+ name: "soong-ui-build-paths",
+ pkgPath: "android/soong/ui/build/paths",
+ srcs: [
+ "paths/config.go",
+ "paths/logs.go",
+ ],
+ testSrcs: [
+ "paths/logs_test.go",
+ ],
+}
+
+bootstrap_go_package {
name: "soong-ui-build",
pkgPath: "android/soong/ui/build",
deps: [
+ "soong-ui-build-paths",
"soong-ui-logger",
"soong-ui-tracer",
"soong-shared",
@@ -33,6 +46,7 @@
"finder.go",
"kati.go",
"ninja.go",
+ "path.go",
"proc_sync.go",
"signal.go",
"soong.go",
diff --git a/ui/build/build.go b/ui/build/build.go
index 78eb6a3..66dbf03 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -140,6 +140,8 @@
ensureEmptyDirectoriesExist(ctx, config.TempDir())
+ SetupPath(ctx, config)
+
if what&BuildProductConfig != 0 {
// Run make for product config
runMakeProductConfig(ctx, config)
diff --git a/ui/build/config.go b/ui/build/config.go
index 5622dff..6f2d24a 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -51,6 +51,8 @@
targetDeviceDir string
brokenDupRules bool
+
+ pathReplaced bool
}
const srcDirFileCheck = "build/soong/root.bp"
diff --git a/ui/build/path.go b/ui/build/path.go
new file mode 100644
index 0000000..52658ef
--- /dev/null
+++ b/ui/build/path.go
@@ -0,0 +1,149 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/google/blueprint/microfactory"
+
+ "android/soong/ui/build/paths"
+)
+
+func parsePathDir(dir string) []string {
+ f, err := os.Open(dir)
+ if err != nil {
+ return nil
+ }
+ defer f.Close()
+
+ if s, err := f.Stat(); err != nil || !s.IsDir() {
+ return nil
+ }
+
+ infos, err := f.Readdir(-1)
+ if err != nil {
+ return nil
+ }
+
+ ret := make([]string, 0, len(infos))
+ for _, info := range infos {
+ if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
+ ret = append(ret, info.Name())
+ }
+ }
+ return ret
+}
+
+func SetupPath(ctx Context, config Config) {
+ if config.pathReplaced {
+ return
+ }
+
+ ctx.BeginTrace("path")
+ defer ctx.EndTrace()
+
+ origPath, _ := config.Environment().Get("PATH")
+ myPath := filepath.Join(config.OutDir(), ".path")
+ interposer := myPath + "_interposer"
+
+ var cfg microfactory.Config
+ cfg.Map("android/soong", "build/soong")
+ cfg.TrimPath, _ = filepath.Abs(".")
+ if _, err := microfactory.Build(&cfg, interposer, "android/soong/cmd/path_interposer"); err != nil {
+ ctx.Fatalln("Failed to build path interposer:", err)
+ }
+
+ if err := ioutil.WriteFile(interposer+"_origpath", []byte(origPath), 0777); err != nil {
+ ctx.Fatalln("Failed to write original path:", err)
+ }
+
+ entries, err := paths.LogListener(ctx.Context, interposer+"_log")
+ if err != nil {
+ ctx.Fatalln("Failed to listen for path logs:", err)
+ }
+
+ go func() {
+ for log := range entries {
+ curPid := os.Getpid()
+ for i, proc := range log.Parents {
+ if proc.Pid == curPid {
+ log.Parents = log.Parents[i:]
+ break
+ }
+ }
+ procPrints := []string{
+ "See https://android.googlesource.com/platform/build/+/master/Changes.md#PATH_Tools for more information.",
+ }
+ if len(log.Parents) > 0 {
+ procPrints = append(procPrints, "Process tree:")
+ for i, proc := range log.Parents {
+ procPrints = append(procPrints, fmt.Sprintf("%s→ %s", strings.Repeat(" ", i), proc.Command))
+ }
+ }
+
+ config := paths.GetConfig(log.Basename)
+ if config.Error {
+ ctx.Printf("Disallowed PATH tool %q used: %#v", log.Basename, log.Args)
+ for _, line := range procPrints {
+ ctx.Println(line)
+ }
+ } else {
+ ctx.Verbosef("Unknown PATH tool %q used: %#v", log.Basename, log.Args)
+ for _, line := range procPrints {
+ ctx.Verboseln(line)
+ }
+ }
+ }
+ }()
+
+ ensureEmptyDirectoriesExist(ctx, myPath)
+
+ var execs []string
+ for _, pathEntry := range filepath.SplitList(origPath) {
+ if pathEntry == "" {
+ // Ignore the current directory
+ continue
+ }
+ // TODO(dwillemsen): remove path entries under TOP? or anything
+ // that looks like an android source dir? They won't exist on
+ // the build servers, since they're added by envsetup.sh.
+ // (Except for the JDK, which is configured in ui/build/config.go)
+
+ execs = append(execs, parsePathDir(pathEntry)...)
+ }
+
+ allowAllSymlinks := config.Environment().IsEnvTrue("TEMPORARY_DISABLE_PATH_RESTRICTIONS")
+ for _, name := range execs {
+ if !paths.GetConfig(name).Symlink && !allowAllSymlinks {
+ continue
+ }
+
+ err := os.Symlink("../.path_interposer", filepath.Join(myPath, name))
+ // Intentionally ignore existing files -- that means that we
+ // just created it, and the first one should win.
+ if err != nil && !os.IsExist(err) {
+ ctx.Fatalln("Failed to create symlink:", err)
+ }
+ }
+
+ myPath, _ = filepath.Abs(myPath)
+ config.Environment().Set("PATH", myPath)
+ config.pathReplaced = true
+}
diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go
new file mode 100644
index 0000000..ed44ced
--- /dev/null
+++ b/ui/build/paths/config.go
@@ -0,0 +1,150 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+type PathConfig struct {
+ // Whether to create the symlink in the new PATH for this tool.
+ Symlink bool
+
+ // Whether to log about usages of this tool to the soong.log
+ Log bool
+
+ // Whether to exit with an error instead of invoking the underlying tool.
+ Error bool
+}
+
+var Allowed = PathConfig{
+ Symlink: true,
+ Log: false,
+ Error: false,
+}
+
+var Forbidden = PathConfig{
+ Symlink: false,
+ Log: true,
+ Error: true,
+}
+
+// The configuration used if the tool is not listed in the config below.
+// Currently this will create the symlink, but log a warning. In the future,
+// I expect this to move closer to Forbidden.
+var Missing = PathConfig{
+ Symlink: true,
+ Log: true,
+ Error: false,
+}
+
+func GetConfig(name string) PathConfig {
+ if config, ok := Configuration[name]; ok {
+ return config
+ }
+ return Missing
+}
+
+var Configuration = map[string]PathConfig{
+ "awk": Allowed,
+ "basename": Allowed,
+ "bash": Allowed,
+ "bzip2": Allowed,
+ "cat": Allowed,
+ "chmod": Allowed,
+ "cmp": Allowed,
+ "comm": Allowed,
+ "cp": Allowed,
+ "cut": Allowed,
+ "date": Allowed,
+ "dd": Allowed,
+ "diff": Allowed,
+ "dirname": Allowed,
+ "echo": Allowed,
+ "egrep": Allowed,
+ "env": Allowed,
+ "expr": Allowed,
+ "find": Allowed,
+ "getconf": Allowed,
+ "getopt": Allowed,
+ "git": Allowed,
+ "grep": Allowed,
+ "gzip": Allowed,
+ "head": Allowed,
+ "hexdump": Allowed,
+ "hostname": Allowed,
+ "jar": Allowed,
+ "java": Allowed,
+ "javap": Allowed,
+ "ln": Allowed,
+ "ls": Allowed,
+ "m4": Allowed,
+ "make": Allowed,
+ "md5sum": Allowed,
+ "mkdir": Allowed,
+ "mktemp": Allowed,
+ "mv": Allowed,
+ "openssl": Allowed,
+ "patch": Allowed,
+ "perl": Allowed,
+ "pstree": Allowed,
+ "python": Allowed,
+ "python2.7": Allowed,
+ "python3": Allowed,
+ "readlink": Allowed,
+ "realpath": Allowed,
+ "rm": Allowed,
+ "rsync": Allowed,
+ "runalarm": Allowed,
+ "sed": Allowed,
+ "setsid": Allowed,
+ "sh": Allowed,
+ "sha256sum": Allowed,
+ "sha512sum": Allowed,
+ "sort": Allowed,
+ "stat": Allowed,
+ "sum": Allowed,
+ "tar": Allowed,
+ "tail": Allowed,
+ "touch": Allowed,
+ "tr": Allowed,
+ "true": Allowed,
+ "uname": Allowed,
+ "uniq": Allowed,
+ "unzip": Allowed,
+ "wc": Allowed,
+ "which": Allowed,
+ "whoami": Allowed,
+ "xargs": Allowed,
+ "xmllint": Allowed,
+ "xz": Allowed,
+ "zip": Allowed,
+ "zipinfo": Allowed,
+
+ // Host toolchain is removed. In-tree toolchain should be used instead.
+ // GCC also can't find cc1 with this implementation.
+ "ar": Forbidden,
+ "as": Forbidden,
+ "cc": Forbidden,
+ "clang": Forbidden,
+ "clang++": Forbidden,
+ "gcc": Forbidden,
+ "g++": Forbidden,
+ "ld": Forbidden,
+ "ld.bfd": Forbidden,
+ "ld.gold": Forbidden,
+ "pkg-config": Forbidden,
+
+ // We've got prebuilts of these
+ //"dtc": Forbidden,
+ //"lz4": Forbidden,
+ //"lz4c": Forbidden,
+}
diff --git a/ui/build/paths/logs.go b/ui/build/paths/logs.go
new file mode 100644
index 0000000..a44994c
--- /dev/null
+++ b/ui/build/paths/logs.go
@@ -0,0 +1,101 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+ "context"
+ "encoding/gob"
+ "net"
+ "os"
+ "time"
+)
+
+type LogProcess struct {
+ Pid int
+ Command string
+}
+
+type LogEntry struct {
+ Basename string
+ Args []string
+ Parents []LogProcess
+}
+
+const timeoutDuration = time.Duration(250) * time.Millisecond
+
+func SendLog(logSocket string, entry *LogEntry, done chan interface{}) {
+ defer close(done)
+
+ dialer := &net.Dialer{}
+ conn, err := dialer.Dial("unix", logSocket)
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+
+ if err := conn.SetDeadline(dialer.Deadline); err != nil {
+ return
+ }
+
+ enc := gob.NewEncoder(conn)
+ enc.Encode(entry)
+}
+
+func LogListener(ctx context.Context, logSocket string) (chan *LogEntry, error) {
+ ret := make(chan *LogEntry, 5)
+
+ if err := os.Remove(logSocket); err != nil && !os.IsNotExist(err) {
+ return nil, err
+ }
+
+ ln, err := net.Listen("unix", logSocket)
+ if err != nil {
+ return nil, err
+ }
+
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ ln.Close()
+ }
+ }
+ }()
+
+ go func() {
+ defer close(ret)
+
+ for {
+ conn, err := ln.Accept()
+ if err != nil {
+ ln.Close()
+ break
+ }
+ conn.SetDeadline(time.Now().Add(timeoutDuration))
+
+ go func() {
+ defer conn.Close()
+
+ dec := gob.NewDecoder(conn)
+ entry := &LogEntry{}
+ if err := dec.Decode(entry); err != nil {
+ return
+ }
+ ret <- entry
+ }()
+ }
+ }()
+ return ret, nil
+}
diff --git a/ui/build/paths/logs_test.go b/ui/build/paths/logs_test.go
new file mode 100644
index 0000000..1ec040d
--- /dev/null
+++ b/ui/build/paths/logs_test.go
@@ -0,0 +1,134 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+ "context"
+ "io/ioutil"
+ "net"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestSendLog(t *testing.T) {
+ d, err := ioutil.TempDir("", "log_socket")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(d)
+ f := filepath.Join(d, "sock")
+
+ ctx, _ := context.WithTimeout(context.Background(), 2*timeoutDuration)
+
+ recv, err := LogListener(ctx, f)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ go func() {
+ for i := 0; i < 10; i++ {
+ SendLog(f, &LogEntry{
+ Basename: "test",
+ Args: []string{"foo", "bar"},
+ }, make(chan interface{}))
+ }
+ }()
+
+ count := 0
+ for {
+ select {
+ case entry := <-recv:
+ ref := LogEntry{
+ Basename: "test",
+ Args: []string{"foo", "bar"},
+ }
+ if !reflect.DeepEqual(ref, *entry) {
+ t.Fatalf("Bad log entry: %v", entry)
+ }
+ count++
+
+ if count == 10 {
+ return
+ }
+ case <-ctx.Done():
+ t.Error("Hit timeout before receiving all logs")
+ }
+ }
+}
+
+func TestSendLogError(t *testing.T) {
+ d, err := ioutil.TempDir("", "log_socket")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(d)
+
+ t.Run("Missing file", func(t *testing.T) {
+ start := time.Now()
+ SendLog(filepath.Join(d, "missing"), &LogEntry{}, make(chan interface{}))
+ elapsed := time.Since(start)
+ if elapsed > timeoutDuration {
+ t.Errorf("Should have been << timeout (%s), but was %s", timeoutDuration, elapsed)
+ }
+ })
+
+ t.Run("Regular file", func(t *testing.T) {
+ f := filepath.Join(d, "file")
+ if fp, err := os.Create(f); err == nil {
+ fp.Close()
+ } else {
+ t.Fatal(err)
+ }
+
+ start := time.Now()
+ SendLog(f, &LogEntry{}, make(chan interface{}))
+ elapsed := time.Since(start)
+ if elapsed > timeoutDuration {
+ t.Errorf("Should have been << timeout (%s), but was %s", timeoutDuration, elapsed)
+ }
+ })
+
+ t.Run("Reader not reading", func(t *testing.T) {
+ f := filepath.Join(d, "sock1")
+
+ ln, err := net.Listen("unix", f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer ln.Close()
+
+ done := make(chan bool, 1)
+ go func() {
+ for i := 0; i < 1000; i++ {
+ SendLog(f, &LogEntry{
+ // Ensure a relatively large payload
+ Basename: strings.Repeat(" ", 100000),
+ }, make(chan interface{}))
+ }
+ done <- true
+ }()
+
+ select {
+ case <-done:
+ break
+ case <-time.After(10 * timeoutDuration):
+ t.Error("Should have finished")
+ }
+ })
+}