Merge "Add multitree apex metadata"
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index e7ff08f..e7b84e3 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -607,7 +607,7 @@
dclaEnabledModules := map[string]bool{}
addToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
return &mixedBuildBazelContext{
- bazelRunner: &builtinBazelRunner{},
+ bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
paths: &paths,
modulesDefaultToBazel: c.BuildMode == BazelDevMode,
bazelEnabledModules: enabledModules,
@@ -684,23 +684,46 @@
return "", "", nil
}
-type builtinBazelRunner struct{}
+type builtinBazelRunner struct {
+ useBazelProxy bool
+ outDir string
+}
// Issues the given bazel command with given build label and additional flags.
// Returns (stdout, stderr, error). The first and second return values are strings
// containing the stdout and stderr of the run command, and an error is returned if
// the invocation returned an error code.
func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (string, string, error) {
- eventHandler.Begin("bazel command")
- defer eventHandler.End("bazel command")
- stderr := &bytes.Buffer{}
- bazelCmd.Stderr = stderr
- if output, err := bazelCmd.Output(); err != nil {
- return "", string(stderr.Bytes()),
- fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
- err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ if r.useBazelProxy {
+ eventHandler.Begin("client_proxy")
+ defer eventHandler.End("client_proxy")
+ proxyClient := bazel.NewProxyClient(r.outDir)
+ // Omit the arg containing the Bazel binary, as that is handled by the proxy
+ // server.
+ bazelFlags := bazelCmd.Args[1:]
+ // TODO(b/270989498): Refactor these functions to not take exec.Cmd, as its
+ // not actually executed for client proxying.
+ resp, err := proxyClient.IssueCommand(bazel.CmdRequest{bazelFlags, bazelCmd.Env})
+
+ if err != nil {
+ return "", "", err
+ }
+ if len(resp.ErrorString) > 0 {
+ return "", "", fmt.Errorf(resp.ErrorString)
+ }
+ return resp.Stdout, resp.Stderr, nil
} else {
- return string(output), string(stderr.Bytes()), nil
+ eventHandler.Begin("bazel command")
+ defer eventHandler.End("bazel command")
+ stderr := &bytes.Buffer{}
+ bazelCmd.Stderr = stderr
+ if output, err := bazelCmd.Output(); err != nil {
+ return "", string(stderr.Bytes()),
+ fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
+ err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ } else {
+ return string(output), string(stderr.Bytes()), nil
+ }
}
}
diff --git a/android/config.go b/android/config.go
index 78da320..6412cb7 100644
--- a/android/config.go
+++ b/android/config.go
@@ -87,6 +87,8 @@
BazelModeDev bool
BazelModeStaging bool
BazelForceEnabledModules string
+
+ UseBazelProxy bool
}
// Build modes that soong_build can run as.
@@ -251,6 +253,10 @@
// specified modules. They are passed via the command-line flag
// "--bazel-force-enabled-modules"
bazelForceEnabledModules map[string]struct{}
+
+ // If true, for any requests to Bazel, communicate with a Bazel proxy using
+ // unix sockets, instead of spawning Bazel as a subprocess.
+ UseBazelProxy bool
}
type deviceConfig struct {
@@ -442,6 +448,8 @@
mixedBuildDisabledModules: make(map[string]struct{}),
mixedBuildEnabledModules: make(map[string]struct{}),
bazelForceEnabledModules: make(map[string]struct{}),
+
+ UseBazelProxy: cmdArgs.UseBazelProxy,
}
config.deviceConfig = &deviceConfig{
@@ -1359,11 +1367,6 @@
}
}
if coverage && len(c.config.productVariables.NativeCoverageExcludePaths) > 0 {
- // Workaround coverage boot failure.
- // http://b/269981180
- if strings.HasPrefix(path, "external/protobuf") {
- coverage = false
- }
if HasAnyPrefix(path, c.config.productVariables.NativeCoverageExcludePaths) {
coverage = false
}
diff --git a/android/sdk_version.go b/android/sdk_version.go
index 8953eae..a7e03dc 100644
--- a/android/sdk_version.go
+++ b/android/sdk_version.go
@@ -211,7 +211,29 @@
if !s.ApiLevel.IsPreview() {
return s.ApiLevel.String(), nil
}
- return ctx.Config().DefaultAppTargetSdk(ctx).String(), nil
+ // Determine the default sdk
+ ret := ctx.Config().DefaultAppTargetSdk(ctx)
+ if !ret.IsPreview() {
+ // If the default sdk has been finalized, return that
+ return ret.String(), nil
+ }
+ // There can be more than one active in-development sdks
+ // If an app is targeting an active sdk, but not the default one, return the requested active sdk.
+ // e.g.
+ // SETUP
+ // In-development: UpsideDownCake, VanillaIceCream
+ // Default: VanillaIceCream
+ // Android.bp
+ // min_sdk_version: `UpsideDownCake`
+ // RETURN
+ // UpsideDownCake and not VanillaIceCream
+ for _, preview := range ctx.Config().PreviewApiLevels() {
+ if s.ApiLevel.String() == preview.String() {
+ return preview.String(), nil
+ }
+ }
+ // Otherwise return the default one
+ return ret.String(), nil
}
var (
diff --git a/android/util.go b/android/util.go
index 6c0ddf4..38e0a4d 100644
--- a/android/util.go
+++ b/android/util.go
@@ -65,21 +65,8 @@
// SortedStringKeys returns the keys of the given map in the ascending order.
//
// Deprecated: Use SortedKeys instead.
-func SortedStringKeys(m interface{}) []string {
- v := reflect.ValueOf(m)
- if v.Kind() != reflect.Map {
- panic(fmt.Sprintf("%#v is not a map", m))
- }
- if v.Len() == 0 {
- return nil
- }
- iter := v.MapRange()
- s := make([]string, 0, v.Len())
- for iter.Next() {
- s = append(s, iter.Key().String())
- }
- sort.Strings(s)
- return s
+func SortedStringKeys[V any](m map[string]V) []string {
+ return SortedKeys(m)
}
type Ordered interface {
diff --git a/android/util_test.go b/android/util_test.go
index 51d8e32..1034d9e 100644
--- a/android/util_test.go
+++ b/android/util_test.go
@@ -671,44 +671,6 @@
testSortedKeysHelper(t, "empty", map[string]string{}, nil)
}
-func TestSortedStringKeys(t *testing.T) {
- testCases := []struct {
- name string
- in interface{}
- expected []string
- }{
- {
- name: "nil",
- in: map[string]string(nil),
- expected: nil,
- },
- {
- name: "empty",
- in: map[string]string{},
- expected: nil,
- },
- {
- name: "simple",
- in: map[string]string{"a": "foo", "b": "bar"},
- expected: []string{"a", "b"},
- },
- {
- name: "interface values",
- in: map[string]interface{}{"a": nil, "b": nil},
- expected: []string{"a", "b"},
- },
- }
-
- for _, tt := range testCases {
- t.Run(tt.name, func(t *testing.T) {
- got := SortedStringKeys(tt.in)
- if g, w := got, tt.expected; !reflect.DeepEqual(g, w) {
- t.Errorf("wanted %q, got %q", w, g)
- }
- })
- }
-}
-
func TestSortedStringValues(t *testing.T) {
testCases := []struct {
name string
diff --git a/apex/apex.go b/apex/apex.go
index d7d76d1..88eb72f 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -99,6 +99,10 @@
// /system/sepolicy/apex/<module_name>_file_contexts.
File_contexts *string `android:"path"`
+ // By default, file_contexts is amended by force-labelling / and /apex_manifest.pb as system_file
+ // to avoid mistakes. When set as true, no force-labelling.
+ Use_file_contexts_as_is *bool
+
// Path to the canned fs config file for customizing file's uid/gid/mod/capabilities. The
// format is /<path_or_glob> <uid> <gid> <mode> [capabilities=0x<cap>], where path_or_glob is a
// path or glob pattern for a file or set of files, uid/gid are numerial values of user ID
diff --git a/apex/apex_test.go b/apex/apex_test.go
index 53e922c..c94bbbb 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -784,6 +784,43 @@
}
}
+func TestFileContexts(t *testing.T) {
+ for _, useFileContextsAsIs := range []bool{true, false} {
+ prop := ""
+ if useFileContextsAsIs {
+ prop = "use_file_contexts_as_is: true,\n"
+ }
+ ctx := testApex(t, `
+ apex {
+ name: "myapex",
+ key: "myapex.key",
+ file_contexts: "file_contexts",
+ updatable: false,
+ vendor: true,
+ `+prop+`
+ }
+
+ apex_key {
+ name: "myapex.key",
+ public_key: "testkey.avbpubkey",
+ private_key: "testkey.pem",
+ }
+ `, withFiles(map[string][]byte{
+ "file_contexts": nil,
+ }))
+
+ rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("file_contexts")
+ forceLabellingCommand := "apex_manifest\\\\.pb u:object_r:system_file:s0"
+ if useFileContextsAsIs {
+ android.AssertStringDoesNotContain(t, "should force-label",
+ rule.RuleParams.Command, forceLabellingCommand)
+ } else {
+ android.AssertStringDoesContain(t, "shouldn't force-label",
+ rule.RuleParams.Command, forceLabellingCommand)
+ }
+ }
+}
+
func TestBasicZipApex(t *testing.T) {
ctx := testApex(t, `
apex {
diff --git a/apex/builder.go b/apex/builder.go
index 7248d97..ee6c473 100644
--- a/apex/builder.go
+++ b/apex/builder.go
@@ -333,6 +333,8 @@
ctx.PropertyErrorf("file_contexts", "cannot find file_contexts file: %q", fileContexts.String())
}
+ useFileContextsAsIs := proptools.Bool(a.properties.Use_file_contexts_as_is)
+
output := android.PathForModuleOut(ctx, "file_contexts")
rule := android.NewRuleBuilder(pctx, ctx)
@@ -344,9 +346,11 @@
rule.Command().Text("cat").Input(fileContexts).Text(">>").Output(output)
// new line
rule.Command().Text("echo").Text(">>").Output(output)
- // force-label /apex_manifest.pb and / as system_file so that apexd can read them
- rule.Command().Text("echo").Flag("/apex_manifest\\\\.pb u:object_r:system_file:s0").Text(">>").Output(output)
- rule.Command().Text("echo").Flag("/ u:object_r:system_file:s0").Text(">>").Output(output)
+ if !useFileContextsAsIs {
+ // force-label /apex_manifest.pb and / as system_file so that apexd can read them
+ rule.Command().Text("echo").Flag("/apex_manifest\\\\.pb u:object_r:system_file:s0").Text(">>").Output(output)
+ rule.Command().Text("echo").Flag("/ u:object_r:system_file:s0").Text(">>").Output(output)
+ }
case flattenedApex:
// For flattened apexes, install path should be prepended.
// File_contexts file should be emiited to make via LOCAL_FILE_CONTEXTS
@@ -359,9 +363,11 @@
rule.Command().Text("awk").Text(`'/object_r/{printf("` + apexPath + `%s\n", $0)}'`).Input(fileContexts).Text(">").Output(output)
// new line
rule.Command().Text("echo").Text(">>").Output(output)
- // force-label /apex_manifest.pb and / as system_file so that apexd can read them
- rule.Command().Text("echo").Flag(apexPath + `/apex_manifest\\.pb u:object_r:system_file:s0`).Text(">>").Output(output)
- rule.Command().Text("echo").Flag(apexPath + "/ u:object_r:system_file:s0").Text(">>").Output(output)
+ if !useFileContextsAsIs {
+ // force-label /apex_manifest.pb and / as system_file so that apexd can read them
+ rule.Command().Text("echo").Flag(apexPath + `/apex_manifest\\.pb u:object_r:system_file:s0`).Text(">>").Output(output)
+ rule.Command().Text("echo").Flag(apexPath + "/ u:object_r:system_file:s0").Text(">>").Output(output)
+ }
default:
panic(fmt.Errorf("unsupported type %v", a.properties.ApexType))
}
diff --git a/bazel/Android.bp b/bazel/Android.bp
index d11c78b..4709f5c 100644
--- a/bazel/Android.bp
+++ b/bazel/Android.bp
@@ -7,6 +7,7 @@
pkgPath: "android/soong/bazel",
srcs: [
"aquery.go",
+ "bazel_proxy.go",
"configurability.go",
"constants.go",
"properties.go",
diff --git a/bazel/bazel_proxy.go b/bazel/bazel_proxy.go
new file mode 100644
index 0000000..d7f5e64
--- /dev/null
+++ b/bazel/bazel_proxy.go
@@ -0,0 +1,219 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bazel
+
+import (
+ "bytes"
+ "encoding/gob"
+ "fmt"
+ "net"
+ os_lib "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// Logs fatal events of ProxyServer.
+type ServerLogger interface {
+ Fatal(v ...interface{})
+ Fatalf(format string, v ...interface{})
+}
+
+// CmdRequest is a request to the Bazel Proxy server.
+type CmdRequest struct {
+ // Args to the Bazel command.
+ Argv []string
+ // Environment variables to pass to the Bazel invocation. Strings should be of
+ // the form "KEY=VALUE".
+ Env []string
+}
+
+// CmdResponse is a response from the Bazel Proxy server.
+type CmdResponse struct {
+ Stdout string
+ Stderr string
+ ErrorString string
+}
+
+// ProxyClient is a client which can issue Bazel commands to the Bazel
+// proxy server. Requests are issued (and responses received) via a unix socket.
+// See ProxyServer for more details.
+type ProxyClient struct {
+ outDir string
+}
+
+// ProxyServer is a server which runs as a background goroutine. Each
+// request to the server describes a Bazel command which the server should run.
+// The server then issues the Bazel command, and returns a response describing
+// the stdout/stderr of the command.
+// Client-server communication is done via a unix socket under the output
+// directory.
+// The server is intended to circumvent sandboxing for subprocesses of the
+// build. The build orchestrator (soong_ui) can launch a server to exist outside
+// of sandboxing, and sandboxed processes (such as soong_build) can issue
+// bazel commands through this socket tunnel. This allows a sandboxed process
+// to issue bazel requests to a bazel that resides outside of sandbox. This
+// is particularly useful to maintain a persistent Bazel server which lives
+// past the duration of a single build.
+// The ProxyServer will only live as long as soong_ui does; the
+// underlying Bazel server will live past the duration of the build.
+type ProxyServer struct {
+ logger ServerLogger
+ outDir string
+ workspaceDir string
+ // The server goroutine will listen on this channel and stop handling requests
+ // once it is written to.
+ done chan struct{}
+}
+
+// NewProxyClient is a constructor for a ProxyClient.
+func NewProxyClient(outDir string) *ProxyClient {
+ return &ProxyClient{
+ outDir: outDir,
+ }
+}
+
+func unixSocketPath(outDir string) string {
+ return filepath.Join(outDir, "bazelsocket.sock")
+}
+
+// IssueCommand issues a request to the Bazel Proxy Server to issue a Bazel
+// request. Returns a response describing the output from the Bazel process
+// (if the Bazel process had an error, then the response will include an error).
+// Returns an error if there was an issue with the connection to the Bazel Proxy
+// server.
+func (b *ProxyClient) IssueCommand(req CmdRequest) (CmdResponse, error) {
+ var resp CmdResponse
+ var err error
+ // Check for connections every 1 second. This is chosen to be a relatively
+ // short timeout, because the proxy server should accept requests quite
+ // quickly.
+ d := net.Dialer{Timeout: 1 * time.Second}
+ var conn net.Conn
+ conn, err = d.Dial("unix", unixSocketPath(b.outDir))
+ if err != nil {
+ return resp, err
+ }
+ defer conn.Close()
+
+ enc := gob.NewEncoder(conn)
+ if err = enc.Encode(req); err != nil {
+ return resp, err
+ }
+ dec := gob.NewDecoder(conn)
+ err = dec.Decode(&resp)
+ return resp, err
+}
+
+// NewProxyServer is a constructor for a ProxyServer.
+func NewProxyServer(logger ServerLogger, outDir string, workspaceDir string) *ProxyServer {
+ return &ProxyServer{
+ logger: logger,
+ outDir: outDir,
+ workspaceDir: workspaceDir,
+ done: make(chan struct{}),
+ }
+}
+
+func (b *ProxyServer) handleRequest(conn net.Conn) error {
+ defer conn.Close()
+
+ dec := gob.NewDecoder(conn)
+ var req CmdRequest
+ if err := dec.Decode(&req); err != nil {
+ return fmt.Errorf("Error decoding request: %s", err)
+ }
+
+ bazelCmd := exec.Command("./build/bazel/bin/bazel", req.Argv...)
+ bazelCmd.Dir = b.workspaceDir
+ bazelCmd.Env = req.Env
+
+ stderr := &bytes.Buffer{}
+ bazelCmd.Stderr = stderr
+ var stdout string
+ var bazelErrString string
+
+ if output, err := bazelCmd.Output(); err != nil {
+ bazelErrString = fmt.Sprintf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
+ err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ } else {
+ stdout = string(output)
+ }
+
+ resp := CmdResponse{stdout, string(stderr.Bytes()), bazelErrString}
+ enc := gob.NewEncoder(conn)
+ if err := enc.Encode(&resp); err != nil {
+ return fmt.Errorf("Error encoding response: %s", err)
+ }
+ return nil
+}
+
+func (b *ProxyServer) listenUntilClosed(listener net.Listener) error {
+ for {
+ // Check for connections every 1 second. This is a blocking operation, so
+ // if the server is closed, the goroutine will not fully close until this
+ // deadline is reached. Thus, this deadline is short (but not too short
+ // so that the routine churns).
+ listener.(*net.UnixListener).SetDeadline(time.Now().Add(time.Second))
+ conn, err := listener.Accept()
+
+ select {
+ case <-b.done:
+ return nil
+ default:
+ }
+
+ if err != nil {
+ if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
+ // Timeout is normal and expected while waiting for client to establish
+ // a connection.
+ continue
+ } else {
+ b.logger.Fatalf("Listener error: %s", err)
+ }
+ }
+
+ err = b.handleRequest(conn)
+ if err != nil {
+ b.logger.Fatal(err)
+ }
+ }
+}
+
+// Start initializes the server unix socket and (in a separate goroutine)
+// handles requests on the socket until the server is closed. Returns an error
+// if a failure occurs during initialization. Will log any post-initialization
+// errors to the server's logger.
+func (b *ProxyServer) Start() error {
+ unixSocketAddr := unixSocketPath(b.outDir)
+ if err := os_lib.RemoveAll(unixSocketAddr); err != nil {
+ return fmt.Errorf("couldn't remove socket '%s': %s", unixSocketAddr, err)
+ }
+ listener, err := net.Listen("unix", unixSocketAddr)
+
+ if err != nil {
+ return fmt.Errorf("error listening on socket '%s': %s", unixSocketAddr, err)
+ }
+
+ go b.listenUntilClosed(listener)
+ return nil
+}
+
+// Close shuts down the server. This will stop the server from listening for
+// additional requests.
+func (b *ProxyServer) Close() {
+ b.done <- struct{}{}
+}
diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go
index 062eba8..d1dfb9d 100644
--- a/bp2build/bp2build.go
+++ b/bp2build/bp2build.go
@@ -17,21 +17,57 @@
import (
"fmt"
"os"
+ "path/filepath"
"strings"
"android/soong/android"
"android/soong/bazel"
+ "android/soong/shared"
)
+func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) {
+ // Delete files that should no longer be present.
+ bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String())
+
+ filesToDelete := make(map[string]struct{})
+ err := filepath.Walk(bp2buildDirAbs,
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() {
+ relPath, err := filepath.Rel(bp2buildDirAbs, path)
+ if err != nil {
+ return err
+ }
+ filesToDelete[relPath] = struct{}{}
+ }
+ return nil
+ })
+ if err != nil {
+ fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err)
+ os.Exit(1)
+ }
+
+ for _, bazelFile := range except {
+ filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename)
+ delete(filesToDelete, filePath)
+ }
+ for f, _ := range filesToDelete {
+ absPath := shared.JoinPath(bp2buildDirAbs, f)
+ if err := os.RemoveAll(absPath); err != nil {
+ fmt.Printf("ERROR deleting %s: %s", absPath, err)
+ os.Exit(1)
+ }
+ }
+}
+
// Codegen is the backend of bp2build. The code generator is responsible for
// writing .bzl files that are equivalent to Android.bp files that are capable
// of being built with Bazel.
func Codegen(ctx *CodegenContext) *CodegenMetrics {
// This directory stores BUILD files that could be eventually checked-in.
bp2buildDir := android.PathForOutput(ctx, "bp2build")
- if err := android.RemoveAllOutputDir(bp2buildDir); err != nil {
- fmt.Printf("ERROR: Encountered error while cleaning %s: %s", bp2buildDir, err.Error())
- }
res, errs := GenerateBazelTargets(ctx, true)
if len(errs) > 0 {
@@ -44,6 +80,12 @@
}
bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
+ // Delete files under the bp2build root which weren't just written. An
+ // alternative would have been to delete the whole directory and write these
+ // files. However, this would regenerate files which were otherwise unchanged
+ // since the last bp2build run, which would have negative incremental
+ // performance implications.
+ deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles)
injectionFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics)
if err != nil {
@@ -51,7 +93,6 @@
os.Exit(1)
}
writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)
-
return &res.metrics
}
diff --git a/cc/config/global.go b/cc/config/global.go
index d65f883..05dc773 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -192,10 +192,6 @@
}
noOverrideGlobalCflags = []string{
- // Workaround for boot loop caused by stack protector.
- // http://b/267839238
- "-mllvm -disable-check-noreturn-call",
-
"-Werror=bool-operation",
"-Werror=implicit-int-float-conversion",
"-Werror=int-in-bool-context",
@@ -257,7 +253,6 @@
"-Wno-bitwise-instead-of-logical",
"-Wno-misleading-indentation",
"-Wno-array-parameter",
- "-Wno-gnu-offsetof-extensions",
}
// Extra cflags for external third-party projects to disable warnings that
@@ -310,8 +305,8 @@
// prebuilts/clang default settings.
ClangDefaultBase = "prebuilts/clang/host"
- ClangDefaultVersion = "clang-r487747"
- ClangDefaultShortVersion = "17"
+ ClangDefaultVersion = "clang-r475365b"
+ ClangDefaultShortVersion = "16.0.2"
// Directories with warnings from Android.bp files.
WarningAllowedProjects = []string{
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 5f27fa7..5c187f6 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -81,6 +81,7 @@
flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
+ flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses")
// Flags that probably shouldn't be flags of soong_build, but we haven't found
// the time to remove them yet
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index fd718c2..ae026ba 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -200,6 +200,7 @@
rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
bazelMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bazel_metrics.pb")
+ soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
//the profile file generated by Bazel"
bazelProfileFile := filepath.Join(logsDir, c.logsPrefix+"analyzed_bazel_profile.txt")
@@ -209,6 +210,7 @@
bp2buildMetricsFile, // high level metrics related to bp2build.
soongMetricsFile, // high level metrics related to this build system.
bazelMetricsFile, // high level metrics related to bazel execution
+ soongBuildMetricsFile, // high level metrics related to soong build(except bp2build)
config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
}
diff --git a/java/app_test.go b/java/app_test.go
index c77f29d..5b16cea 100644
--- a/java/app_test.go
+++ b/java/app_test.go
@@ -1005,6 +1005,7 @@
platformSdkInt int
platformSdkCodename string
platformSdkFinal bool
+ minSdkVersionBp string
expectedMinSdkVersion string
platformApis bool
activeCodenames []string
@@ -1052,6 +1053,14 @@
platformSdkCodename: "S",
activeCodenames: []string{"S"},
},
+ {
+ name: "two active SDKs",
+ sdkVersion: "module_current",
+ minSdkVersionBp: "UpsideDownCake",
+ expectedMinSdkVersion: "UpsideDownCake", // And not VanillaIceCream
+ platformSdkCodename: "VanillaIceCream",
+ activeCodenames: []string{"UpsideDownCake", "VanillaIceCream"},
+ },
}
for _, moduleType := range []string{"android_app", "android_library"} {
@@ -1061,12 +1070,17 @@
if test.platformApis {
platformApiProp = "platform_apis: true,"
}
+ minSdkVersionProp := ""
+ if test.minSdkVersionBp != "" {
+ minSdkVersionProp = fmt.Sprintf(` min_sdk_version: "%s",`, test.minSdkVersionBp)
+ }
bp := fmt.Sprintf(`%s {
name: "foo",
srcs: ["a.java"],
sdk_version: "%s",
%s
- }`, moduleType, test.sdkVersion, platformApiProp)
+ %s
+ }`, moduleType, test.sdkVersion, platformApiProp, minSdkVersionProp)
result := android.GroupFixturePreparers(
prepareForJavaTest,
diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh
index 878b4a1..1ff1b5b 100755
--- a/tests/bp2build_bazel_test.sh
+++ b/tests/bp2build_bazel_test.sh
@@ -21,6 +21,68 @@
fi
}
+# Tests that, if bp2build reruns due to a blueprint file changing, that
+# BUILD files whose contents are unchanged are not regenerated.
+function test_bp2build_unchanged {
+ setup
+
+ mkdir -p pkg
+ touch pkg/x.txt
+ cat > pkg/Android.bp <<'EOF'
+filegroup {
+ name: "x",
+ srcs: ["x.txt"],
+ bazel_module: {bp2build_available: true},
+ }
+EOF
+
+ run_soong bp2build
+ local -r buildfile_mtime1=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)
+ local -r marker_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker)
+
+ # Force bp2build to rerun by updating the timestamp of a blueprint file.
+ touch pkg/Android.bp
+
+ run_soong bp2build
+ local -r buildfile_mtime2=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)
+ local -r marker_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker)
+
+ if [[ "$marker_mtime1" == "$marker_mtime2" ]]; then
+ fail "Expected bp2build marker file to change"
+ fi
+ if [[ "$buildfile_mtime1" != "$buildfile_mtime2" ]]; then
+ fail "BUILD.bazel was updated even though contents are same"
+ fi
+}
+
+# Tests that blueprint files that are deleted are not present when the
+# bp2build tree is regenerated.
+function test_bp2build_deleted_blueprint {
+ setup
+
+ mkdir -p pkg
+ touch pkg/x.txt
+ cat > pkg/Android.bp <<'EOF'
+filegroup {
+ name: "x",
+ srcs: ["x.txt"],
+ bazel_module: {bp2build_available: true},
+ }
+EOF
+
+ run_soong bp2build
+ if [[ ! -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then
+ fail "Expected pkg/BUILD.bazel to be generated"
+ fi
+
+ rm pkg/Android.bp
+
+ run_soong bp2build
+ if [[ -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then
+ fail "Expected pkg/BUILD.bazel to be deleted"
+ fi
+}
+
function test_bp2build_null_build_with_globs {
setup
diff --git a/tests/persistent_bazel_test.sh b/tests/persistent_bazel_test.sh
new file mode 100755
index 0000000..4e2982a
--- /dev/null
+++ b/tests/persistent_bazel_test.sh
@@ -0,0 +1,83 @@
+#!/bin/bash -eu
+
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o pipefail
+
+source "$(dirname "$0")/lib.sh"
+
+# This test verifies that adding USE_PERSISTENT_BAZEL creates a Bazel process
+# that outlasts the build process.
+# This test should only be run in sandboxed environments (because this test
+# verifies a Bazel process using global process list, and may spawn lingering
+# Bazel processes).
+function test_persistent_bazel {
+ setup
+
+ # Ensure no existing Bazel process.
+ if [[ -e out/bazel/output/server/server.pid.txt ]]; then
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+ if kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Error killing pre-setup bazel"
+ fi
+ fi
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing
+
+ if ! kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Persistent bazel process expected, but not found after first build"
+ fi
+ BAZEL_PID=$(cat out/bazel/output/server/server.pid.txt)
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing
+
+ if ! kill -0 $BAZEL_PID 2>/dev/null ; then
+ fail "Bazel pid $BAZEL_PID was killed after second build"
+ fi
+
+ kill $BAZEL_PID 2>/dev/null
+ if ! kill -0 $BAZEL_PID 2>/dev/null ; then
+ fail "Error killing bazel on shutdown"
+ fi
+}
+
+# Verifies that USE_PERSISTENT_BAZEL mode operates as expected in the event
+# that there are Bazel failures.
+function test_bazel_failure {
+ setup
+
+ # Ensure no existing Bazel process.
+ if [[ -e out/bazel/output/server/server.pid.txt ]]; then
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+ if kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Error killing pre-setup bazel"
+ fi
+ fi
+
+ # Introduce a syntax error in a BUILD file which is used in every build
+ # (Note this is a BUILD file which is copied as part of test setup, so this
+ # has no effect on sources outside of this test.
+ rm -rf build/bazel/rules
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing 1>out/failurelog.txt 2>&1 && fail "Expected build failure" || true
+
+ if ! grep -sq "'build/bazel/rules' is not a package" out/failurelog.txt ; then
+ fail "Expected error to contain 'build/bazel/rules' is not a package, instead got:\n$(cat out/failurelog.txt)"
+ fi
+
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+}
+
+scan_and_run_tests
diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh
index 8ba2984..a91ccf4 100755
--- a/tests/run_integration_tests.sh
+++ b/tests/run_integration_tests.sh
@@ -7,6 +7,7 @@
"$TOP/build/soong/tests/bootstrap_test.sh"
"$TOP/build/soong/tests/mixed_mode_test.sh"
"$TOP/build/soong/tests/bp2build_bazel_test.sh"
+"$TOP/build/soong/tests/persistent_bazel_test.sh"
"$TOP/build/soong/tests/soong_test.sh"
"$TOP/build/bazel/ci/rbc_regression_test.sh" aosp_arm64-userdebug
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 7a8fca9..b79754c 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -50,6 +50,7 @@
"cleanbuild.go",
"config.go",
"context.go",
+ "staging_snapshot.go",
"dumpvars.go",
"environment.go",
"exec.go",
@@ -70,10 +71,11 @@
"cleanbuild_test.go",
"config_test.go",
"environment_test.go",
+ "proc_sync_test.go",
"rbe_test.go",
+ "staging_snapshot_test.go",
"upload_test.go",
"util_test.go",
- "proc_sync_test.go",
],
darwin: {
srcs: [
diff --git a/ui/build/build.go b/ui/build/build.go
index d49a754..edc595d 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -102,9 +102,9 @@
// Whether to include the kati-generated ninja file in the combined ninja.
RunKatiNinja = 1 << iota
// Whether to run ninja on the combined ninja.
- RunNinja = 1 << iota
- RunBuildTests = 1 << iota
- RunAll = RunProductConfig | RunSoong | RunKati | RunKatiNinja | RunNinja
+ RunNinja = 1 << iota
+ RunDistActions = 1 << iota
+ RunBuildTests = 1 << iota
)
// checkBazelMode fails the build if there are conflicting arguments for which bazel
@@ -322,34 +322,42 @@
runNinjaForBuild(ctx, config)
}
+
+ if what&RunDistActions != 0 {
+ runDistActions(ctx, config)
+ }
}
func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int {
//evaluate what to run
- what := RunAll
+ what := 0
if config.Checkbuild() {
what |= RunBuildTests
}
- if config.SkipConfig() {
+ if !config.SkipConfig() {
+ what |= RunProductConfig
+ } else {
verboseln("Skipping Config as requested")
- what = what &^ RunProductConfig
}
- if config.SkipKati() {
- verboseln("Skipping Kati as requested")
- what = what &^ RunKati
- }
- if config.SkipKatiNinja() {
- verboseln("Skipping use of Kati ninja as requested")
- what = what &^ RunKatiNinja
- }
- if config.SkipSoong() {
+ if !config.SkipSoong() {
+ what |= RunSoong
+ } else {
verboseln("Skipping use of Soong as requested")
- what = what &^ RunSoong
}
-
- if config.SkipNinja() {
+ if !config.SkipKati() {
+ what |= RunKati
+ } else {
+ verboseln("Skipping Kati as requested")
+ }
+ if !config.SkipKatiNinja() {
+ what |= RunKatiNinja
+ } else {
+ verboseln("Skipping use of Kati ninja as requested")
+ }
+ if !config.SkipNinja() {
+ what |= RunNinja
+ } else {
verboseln("Skipping Ninja as requested")
- what = what &^ RunNinja
}
if !config.SoongBuildInvocationNeeded() {
@@ -361,6 +369,11 @@
what = what &^ RunNinja
what = what &^ RunKati
}
+
+ if config.Dist() {
+ what |= RunDistActions
+ }
+
return what
}
@@ -419,3 +432,9 @@
}
}()
}
+
+// Actions to run on every build where 'dist' is in the actions.
+// Be careful, anything added here slows down EVERY CI build
+func runDistActions(ctx Context, config Config) {
+ runStagingSnapshot(ctx, config)
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index 73e2c45..b5ee440 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1568,6 +1568,10 @@
return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
}
+func (c *configImpl) IsPersistentBazelEnabled() bool {
+ return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
+}
+
func (c *configImpl) BazelModulesForceEnabledByFlag() string {
return c.bazelForceEnabledModules
}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index e6543ec..a5a3263 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -21,6 +21,7 @@
"strconv"
"strings"
+ "android/soong/bazel"
"android/soong/ui/metrics"
"android/soong/ui/status"
@@ -268,6 +269,9 @@
if config.bazelStagingMode {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
}
+ if config.IsPersistentBazelEnabled() {
+ mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
+ }
if len(config.bazelForceEnabledModules) > 0 {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
}
@@ -497,6 +501,12 @@
ctx.BeginTrace(metrics.RunSoong, name)
defer ctx.EndTrace()
+ if config.IsPersistentBazelEnabled() {
+ bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"))
+ bazelProxy.Start()
+ defer bazelProxy.Close()
+ }
+
fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close()
diff --git a/ui/build/staging_snapshot.go b/ui/build/staging_snapshot.go
new file mode 100644
index 0000000..377aa64
--- /dev/null
+++ b/ui/build/staging_snapshot.go
@@ -0,0 +1,246 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "crypto/sha1"
+ "encoding/hex"
+ "encoding/json"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "android/soong/shared"
+ "android/soong/ui/metrics"
+)
+
+// Metadata about a staged file
+type fileEntry struct {
+ Name string `json:"name"`
+ Mode fs.FileMode `json:"mode"`
+ Size int64 `json:"size"`
+ Sha1 string `json:"sha1"`
+}
+
+func fileEntryEqual(a fileEntry, b fileEntry) bool {
+ return a.Name == b.Name && a.Mode == b.Mode && a.Size == b.Size && a.Sha1 == b.Sha1
+}
+
+func sha1_hash(filename string) (string, error) {
+ f, err := os.Open(filename)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+
+ h := sha1.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+
+ return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// Subdirs of PRODUCT_OUT to scan
+var stagingSubdirs = []string{
+ "apex",
+ "cache",
+ "coverage",
+ "data",
+ "debug_ramdisk",
+ "fake_packages",
+ "installer",
+ "oem",
+ "product",
+ "ramdisk",
+ "recovery",
+ "root",
+ "sysloader",
+ "system",
+ "system_dlkm",
+ "system_ext",
+ "system_other",
+ "testcases",
+ "test_harness_ramdisk",
+ "vendor",
+ "vendor_debug_ramdisk",
+ "vendor_kernel_ramdisk",
+ "vendor_ramdisk",
+}
+
+// Return an array of stagedFileEntrys, one for each file in the staging directories inside
+// productOut
+func takeStagingSnapshot(ctx Context, productOut string, subdirs []string) ([]fileEntry, error) {
+ var outer_err error
+ if !strings.HasSuffix(productOut, "/") {
+ productOut += "/"
+ }
+ result := []fileEntry{}
+ for _, subdir := range subdirs {
+ filepath.WalkDir(productOut+subdir,
+ func(filename string, dirent fs.DirEntry, err error) error {
+ // Ignore errors. The most common one is that one of the subdirectories
+ // hasn't been built, in which case we just report it as empty.
+ if err != nil {
+ ctx.Verbosef("scanModifiedStagingOutputs error: %s", err)
+ return nil
+ }
+ if dirent.Type().IsRegular() {
+ fileInfo, _ := dirent.Info()
+ relative := strings.TrimPrefix(filename, productOut)
+ sha, err := sha1_hash(filename)
+ if err != nil {
+ outer_err = err
+ }
+ result = append(result, fileEntry{
+ Name: relative,
+ Mode: fileInfo.Mode(),
+ Size: fileInfo.Size(),
+ Sha1: sha,
+ })
+ }
+ return nil
+ })
+ }
+
+ sort.Slice(result, func(l, r int) bool { return result[l].Name < result[r].Name })
+
+ return result, outer_err
+}
+
+// Read json into an array of fileEntry. On error return empty array.
+func readJson(filename string) ([]fileEntry, error) {
+ buf, err := os.ReadFile(filename)
+ if err != nil {
+ // Not an error, just missing, which is empty.
+ return []fileEntry{}, nil
+ }
+
+ var result []fileEntry
+ err = json.Unmarshal(buf, &result)
+ if err != nil {
+ // Bad formatting. This is an error
+ return []fileEntry{}, err
+ }
+
+ return result, nil
+}
+
+// Write obj to filename.
+func writeJson(filename string, obj interface{}) error {
+ buf, err := json.MarshalIndent(obj, "", " ")
+ if err != nil {
+ return err
+ }
+
+ return os.WriteFile(filename, buf, 0660)
+}
+
+type snapshotDiff struct {
+ Added []string `json:"added"`
+ Changed []string `json:"changed"`
+ Removed []string `json:"removed"`
+}
+
+// Diff the two snapshots, returning a snapshotDiff.
+func diffSnapshots(previous []fileEntry, current []fileEntry) snapshotDiff {
+ result := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ found := make(map[string]bool)
+
+ prev := make(map[string]fileEntry)
+ for _, pre := range previous {
+ prev[pre.Name] = pre
+ }
+
+ for _, cur := range current {
+ pre, ok := prev[cur.Name]
+ found[cur.Name] = true
+ // Added
+ if !ok {
+ result.Added = append(result.Added, cur.Name)
+ continue
+ }
+ // Changed
+ if !fileEntryEqual(pre, cur) {
+ result.Changed = append(result.Changed, cur.Name)
+ }
+ }
+
+ // Removed
+ for _, pre := range previous {
+ if !found[pre.Name] {
+ result.Removed = append(result.Removed, pre.Name)
+ }
+ }
+
+ // Sort the results
+ sort.Strings(result.Added)
+ sort.Strings(result.Changed)
+ sort.Strings(result.Removed)
+
+ return result
+}
+
+// Write a json files to dist:
+// - A list of which files have changed in this build.
+//
+// And record in out/soong:
+// - A list of all files in the staging directories, including their hashes.
+func runStagingSnapshot(ctx Context, config Config) {
+ ctx.BeginTrace(metrics.RunSoong, "runStagingSnapshot")
+ defer ctx.EndTrace()
+
+ snapshotFilename := shared.JoinPath(config.SoongOutDir(), "staged_files.json")
+
+ // Read the existing snapshot file. If it doesn't exist, this is a full
+ // build, so all files will be treated as new.
+ previous, err := readJson(snapshotFilename)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Take a snapshot of the current out directory
+ current, err := takeStagingSnapshot(ctx, config.ProductOut(), stagingSubdirs)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Diff the snapshots
+ diff := diffSnapshots(previous, current)
+
+ // Write the diff (use RealDistDir, not one that might have been faked for bazel)
+ err = writeJson(shared.JoinPath(config.RealDistDir(), "modified_files.json"), diff)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Update the snapshot
+ err = writeJson(snapshotFilename, current)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+}
diff --git a/ui/build/staging_snapshot_test.go b/ui/build/staging_snapshot_test.go
new file mode 100644
index 0000000..7ac5443
--- /dev/null
+++ b/ui/build/staging_snapshot_test.go
@@ -0,0 +1,188 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+func assertDeepEqual(t *testing.T, expected interface{}, actual interface{}) {
+ if !reflect.DeepEqual(actual, expected) {
+ t.Fatalf("expected:\n %#v\n actual:\n %#v", expected, actual)
+ }
+}
+
+// Make a temp directory containing the supplied contents
+func makeTempDir(files []string, directories []string, symlinks []string) string {
+ temp, _ := os.MkdirTemp("", "soon_staging_snapshot_test_")
+
+ for _, file := range files {
+ os.MkdirAll(temp+"/"+filepath.Dir(file), 0700)
+ os.WriteFile(temp+"/"+file, []byte(file), 0600)
+ }
+
+ for _, dir := range directories {
+ os.MkdirAll(temp+"/"+dir, 0770)
+ }
+
+ for _, symlink := range symlinks {
+ os.MkdirAll(temp+"/"+filepath.Dir(symlink), 0770)
+ os.Symlink(temp, temp+"/"+symlink)
+ }
+
+ return temp
+}
+
+// If this is a clean build, we won't have any preexisting files, make sure we get back an empty
+// list and not errors.
+func TestEmptyOut(t *testing.T) {
+ ctx := testContext()
+
+ temp := makeTempDir(nil, nil, nil)
+ defer os.RemoveAll(temp)
+
+ actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"})
+
+ expected := []fileEntry{}
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure only the listed directories are picked up, and only regular files
+func TestNoExtraSubdirs(t *testing.T) {
+ ctx := testContext()
+
+ temp := makeTempDir([]string{"a/b", "a/c", "d", "e/f"}, []string{"g/h"}, []string{"e/symlink"})
+ defer os.RemoveAll(temp)
+
+ actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"})
+
+ expected := []fileEntry{
+ {"a/b", 0600, 3, "3ec69c85a4ff96830024afeef2d4e512181c8f7b"},
+ {"a/c", 0600, 3, "592d70e4e03ee6f6780c71b0bf3b9608dbf1e201"},
+ {"e/f", 0600, 3, "9e164bef74aceede0974b857170100409efe67f1"},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles empty lists
+func TestDiffEmpty(t *testing.T) {
+ actual := diffSnapshots(nil, []fileEntry{})
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles adding
+func TestDiffAdd(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{"b"},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing mode
+func TestDiffChangeMode(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0600, 2, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing size
+func TestDiffChangeSize(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 3, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing contents
+func TestDiffChangeContents(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "aaaa"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles removing
+func TestDiffRemove(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{"b"},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}