Pass rsp files into sbox and rewrapper
The current implementation causes inputs listed in an rsp file used with
sbox to be duplicated 3 times in the build.ninja file; once as a
dependency of the rule, once in the rspfile_content field of the rule
with the paths rewritten to be relative to the sandbox, and once in the
rule to write the sbox manifest. When RBE is enabled it also gets a
fourth copy in the list of files to be treated as inputs by rewrapper.
Reduce this to a single copy by using "$in" for the rspfile_content so
that the files only have to be listed in the input dependencies of the
rule, and then add support to sbox to rewrite the rsp file while copying
it into the sandbox, and pass it to rewrapper as well.
Test: m lint-check
Change-Id: I3f46f61119508d39a8bb231c99fc130153fb6f04
diff --git a/cmd/sbox/Android.bp b/cmd/sbox/Android.bp
index d88505f..b8d75ed 100644
--- a/cmd/sbox/Android.bp
+++ b/cmd/sbox/Android.bp
@@ -21,6 +21,7 @@
deps: [
"sbox_proto",
"soong-makedeps",
+ "soong-response",
],
srcs: [
"sbox.go",
diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go
index f47c601..fcc80a9 100644
--- a/cmd/sbox/sbox.go
+++ b/cmd/sbox/sbox.go
@@ -32,6 +32,7 @@
"android/soong/cmd/sbox/sbox_proto"
"android/soong/makedeps"
+ "android/soong/response"
"github.com/golang/protobuf/proto"
)
@@ -218,6 +219,11 @@
return "", fmt.Errorf("command is required")
}
+ pathToTempDirInSbox := tempDir
+ if command.GetChdir() {
+ pathToTempDirInSbox = "."
+ }
+
err = os.MkdirAll(tempDir, 0777)
if err != nil {
return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
@@ -228,10 +234,9 @@
if err != nil {
return "", err
}
-
- pathToTempDirInSbox := tempDir
- if command.GetChdir() {
- pathToTempDirInSbox = "."
+ err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
+ if err != nil {
+ return "", err
}
if strings.Contains(rawCommand, depFilePlaceholder) {
@@ -409,6 +414,83 @@
return nil
}
+// copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
+// listed into the sandbox.
+func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
+ for _, rspFile := range rspFiles {
+ err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
+// listed into the sandbox.
+func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
+ in, err := os.Open(rspFile.GetFile())
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+
+ files, err := response.ReadRspFile(in)
+ if err != nil {
+ return err
+ }
+
+ for i, from := range files {
+ // Convert the real path of the input file into the path inside the sandbox using the
+ // path mappings.
+ to := applyPathMappings(rspFile.PathMappings, from)
+
+ // Copy the file into the sandbox.
+ err := copyOneFile(from, joinPath(toDir, to), false)
+ if err != nil {
+ return err
+ }
+
+ // Rewrite the name in the list of files to be relative to the sandbox directory.
+ files[i] = joinPath(toDirInSandbox, to)
+ }
+
+ // Convert the real path of the rsp file into the path inside the sandbox using the path
+ // mappings.
+ outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
+
+ err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
+ if err != nil {
+ return err
+ }
+
+ out, err := os.Create(outRspFile)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ // Write the rsp file with converted paths into the sandbox.
+ err = response.WriteRspFile(out, files)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// applyPathMappings takes a list of path mappings and a path, and returns the path with the first
+// matching path mapping applied. If the path does not match any of the path mappings then it is
+// returned unmodified.
+func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
+ for _, mapping := range pathMappings {
+ if strings.HasPrefix(path, mapping.GetFrom()+"/") {
+ return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
+ }
+ }
+ return path
+}
+
// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
// to moving files where the source and destination are in the same filesystem. This is OK for
// sbox because the temporary directory is inside the out directory. It updates the timestamp
diff --git a/cmd/sbox/sbox_proto/sbox.pb.go b/cmd/sbox/sbox_proto/sbox.pb.go
index 79bb90c..b996481 100644
--- a/cmd/sbox/sbox_proto/sbox.pb.go
+++ b/cmd/sbox/sbox_proto/sbox.pb.go
@@ -86,10 +86,13 @@
CopyAfter []*Copy `protobuf:"bytes,4,rep,name=copy_after,json=copyAfter" json:"copy_after,omitempty"`
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
- InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
+ InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
+ // A list of files that will be copied before the sandboxed command, and whose contents should be
+ // copied as if they were listed in copy_before.
+ RspFiles []*RspFile `protobuf:"bytes,6,rep,name=rsp_files,json=rspFiles" json:"rsp_files,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
}
func (m *Command) Reset() { *m = Command{} }
@@ -152,6 +155,13 @@
return ""
}
+func (m *Command) GetRspFiles() []*RspFile {
+ if m != nil {
+ return m.RspFiles
+ }
+ return nil
+}
+
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
// are relative to is specific to the context the Copy is used in and will be different for
// from and to.
@@ -211,10 +221,110 @@
return false
}
+// RspFile describes an rspfile that should be copied into the sandbox directory.
+type RspFile struct {
+ // The path to the rsp file.
+ File *string `protobuf:"bytes,1,req,name=file" json:"file,omitempty"`
+ // A list of path mappings that should be applied to each file listed in the rsp file.
+ PathMappings []*PathMapping `protobuf:"bytes,2,rep,name=path_mappings,json=pathMappings" json:"path_mappings,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *RspFile) Reset() { *m = RspFile{} }
+func (m *RspFile) String() string { return proto.CompactTextString(m) }
+func (*RspFile) ProtoMessage() {}
+func (*RspFile) Descriptor() ([]byte, []int) {
+ return fileDescriptor_9d0425bf0de86ed1, []int{3}
+}
+
+func (m *RspFile) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_RspFile.Unmarshal(m, b)
+}
+func (m *RspFile) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_RspFile.Marshal(b, m, deterministic)
+}
+func (m *RspFile) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_RspFile.Merge(m, src)
+}
+func (m *RspFile) XXX_Size() int {
+ return xxx_messageInfo_RspFile.Size(m)
+}
+func (m *RspFile) XXX_DiscardUnknown() {
+ xxx_messageInfo_RspFile.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RspFile proto.InternalMessageInfo
+
+func (m *RspFile) GetFile() string {
+ if m != nil && m.File != nil {
+ return *m.File
+ }
+ return ""
+}
+
+func (m *RspFile) GetPathMappings() []*PathMapping {
+ if m != nil {
+ return m.PathMappings
+ }
+ return nil
+}
+
+// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
+type PathMapping struct {
+ From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"`
+ To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PathMapping) Reset() { *m = PathMapping{} }
+func (m *PathMapping) String() string { return proto.CompactTextString(m) }
+func (*PathMapping) ProtoMessage() {}
+func (*PathMapping) Descriptor() ([]byte, []int) {
+ return fileDescriptor_9d0425bf0de86ed1, []int{4}
+}
+
+func (m *PathMapping) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PathMapping.Unmarshal(m, b)
+}
+func (m *PathMapping) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PathMapping.Marshal(b, m, deterministic)
+}
+func (m *PathMapping) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PathMapping.Merge(m, src)
+}
+func (m *PathMapping) XXX_Size() int {
+ return xxx_messageInfo_PathMapping.Size(m)
+}
+func (m *PathMapping) XXX_DiscardUnknown() {
+ xxx_messageInfo_PathMapping.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PathMapping proto.InternalMessageInfo
+
+func (m *PathMapping) GetFrom() string {
+ if m != nil && m.From != nil {
+ return *m.From
+ }
+ return ""
+}
+
+func (m *PathMapping) GetTo() string {
+ if m != nil && m.To != nil {
+ return *m.To
+ }
+ return ""
+}
+
func init() {
proto.RegisterType((*Manifest)(nil), "sbox.Manifest")
proto.RegisterType((*Command)(nil), "sbox.Command")
proto.RegisterType((*Copy)(nil), "sbox.Copy")
+ proto.RegisterType((*RspFile)(nil), "sbox.RspFile")
+ proto.RegisterType((*PathMapping)(nil), "sbox.PathMapping")
}
func init() {
@@ -222,22 +332,27 @@
}
var fileDescriptor_9d0425bf0de86ed1 = []byte{
- // 268 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x4f, 0x4b, 0xc3, 0x40,
- 0x10, 0xc5, 0xc9, 0x9f, 0xd2, 0x64, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x09, 0x01, 0x21, 0x45,
- 0xe8, 0xc1, 0x6f, 0x60, 0xf5, 0x20, 0x82, 0x97, 0x1c, 0x45, 0x08, 0x9b, 0x64, 0x43, 0x02, 0x4d,
- 0x26, 0xec, 0x6e, 0xa0, 0xfd, 0x56, 0x7e, 0x44, 0xd9, 0x49, 0x2a, 0x82, 0xb7, 0x99, 0xdf, 0xe3,
- 0xcd, 0x7b, 0x0c, 0x80, 0x29, 0xe9, 0x7c, 0x18, 0x35, 0x59, 0xc2, 0xd0, 0xcd, 0xe9, 0x17, 0x44,
- 0x1f, 0x72, 0xe8, 0x1a, 0x65, 0x2c, 0xee, 0x21, 0xaa, 0xa8, 0xef, 0xe5, 0x50, 0x1b, 0xe1, 0x25,
- 0x41, 0xb6, 0x79, 0xda, 0x1e, 0xd8, 0xf0, 0x32, 0xd3, 0xfc, 0x57, 0xc6, 0x07, 0xd8, 0xd1, 0x64,
- 0xc7, 0xc9, 0x16, 0xb5, 0x1a, 0x9b, 0xee, 0xa4, 0x84, 0x9f, 0x78, 0x59, 0x9c, 0x6f, 0x67, 0xfa,
- 0x3a, 0xc3, 0xf4, 0xdb, 0x83, 0xf5, 0x62, 0xc6, 0x47, 0xd8, 0x54, 0x34, 0x5e, 0x8a, 0x52, 0x35,
- 0xa4, 0xd5, 0x12, 0x00, 0xd7, 0x80, 0xf1, 0x92, 0x83, 0x93, 0x8f, 0xac, 0xe2, 0x2d, 0xac, 0xaa,
- 0xb6, 0xee, 0x34, 0x9f, 0x8d, 0xf2, 0x79, 0x41, 0x01, 0xeb, 0xa5, 0x81, 0x08, 0x12, 0x3f, 0x8b,
- 0xf3, 0xeb, 0x8a, 0x7b, 0x60, 0x77, 0x21, 0x1b, 0xab, 0xb4, 0x08, 0xff, 0xdd, 0x8e, 0x9d, 0xfa,
- 0xec, 0x44, 0xbc, 0x03, 0xe8, 0x06, 0xd7, 0xbc, 0x95, 0xa6, 0x15, 0x2b, 0xae, 0x1d, 0x33, 0x79,
- 0x93, 0xa6, 0x4d, 0xdf, 0x21, 0x74, 0x0e, 0x44, 0x08, 0x1b, 0x4d, 0xbd, 0xf0, 0x38, 0x88, 0x67,
- 0xdc, 0x81, 0x6f, 0x49, 0xf8, 0x4c, 0x7c, 0x4b, 0x78, 0x0f, 0xa0, 0xce, 0xaa, 0x9a, 0xac, 0x2c,
- 0x4f, 0x4a, 0x04, 0x5c, 0xf5, 0x0f, 0x39, 0xde, 0x7c, 0xf2, 0xc3, 0x0b, 0x7e, 0xf8, 0x4f, 0x00,
- 0x00, 0x00, 0xff, 0xff, 0x78, 0x37, 0x3e, 0x6a, 0x7d, 0x01, 0x00, 0x00,
+ // 342 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x5f, 0x4b, 0xc3, 0x30,
+ 0x14, 0xc5, 0x69, 0xd7, 0xb9, 0xf6, 0xee, 0x0f, 0x18, 0x7c, 0xc8, 0x8b, 0x32, 0x0a, 0xc2, 0xa6,
+ 0x30, 0xd0, 0x07, 0xdf, 0x9d, 0x22, 0x22, 0x0c, 0x24, 0xe0, 0x8b, 0x08, 0x25, 0xeb, 0x52, 0x5b,
+ 0x58, 0x9b, 0x90, 0x64, 0xb0, 0x7d, 0x57, 0x3f, 0x8c, 0xe4, 0xa6, 0xd3, 0x82, 0x2f, 0xbe, 0xdd,
+ 0x7b, 0x0e, 0xf7, 0xdc, 0x5f, 0xc2, 0x05, 0x30, 0x6b, 0xb9, 0x5f, 0x28, 0x2d, 0xad, 0x24, 0x91,
+ 0xab, 0xd3, 0x0f, 0x88, 0x57, 0xbc, 0xa9, 0x0a, 0x61, 0x2c, 0x99, 0x43, 0x9c, 0xcb, 0xba, 0xe6,
+ 0xcd, 0xc6, 0xd0, 0x60, 0xda, 0x9b, 0x0d, 0x6f, 0xc7, 0x0b, 0x1c, 0x78, 0xf0, 0x2a, 0xfb, 0xb1,
+ 0xc9, 0x25, 0x4c, 0xe4, 0xce, 0xaa, 0x9d, 0xcd, 0x36, 0x42, 0x15, 0xd5, 0x56, 0xd0, 0x70, 0x1a,
+ 0xcc, 0x12, 0x36, 0xf6, 0xea, 0xa3, 0x17, 0xd3, 0xaf, 0x00, 0x06, 0xed, 0x30, 0xb9, 0x86, 0x61,
+ 0x2e, 0xd5, 0x21, 0x5b, 0x8b, 0x42, 0x6a, 0xd1, 0x2e, 0x80, 0xe3, 0x02, 0x75, 0x60, 0xe0, 0xec,
+ 0x25, 0xba, 0xe4, 0x0c, 0xfa, 0x79, 0xb9, 0xa9, 0x34, 0xc6, 0xc6, 0xcc, 0x37, 0x84, 0xc2, 0xa0,
+ 0x25, 0xa0, 0xbd, 0x69, 0x38, 0x4b, 0xd8, 0xb1, 0x25, 0x73, 0xc0, 0xe9, 0x8c, 0x17, 0x56, 0x68,
+ 0x1a, 0xfd, 0xc9, 0x4e, 0x9c, 0x7b, 0xef, 0x4c, 0x72, 0x0e, 0x50, 0x35, 0x8e, 0xbc, 0xe4, 0xa6,
+ 0xa4, 0x7d, 0xc4, 0x4e, 0x50, 0x79, 0xe6, 0xa6, 0x24, 0x57, 0x90, 0x68, 0xa3, 0x32, 0x87, 0x6f,
+ 0xe8, 0x49, 0xf7, 0x17, 0x98, 0x51, 0x4f, 0xd5, 0x56, 0xb0, 0x58, 0xfb, 0xc2, 0xa4, 0x2f, 0x10,
+ 0xb9, 0x74, 0x42, 0x20, 0x2a, 0xb4, 0xac, 0x69, 0x80, 0x50, 0x58, 0x93, 0x09, 0x84, 0x56, 0xd2,
+ 0x10, 0x95, 0xd0, 0x4a, 0x72, 0x01, 0x20, 0xf6, 0x22, 0xdf, 0x59, 0xbe, 0xde, 0x0a, 0xda, 0xc3,
+ 0x67, 0x75, 0x94, 0xf4, 0x0d, 0x06, 0xed, 0x02, 0x8c, 0x73, 0x5f, 0x7a, 0x8c, 0x73, 0xda, 0x1d,
+ 0x8c, 0x15, 0xb7, 0x65, 0x56, 0x73, 0xa5, 0xaa, 0xe6, 0xd3, 0xd0, 0x10, 0xd1, 0x4e, 0x3d, 0xda,
+ 0x2b, 0xb7, 0xe5, 0xca, 0x3b, 0x6c, 0xa4, 0x7e, 0x1b, 0x93, 0xde, 0xc0, 0xb0, 0x63, 0xfe, 0x87,
+ 0x74, 0x39, 0x7a, 0xc7, 0x33, 0xc9, 0xf0, 0x4c, 0xbe, 0x03, 0x00, 0x00, 0xff, 0xff, 0x83, 0x82,
+ 0xb0, 0xc3, 0x33, 0x02, 0x00, 0x00,
}
diff --git a/cmd/sbox/sbox_proto/sbox.proto b/cmd/sbox/sbox_proto/sbox.proto
index 695b0e8..bdf92c6 100644
--- a/cmd/sbox/sbox_proto/sbox.proto
+++ b/cmd/sbox/sbox_proto/sbox.proto
@@ -47,6 +47,10 @@
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
optional string input_hash = 5;
+
+ // A list of files that will be copied before the sandboxed command, and whose contents should be
+ // copied as if they were listed in copy_before.
+ repeated RspFile rsp_files = 6;
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
@@ -58,4 +62,19 @@
// If true, make the file executable after copying it.
optional bool executable = 3;
-}
\ No newline at end of file
+}
+
+// RspFile describes an rspfile that should be copied into the sandbox directory.
+message RspFile {
+ // The path to the rsp file.
+ required string file = 1;
+
+ // A list of path mappings that should be applied to each file listed in the rsp file.
+ repeated PathMapping path_mappings = 2;
+}
+
+// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
+message PathMapping {
+ required string from = 1;
+ required string to = 2;
+}