blob: e69a93067d0a8c167d07b5cd9ce5b1fbbc3bdff3 [file] [log] [blame]
Jeff Gastonefc1b412017-03-29 17:29:06 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
Dan Willemsenc89b6f12019-08-29 14:47:40 -070018 "bytes"
Ulf Adamsb73daa52020-11-25 23:09:09 +010019 "crypto/sha1"
20 "encoding/hex"
Jeff Gaston90cfb092017-09-26 16:46:10 -070021 "errors"
Jeff Gaston93f0f372017-11-01 13:33:02 -070022 "flag"
Jeff Gastonefc1b412017-03-29 17:29:06 -070023 "fmt"
Colin Crosse16ce362020-11-12 08:29:30 -080024 "io"
Cole Faustca355af2023-12-14 23:38:38 +000025 "io/fs"
Jeff Gastonefc1b412017-03-29 17:29:06 -070026 "io/ioutil"
27 "os"
28 "os/exec"
Jeff Gastonefc1b412017-03-29 17:29:06 -070029 "path/filepath"
Colin Crosse16ce362020-11-12 08:29:30 -080030 "strconv"
Jeff Gastonefc1b412017-03-29 17:29:06 -070031 "strings"
Colin Crossd1c1e6f2019-03-29 13:54:39 -070032 "time"
Dan Willemsenc89b6f12019-08-29 14:47:40 -070033
Colin Crosse16ce362020-11-12 08:29:30 -080034 "android/soong/cmd/sbox/sbox_proto"
Dan Willemsenc89b6f12019-08-29 14:47:40 -070035 "android/soong/makedeps"
Colin Crosse55bd422021-03-23 13:44:30 -070036 "android/soong/response"
Colin Crosse16ce362020-11-12 08:29:30 -080037
Dan Willemsen4591b642021-05-24 14:24:12 -070038 "google.golang.org/protobuf/encoding/prototext"
Jeff Gastonefc1b412017-03-29 17:29:06 -070039)
40
Jeff Gaston93f0f372017-11-01 13:33:02 -070041var (
Colin Crosse52c2ac2022-03-28 17:03:35 -070042 sandboxesRoot string
43 outputDir string
44 manifestFile string
45 keepOutDir bool
46 writeIfChanged bool
Colin Crosse16ce362020-11-12 08:29:30 -080047)
48
49const (
50 depFilePlaceholder = "__SBOX_DEPFILE__"
51 sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
Jeff Gaston93f0f372017-11-01 13:33:02 -070052)
53
54func init() {
55 flag.StringVar(&sandboxesRoot, "sandbox-path", "",
56 "root of temp directory to put the sandbox into")
Colin Crosse52c2ac2022-03-28 17:03:35 -070057 flag.StringVar(&outputDir, "output-dir", "",
58 "directory which will contain all output files and only output files")
Colin Crosse16ce362020-11-12 08:29:30 -080059 flag.StringVar(&manifestFile, "manifest", "",
60 "textproto manifest describing the sandboxed command(s)")
Jeff Gaston93f0f372017-11-01 13:33:02 -070061 flag.BoolVar(&keepOutDir, "keep-out-dir", false,
62 "whether to keep the sandbox directory when done")
Colin Crosse52c2ac2022-03-28 17:03:35 -070063 flag.BoolVar(&writeIfChanged, "write-if-changed", false,
64 "only write the output files if they have changed")
Jeff Gaston93f0f372017-11-01 13:33:02 -070065}
66
67func usageViolation(violation string) {
68 if violation != "" {
69 fmt.Fprintf(os.Stderr, "Usage error: %s.\n\n", violation)
70 }
71
72 fmt.Fprintf(os.Stderr,
Colin Crosse16ce362020-11-12 08:29:30 -080073 "Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
Jeff Gaston93f0f372017-11-01 13:33:02 -070074
75 flag.PrintDefaults()
76
77 os.Exit(1)
78}
79
Jeff Gastonefc1b412017-03-29 17:29:06 -070080func main() {
Jeff Gaston93f0f372017-11-01 13:33:02 -070081 flag.Usage = func() {
82 usageViolation("")
83 }
84 flag.Parse()
85
Jeff Gastonefc1b412017-03-29 17:29:06 -070086 error := run()
87 if error != nil {
88 fmt.Fprintln(os.Stderr, error)
89 os.Exit(1)
90 }
91}
92
Jeff Gaston90cfb092017-09-26 16:46:10 -070093func findAllFilesUnder(root string) (paths []string) {
94 paths = []string{}
95 filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
96 if !info.IsDir() {
97 relPath, err := filepath.Rel(root, path)
98 if err != nil {
99 // couldn't find relative path from ancestor?
100 panic(err)
101 }
102 paths = append(paths, relPath)
103 }
104 return nil
105 })
106 return paths
107}
108
Jeff Gastonefc1b412017-03-29 17:29:06 -0700109func run() error {
Colin Crosse16ce362020-11-12 08:29:30 -0800110 if manifestFile == "" {
111 usageViolation("--manifest <manifest> is required and must be non-empty")
Jeff Gastonefc1b412017-03-29 17:29:06 -0700112 }
Jeff Gaston02a684b2017-10-27 14:59:27 -0700113 if sandboxesRoot == "" {
Jeff Gastonefc1b412017-03-29 17:29:06 -0700114 // In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
115 // and the sbox executable will most likely be at a fixed location relative to OUT_DIR too, so
116 // the value of sandboxesRoot will most likely be at a fixed location relative to the sbox executable
117 // However, Soong also needs to be able to separately remove the sandbox directory on startup (if it has anything left in it)
118 // and by passing it as a parameter we don't need to duplicate its value
Jeff Gaston93f0f372017-11-01 13:33:02 -0700119 usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
Jeff Gastonefc1b412017-03-29 17:29:06 -0700120 }
Colin Crosse16ce362020-11-12 08:29:30 -0800121
122 manifest, err := readManifest(manifestFile)
Sam Delmerico285b66a2023-09-25 12:13:17 +0000123 if err != nil {
124 return err
125 }
Colin Crosse16ce362020-11-12 08:29:30 -0800126
127 if len(manifest.Commands) == 0 {
128 return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
Jeff Gaston193f2fb2017-06-12 15:00:12 -0700129 }
130
Colin Crosse16ce362020-11-12 08:29:30 -0800131 // setup sandbox directory
132 err = os.MkdirAll(sandboxesRoot, 0777)
Jeff Gaston8a88db52017-11-06 13:33:14 -0800133 if err != nil {
Colin Crosse16ce362020-11-12 08:29:30 -0800134 return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
Jeff Gaston8a88db52017-11-06 13:33:14 -0800135 }
Jeff Gastonefc1b412017-03-29 17:29:06 -0700136
Ulf Adamsb73daa52020-11-25 23:09:09 +0100137 // This tool assumes that there are no two concurrent runs with the same
138 // manifestFile. It should therefore be safe to use the hash of the
139 // manifestFile as the temporary directory name. We do this because it
140 // makes the temporary directory name deterministic. There are some
141 // tools that embed the name of the temporary output in the output, and
142 // they otherwise cause non-determinism, which then poisons actions
143 // depending on this one.
144 hash := sha1.New()
145 hash.Write([]byte(manifestFile))
146 tempDir := filepath.Join(sandboxesRoot, "sbox", hex.EncodeToString(hash.Sum(nil)))
147
148 err = os.RemoveAll(tempDir)
149 if err != nil {
150 return err
151 }
152 err = os.MkdirAll(tempDir, 0777)
Jeff Gastonefc1b412017-03-29 17:29:06 -0700153 if err != nil {
Colin Crosse16ce362020-11-12 08:29:30 -0800154 return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
Jeff Gastonefc1b412017-03-29 17:29:06 -0700155 }
156
157 // In the common case, the following line of code is what removes the sandbox
158 // If a fatal error occurs (such as if our Go process is killed unexpectedly),
Colin Crosse16ce362020-11-12 08:29:30 -0800159 // then at the beginning of the next build, Soong will wipe the temporary
160 // directory.
Jeff Gastonf49082a2017-06-07 13:22:22 -0700161 defer func() {
162 // in some cases we decline to remove the temp dir, to facilitate debugging
Jeff Gaston93f0f372017-11-01 13:33:02 -0700163 if !keepOutDir {
Jeff Gastonf49082a2017-06-07 13:22:22 -0700164 os.RemoveAll(tempDir)
165 }
166 }()
Jeff Gastonefc1b412017-03-29 17:29:06 -0700167
Colin Crosse16ce362020-11-12 08:29:30 -0800168 // If there is more than one command in the manifest use a separate directory for each one.
169 useSubDir := len(manifest.Commands) > 1
170 var commandDepFiles []string
Jeff Gastonefc1b412017-03-29 17:29:06 -0700171
Colin Crosse16ce362020-11-12 08:29:30 -0800172 for i, command := range manifest.Commands {
173 localTempDir := tempDir
174 if useSubDir {
175 localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
Jeff Gastonefc1b412017-03-29 17:29:06 -0700176 }
Kevin DuBois0a5da702021-10-05 15:41:02 -0700177 depFile, err := runCommand(command, localTempDir, i)
Jeff Gaston02a684b2017-10-27 14:59:27 -0700178 if err != nil {
Colin Crosse16ce362020-11-12 08:29:30 -0800179 // Running the command failed, keep the temporary output directory around in
180 // case a user wants to inspect it for debugging purposes. Soong will delete
181 // it at the beginning of the next build anyway.
182 keepOutDir = true
Jeff Gaston02a684b2017-10-27 14:59:27 -0700183 return err
184 }
Colin Crosse16ce362020-11-12 08:29:30 -0800185 if depFile != "" {
186 commandDepFiles = append(commandDepFiles, depFile)
187 }
188 }
189
190 outputDepFile := manifest.GetOutputDepfile()
191 if len(commandDepFiles) > 0 && outputDepFile == "" {
192 return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
193 depFilePlaceholder)
194 }
195
196 if outputDepFile != "" {
197 // Merge the depfiles from each command in the manifest to a single output depfile.
198 err = rewriteDepFiles(commandDepFiles, outputDepFile)
199 if err != nil {
200 return fmt.Errorf("failed merging depfiles: %w", err)
201 }
202 }
203
204 return nil
205}
206
Kevin DuBois0a5da702021-10-05 15:41:02 -0700207// createCommandScript will create and return an exec.Cmd that runs rawCommand.
208//
209// rawCommand is executed via a script in the sandbox.
Colin Cross40cade42022-04-04 18:16:04 -0700210// scriptPath is the temporary where the script is created.
211// scriptPathInSandbox is the path to the script in the sbox environment.
Kevin DuBois0a5da702021-10-05 15:41:02 -0700212//
213// returns an exec.Cmd that can be ran from within sbox context if no error, or nil if error.
214// caller must ensure script is cleaned up if function succeeds.
Colin Cross40cade42022-04-04 18:16:04 -0700215func createCommandScript(rawCommand, scriptPath, scriptPathInSandbox string) (*exec.Cmd, error) {
216 err := os.WriteFile(scriptPath, []byte(rawCommand), 0644)
Kevin DuBois0a5da702021-10-05 15:41:02 -0700217 if err != nil {
218 return nil, fmt.Errorf("failed to write command %s... to %s",
Colin Cross40cade42022-04-04 18:16:04 -0700219 rawCommand[0:40], scriptPath)
Kevin DuBois0a5da702021-10-05 15:41:02 -0700220 }
Colin Cross40cade42022-04-04 18:16:04 -0700221 return exec.Command("bash", scriptPathInSandbox), nil
Kevin DuBois0a5da702021-10-05 15:41:02 -0700222}
223
Colin Crosse16ce362020-11-12 08:29:30 -0800224// readManifest reads an sbox manifest from a textproto file.
225func readManifest(file string) (*sbox_proto.Manifest, error) {
226 manifestData, err := ioutil.ReadFile(file)
227 if err != nil {
228 return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
229 }
230
231 manifest := sbox_proto.Manifest{}
232
Dan Willemsen4591b642021-05-24 14:24:12 -0700233 err = prototext.Unmarshal(manifestData, &manifest)
Colin Crosse16ce362020-11-12 08:29:30 -0800234 if err != nil {
235 return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
236 }
237
238 return &manifest, nil
239}
240
241// runCommand runs a single command from a manifest. If the command references the
242// __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
Kevin DuBois0a5da702021-10-05 15:41:02 -0700243func runCommand(command *sbox_proto.Command, tempDir string, commandIndex int) (depFile string, err error) {
Colin Crosse16ce362020-11-12 08:29:30 -0800244 rawCommand := command.GetCommand()
245 if rawCommand == "" {
246 return "", fmt.Errorf("command is required")
247 }
248
Colin Crosse52c2ac2022-03-28 17:03:35 -0700249 // Remove files from the output directory
250 err = clearOutputDirectory(command.CopyAfter, outputDir, writeType(writeIfChanged))
251 if err != nil {
252 return "", err
253 }
254
Colin Crosse55bd422021-03-23 13:44:30 -0700255 pathToTempDirInSbox := tempDir
256 if command.GetChdir() {
257 pathToTempDirInSbox = "."
258 }
259
Colin Crosse16ce362020-11-12 08:29:30 -0800260 err = os.MkdirAll(tempDir, 0777)
261 if err != nil {
262 return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
263 }
264
265 // Copy in any files specified by the manifest.
Colin Crosse52c2ac2022-03-28 17:03:35 -0700266 err = copyFiles(command.CopyBefore, "", tempDir, requireFromExists, alwaysWrite)
Colin Crosse16ce362020-11-12 08:29:30 -0800267 if err != nil {
268 return "", err
269 }
Colin Crosse55bd422021-03-23 13:44:30 -0700270 err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
271 if err != nil {
272 return "", err
Colin Crossc590ec42021-03-11 17:20:02 -0800273 }
274
Colin Crosse16ce362020-11-12 08:29:30 -0800275 if strings.Contains(rawCommand, depFilePlaceholder) {
Colin Crossc590ec42021-03-11 17:20:02 -0800276 depFile = filepath.Join(pathToTempDirInSbox, "deps.d")
Colin Crosse16ce362020-11-12 08:29:30 -0800277 rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
278 }
279
280 if strings.Contains(rawCommand, sandboxDirPlaceholder) {
Colin Crossc590ec42021-03-11 17:20:02 -0800281 rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, pathToTempDirInSbox, -1)
Colin Crosse16ce362020-11-12 08:29:30 -0800282 }
283
284 // Emulate ninja's behavior of creating the directories for any output files before
285 // running the command.
286 err = makeOutputDirs(command.CopyAfter, tempDir)
287 if err != nil {
288 return "", err
Jeff Gastonefc1b412017-03-29 17:29:06 -0700289 }
290
Colin Cross40cade42022-04-04 18:16:04 -0700291 scriptName := fmt.Sprintf("sbox_command.%d.bash", commandIndex)
292 scriptPath := joinPath(tempDir, scriptName)
293 scriptPathInSandbox := joinPath(pathToTempDirInSbox, scriptName)
294 cmd, err := createCommandScript(rawCommand, scriptPath, scriptPathInSandbox)
Kevin DuBois0a5da702021-10-05 15:41:02 -0700295 if err != nil {
296 return "", err
297 }
298
Colin Cross4258a392021-04-15 18:50:11 -0700299 buf := &bytes.Buffer{}
Jeff Gastonefc1b412017-03-29 17:29:06 -0700300 cmd.Stdin = os.Stdin
Colin Cross4258a392021-04-15 18:50:11 -0700301 cmd.Stdout = buf
302 cmd.Stderr = buf
Colin Crosse16ce362020-11-12 08:29:30 -0800303
304 if command.GetChdir() {
305 cmd.Dir = tempDir
Colin Crossc590ec42021-03-11 17:20:02 -0800306 path := os.Getenv("PATH")
307 absPath, err := makeAbsPathEnv(path)
308 if err != nil {
309 return "", err
310 }
311 err = os.Setenv("PATH", absPath)
312 if err != nil {
313 return "", fmt.Errorf("Failed to update PATH: %w", err)
314 }
Colin Crosse16ce362020-11-12 08:29:30 -0800315 }
Jeff Gastonefc1b412017-03-29 17:29:06 -0700316 err = cmd.Run()
Jeff Gaston193f2fb2017-06-12 15:00:12 -0700317
Colin Crossfc2d8422021-04-15 18:36:42 -0700318 if err != nil {
319 // The command failed, do a best effort copy of output files out of the sandbox. This is
320 // especially useful for linters with baselines that print an error message on failure
321 // with a command to copy the output lint errors to the new baseline. Use a copy instead of
322 // a move to leave the sandbox intact for manual inspection
Colin Crosse52c2ac2022-03-28 17:03:35 -0700323 copyFiles(command.CopyAfter, tempDir, "", allowFromNotExists, writeType(writeIfChanged))
Colin Crossfc2d8422021-04-15 18:36:42 -0700324 }
325
Colin Cross4258a392021-04-15 18:50:11 -0700326 // If the command was executed but failed with an error, print a debugging message before
327 // the command's output so it doesn't scroll the real error message off the screen.
Jeff Gastonefc1b412017-03-29 17:29:06 -0700328 if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
Colin Cross4258a392021-04-15 18:50:11 -0700329 fmt.Fprintf(os.Stderr,
330 "The failing command was run inside an sbox sandbox in temporary directory\n"+
331 "%s\n"+
Colin Cross40cade42022-04-04 18:16:04 -0700332 "The failing command line can be found in\n"+
Colin Cross4258a392021-04-15 18:50:11 -0700333 "%s\n",
Colin Cross40cade42022-04-04 18:16:04 -0700334 tempDir, scriptPath)
Colin Cross4258a392021-04-15 18:50:11 -0700335 }
336
337 // Write the command's combined stdout/stderr.
338 os.Stdout.Write(buf.Bytes())
339
340 if err != nil {
Colin Crosse16ce362020-11-12 08:29:30 -0800341 return "", err
Jeff Gastonefc1b412017-03-29 17:29:06 -0700342 }
343
Colin Crosse52c2ac2022-03-28 17:03:35 -0700344 err = validateOutputFiles(command.CopyAfter, tempDir, outputDir, rawCommand)
345 if err != nil {
346 return "", err
Jeff Gastonf49082a2017-06-07 13:22:22 -0700347 }
Colin Crosse52c2ac2022-03-28 17:03:35 -0700348
Jeff Gastonf49082a2017-06-07 13:22:22 -0700349 // the created files match the declared files; now move them
Colin Crosse52c2ac2022-03-28 17:03:35 -0700350 err = moveFiles(command.CopyAfter, tempDir, "", writeType(writeIfChanged))
351 if err != nil {
352 return "", err
353 }
Colin Crosse16ce362020-11-12 08:29:30 -0800354
355 return depFile, nil
356}
357
358// makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
359// out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
360// so that the tools don't have to.
361func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
362 for _, copyPair := range copies {
363 dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
364 err := os.MkdirAll(dir, 0777)
365 if err != nil {
366 return err
Jeff Gaston193f2fb2017-06-12 15:00:12 -0700367 }
Colin Crosse16ce362020-11-12 08:29:30 -0800368 }
369 return nil
370}
371
372// validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
373// were created by the command.
Colin Crosse52c2ac2022-03-28 17:03:35 -0700374func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir, outputDir, rawCommand string) error {
Colin Crosse16ce362020-11-12 08:29:30 -0800375 var missingOutputErrors []error
Colin Crosse52c2ac2022-03-28 17:03:35 -0700376 var incorrectOutputDirectoryErrors []error
Colin Crosse16ce362020-11-12 08:29:30 -0800377 for _, copyPair := range copies {
378 fromPath := joinPath(sandboxDir, copyPair.GetFrom())
379 fileInfo, err := os.Stat(fromPath)
380 if err != nil {
381 missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
382 continue
383 }
384 if fileInfo.IsDir() {
385 missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
386 }
Colin Crosse52c2ac2022-03-28 17:03:35 -0700387
388 toPath := copyPair.GetTo()
389 if rel, err := filepath.Rel(outputDir, toPath); err != nil {
390 return err
391 } else if strings.HasPrefix(rel, "../") {
392 incorrectOutputDirectoryErrors = append(incorrectOutputDirectoryErrors,
393 fmt.Errorf("%s is not under %s", toPath, outputDir))
394 }
Colin Crosse16ce362020-11-12 08:29:30 -0800395 }
Colin Crosse52c2ac2022-03-28 17:03:35 -0700396
Steven Morelandee975ee2023-03-28 22:39:00 +0000397 const maxErrors = 25
Colin Crosse52c2ac2022-03-28 17:03:35 -0700398
399 if len(incorrectOutputDirectoryErrors) > 0 {
400 errorMessage := ""
401 more := 0
402 if len(incorrectOutputDirectoryErrors) > maxErrors {
403 more = len(incorrectOutputDirectoryErrors) - maxErrors
404 incorrectOutputDirectoryErrors = incorrectOutputDirectoryErrors[:maxErrors]
405 }
406
407 for _, err := range incorrectOutputDirectoryErrors {
408 errorMessage += err.Error() + "\n"
409 }
410 if more > 0 {
411 errorMessage += fmt.Sprintf("...%v more", more)
412 }
413
414 return errors.New(errorMessage)
415 }
416
417 if len(missingOutputErrors) > 0 {
418 // find all created files for making a more informative error message
419 createdFiles := findAllFilesUnder(sandboxDir)
420
421 // build error message
422 errorMessage := "mismatch between declared and actual outputs\n"
423 errorMessage += "in sbox command(" + rawCommand + ")\n\n"
424 errorMessage += "in sandbox " + sandboxDir + ",\n"
425 errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
426 for _, missingOutputError := range missingOutputErrors {
427 errorMessage += " " + missingOutputError.Error() + "\n"
428 }
429 if len(createdFiles) < 1 {
430 errorMessage += "created 0 files."
431 } else {
432 errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
433 creationMessages := createdFiles
434 if len(creationMessages) > maxErrors {
435 creationMessages = creationMessages[:maxErrors]
436 creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxErrors))
437 }
438 for _, creationMessage := range creationMessages {
439 errorMessage += " " + creationMessage + "\n"
440 }
441 }
442
443 return errors.New(errorMessage)
444 }
445
446 return nil
Colin Crosse16ce362020-11-12 08:29:30 -0800447}
448
Colin Crosse52c2ac2022-03-28 17:03:35 -0700449type existsType bool
450
451const (
452 requireFromExists existsType = false
453 allowFromNotExists = true
454)
455
456type writeType bool
457
458const (
459 alwaysWrite writeType = false
460 onlyWriteIfChanged = true
461)
462
463// copyFiles copies files in or out of the sandbox. If exists is allowFromNotExists then errors
464// caused by a from path not existing are ignored. If write is onlyWriteIfChanged then the output
465// file is compared to the input file and not written to if it is the same, avoiding updating
466// the timestamp.
467func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string, exists existsType, write writeType) error {
Colin Crosse16ce362020-11-12 08:29:30 -0800468 for _, copyPair := range copies {
469 fromPath := joinPath(fromDir, copyPair.GetFrom())
470 toPath := joinPath(toDir, copyPair.GetTo())
Colin Crosse52c2ac2022-03-28 17:03:35 -0700471 err := copyOneFile(fromPath, toPath, copyPair.GetExecutable(), exists, write)
Colin Crosse16ce362020-11-12 08:29:30 -0800472 if err != nil {
473 return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
474 }
475 }
476 return nil
477}
478
Colin Crossfc2d8422021-04-15 18:36:42 -0700479// copyOneFile copies a file and its permissions. If forceExecutable is true it adds u+x to the
Colin Crosse52c2ac2022-03-28 17:03:35 -0700480// permissions. If exists is allowFromNotExists it returns nil if the from path doesn't exist.
481// If write is onlyWriteIfChanged then the output file is compared to the input file and not written to
Cole Faustca355af2023-12-14 23:38:38 +0000482// if it is the same, avoiding updating the timestamp. If from is a symlink, the symlink itself
483// will be copied, instead of what it points to.
Colin Crosse52c2ac2022-03-28 17:03:35 -0700484func copyOneFile(from string, to string, forceExecutable bool, exists existsType,
485 write writeType) error {
Colin Crosse16ce362020-11-12 08:29:30 -0800486 err := os.MkdirAll(filepath.Dir(to), 0777)
487 if err != nil {
488 return err
489 }
490
Cole Faustca355af2023-12-14 23:38:38 +0000491 stat, err := os.Lstat(from)
Colin Crosse16ce362020-11-12 08:29:30 -0800492 if err != nil {
Colin Crosse52c2ac2022-03-28 17:03:35 -0700493 if os.IsNotExist(err) && exists == allowFromNotExists {
Colin Crossfc2d8422021-04-15 18:36:42 -0700494 return nil
495 }
Colin Crosse16ce362020-11-12 08:29:30 -0800496 return err
497 }
498
Cole Faustca355af2023-12-14 23:38:38 +0000499 if stat.Mode()&fs.ModeSymlink != 0 {
500 linkTarget, err := os.Readlink(from)
501 if err != nil {
502 return err
503 }
504 if write == onlyWriteIfChanged {
505 toLinkTarget, err := os.Readlink(to)
506 if err == nil && toLinkTarget == linkTarget {
507 return nil
508 }
509 }
510 err = os.Remove(to)
511 if err != nil && !os.IsNotExist(err) {
512 return err
513 }
514
515 return os.Symlink(linkTarget, to)
516 }
517
Colin Crosse16ce362020-11-12 08:29:30 -0800518 perm := stat.Mode()
Colin Crossfc2d8422021-04-15 18:36:42 -0700519 if forceExecutable {
Colin Cross859dfd92020-11-30 20:12:47 -0800520 perm = perm | 0100 // u+x
521 }
Colin Crosse16ce362020-11-12 08:29:30 -0800522
Colin Crosse52c2ac2022-03-28 17:03:35 -0700523 if write == onlyWriteIfChanged && filesHaveSameContents(from, to) {
524 return nil
525 }
526
Colin Crosse16ce362020-11-12 08:29:30 -0800527 in, err := os.Open(from)
528 if err != nil {
529 return err
530 }
531 defer in.Close()
532
Colin Cross607c0b72021-03-31 12:54:06 -0700533 // Remove the target before copying. In most cases the file won't exist, but if there are
534 // duplicate copy rules for a file and the source file was read-only the second copy could
535 // fail.
536 err = os.Remove(to)
537 if err != nil && !os.IsNotExist(err) {
538 return err
539 }
540
Colin Crosse16ce362020-11-12 08:29:30 -0800541 out, err := os.Create(to)
542 if err != nil {
543 return err
544 }
545 defer func() {
546 out.Close()
547 if err != nil {
548 os.Remove(to)
549 }
550 }()
551
552 _, err = io.Copy(out, in)
553 if err != nil {
554 return err
555 }
556
557 if err = out.Close(); err != nil {
558 return err
559 }
560
561 if err = os.Chmod(to, perm); err != nil {
562 return err
563 }
564
565 return nil
566}
567
Colin Crosse55bd422021-03-23 13:44:30 -0700568// copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
569// listed into the sandbox.
570func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
571 for _, rspFile := range rspFiles {
572 err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
573 if err != nil {
574 return err
575 }
576 }
577 return nil
578}
579
580// copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
581// listed into the sandbox.
582func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
583 in, err := os.Open(rspFile.GetFile())
584 if err != nil {
585 return err
586 }
587 defer in.Close()
588
589 files, err := response.ReadRspFile(in)
590 if err != nil {
591 return err
592 }
593
594 for i, from := range files {
595 // Convert the real path of the input file into the path inside the sandbox using the
596 // path mappings.
597 to := applyPathMappings(rspFile.PathMappings, from)
598
599 // Copy the file into the sandbox.
Colin Crosse52c2ac2022-03-28 17:03:35 -0700600 err := copyOneFile(from, joinPath(toDir, to), false, requireFromExists, alwaysWrite)
Colin Crosse55bd422021-03-23 13:44:30 -0700601 if err != nil {
602 return err
603 }
604
605 // Rewrite the name in the list of files to be relative to the sandbox directory.
606 files[i] = joinPath(toDirInSandbox, to)
607 }
608
609 // Convert the real path of the rsp file into the path inside the sandbox using the path
610 // mappings.
611 outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
612
613 err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
614 if err != nil {
615 return err
616 }
617
618 out, err := os.Create(outRspFile)
619 if err != nil {
620 return err
621 }
622 defer out.Close()
623
624 // Write the rsp file with converted paths into the sandbox.
625 err = response.WriteRspFile(out, files)
626 if err != nil {
627 return err
628 }
629
630 return nil
631}
632
633// applyPathMappings takes a list of path mappings and a path, and returns the path with the first
634// matching path mapping applied. If the path does not match any of the path mappings then it is
635// returned unmodified.
636func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
637 for _, mapping := range pathMappings {
638 if strings.HasPrefix(path, mapping.GetFrom()+"/") {
639 return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
640 }
641 }
642 return path
643}
644
Colin Crosse16ce362020-11-12 08:29:30 -0800645// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
646// to moving files where the source and destination are in the same filesystem. This is OK for
Colin Crosse52c2ac2022-03-28 17:03:35 -0700647// sbox because the temporary directory is inside the out directory. If write is onlyWriteIfChanged
648// then the output file is compared to the input file and not written to if it is the same, avoiding
649// updating the timestamp. Otherwise it always updates the timestamp of the new file.
650func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string, write writeType) error {
Colin Crosse16ce362020-11-12 08:29:30 -0800651 for _, copyPair := range copies {
652 fromPath := joinPath(fromDir, copyPair.GetFrom())
653 toPath := joinPath(toDir, copyPair.GetTo())
654 err := os.MkdirAll(filepath.Dir(toPath), 0777)
655 if err != nil {
656 return err
657 }
658
Colin Crosse52c2ac2022-03-28 17:03:35 -0700659 if write == onlyWriteIfChanged && filesHaveSameContents(fromPath, toPath) {
660 continue
661 }
662
Colin Crosse16ce362020-11-12 08:29:30 -0800663 err = os.Rename(fromPath, toPath)
Jeff Gaston8a88db52017-11-06 13:33:14 -0800664 if err != nil {
665 return err
666 }
Colin Crossd1c1e6f2019-03-29 13:54:39 -0700667
668 // Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
669 // files with old timestamps).
670 now := time.Now()
Colin Crosse16ce362020-11-12 08:29:30 -0800671 err = os.Chtimes(toPath, now, now)
Jeff Gastonefc1b412017-03-29 17:29:06 -0700672 if err != nil {
673 return err
674 }
675 }
Jeff Gastonefc1b412017-03-29 17:29:06 -0700676 return nil
677}
Colin Crosse16ce362020-11-12 08:29:30 -0800678
Colin Crosse52c2ac2022-03-28 17:03:35 -0700679// clearOutputDirectory removes all files in the output directory if write is alwaysWrite, or
680// any files not listed in copies if write is onlyWriteIfChanged
681func clearOutputDirectory(copies []*sbox_proto.Copy, outputDir string, write writeType) error {
682 if outputDir == "" {
683 return fmt.Errorf("output directory must be set")
684 }
685
686 if write == alwaysWrite {
687 // When writing all the output files remove the whole output directory
688 return os.RemoveAll(outputDir)
689 }
690
691 outputFiles := make(map[string]bool, len(copies))
692 for _, copyPair := range copies {
693 outputFiles[copyPair.GetTo()] = true
694 }
695
696 existingFiles := findAllFilesUnder(outputDir)
697 for _, existingFile := range existingFiles {
698 fullExistingFile := filepath.Join(outputDir, existingFile)
699 if !outputFiles[fullExistingFile] {
700 err := os.Remove(fullExistingFile)
701 if err != nil {
702 return fmt.Errorf("failed to remove obsolete output file %s: %w", fullExistingFile, err)
703 }
704 }
705 }
706
707 return nil
708}
709
Colin Crosse16ce362020-11-12 08:29:30 -0800710// Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
711// to an output file.
712func rewriteDepFiles(ins []string, out string) error {
713 var mergedDeps []string
714 for _, in := range ins {
715 data, err := ioutil.ReadFile(in)
716 if err != nil {
717 return err
718 }
719
720 deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
721 if err != nil {
722 return err
723 }
724 mergedDeps = append(mergedDeps, deps.Inputs...)
725 }
726
727 deps := makedeps.Deps{
728 // Ninja doesn't care what the output file is, so we can use any string here.
729 Output: "outputfile",
730 Inputs: mergedDeps,
731 }
732
733 // Make the directory for the output depfile in case it is in a different directory
734 // than any of the output files.
735 outDir := filepath.Dir(out)
736 err := os.MkdirAll(outDir, 0777)
737 if err != nil {
738 return fmt.Errorf("failed to create %q: %w", outDir, err)
739 }
740
741 return ioutil.WriteFile(out, deps.Print(), 0666)
742}
743
744// joinPath wraps filepath.Join but returns file without appending to dir if file is
745// absolute.
746func joinPath(dir, file string) string {
747 if filepath.IsAbs(file) {
748 return file
749 }
750 return filepath.Join(dir, file)
751}
Colin Crossc590ec42021-03-11 17:20:02 -0800752
Colin Crosse52c2ac2022-03-28 17:03:35 -0700753// filesHaveSameContents compares the contents if two files, returning true if they are the same
754// and returning false if they are different or any errors occur.
755func filesHaveSameContents(a, b string) bool {
756 // Compare the sizes of the two files
757 statA, err := os.Stat(a)
758 if err != nil {
759 return false
760 }
761 statB, err := os.Stat(b)
762 if err != nil {
763 return false
764 }
765
766 if statA.Size() != statB.Size() {
767 return false
768 }
769
770 // Open the two files
771 fileA, err := os.Open(a)
772 if err != nil {
773 return false
774 }
775 defer fileA.Close()
Colin Crossfa8e9cc2022-04-12 17:26:58 -0700776 fileB, err := os.Open(b)
Colin Crosse52c2ac2022-03-28 17:03:35 -0700777 if err != nil {
778 return false
779 }
780 defer fileB.Close()
781
782 // Compare the files 1MB at a time
783 const bufSize = 1 * 1024 * 1024
784 bufA := make([]byte, bufSize)
785 bufB := make([]byte, bufSize)
786
787 remain := statA.Size()
788 for remain > 0 {
789 toRead := int64(bufSize)
790 if toRead > remain {
791 toRead = remain
792 }
793
794 _, err = io.ReadFull(fileA, bufA[:toRead])
795 if err != nil {
796 return false
797 }
798 _, err = io.ReadFull(fileB, bufB[:toRead])
799 if err != nil {
800 return false
801 }
802
803 if bytes.Compare(bufA[:toRead], bufB[:toRead]) != 0 {
804 return false
805 }
806
807 remain -= toRead
808 }
809
810 return true
811}
812
Colin Crossc590ec42021-03-11 17:20:02 -0800813func makeAbsPathEnv(pathEnv string) (string, error) {
814 pathEnvElements := filepath.SplitList(pathEnv)
815 for i, p := range pathEnvElements {
816 if !filepath.IsAbs(p) {
817 absPath, err := filepath.Abs(p)
818 if err != nil {
819 return "", fmt.Errorf("failed to make PATH entry %q absolute: %w", p, err)
820 }
821 pathEnvElements[i] = absPath
822 }
823 }
824 return strings.Join(pathEnvElements, string(filepath.ListSeparator)), nil
825}