soong_zip: add --ignore_missing_files flag
soong_zip builds a list of files to zip early and then starts
zipping them all. If a directory being zipped is concurrently
modified, a file that existed when soong_zip started may not
still exist. Add a flag that continues when an expected file
does not exist. Print a warning, since this should be rare
in normal usages but is a sign of a problem if it happens
regularly.
Test: zip_test.go
Test: m checkbuild
Test: m platform
Change-Id: I78426fe66fded8528ddd436c0f71a7442183cfeb
diff --git a/zip/cmd/main.go b/zip/cmd/main.go
index 4a08491..b4f75f7 100644
--- a/zip/cmd/main.go
+++ b/zip/cmd/main.go
@@ -136,7 +136,7 @@
compLevel := flags.Int("L", 5, "deflate compression level (0-9)")
emulateJar := flags.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
-
+ ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
@@ -200,6 +200,7 @@
NonDeflatedFiles: nonDeflatedFiles,
WriteIfChanged: *writeIfChanged,
StoreSymlinks: *symlinks,
+ IgnoreMissingFiles: *ignoreMissingFiles,
})
if err != nil {
fmt.Fprintln(os.Stderr, "error:", err.Error())
diff --git a/zip/zip.go b/zip/zip.go
index d8507df..774966a 100644
--- a/zip/zip.go
+++ b/zip/zip.go
@@ -188,9 +188,11 @@
compressorPool sync.Pool
compLevel int
- followSymlinks pathtools.ShouldFollowSymlinks
+ followSymlinks pathtools.ShouldFollowSymlinks
+ ignoreMissingFiles bool
- fs pathtools.FileSystem
+ stderr io.Writer
+ fs pathtools.FileSystem
}
type zipEntry struct {
@@ -215,7 +217,9 @@
NonDeflatedFiles map[string]bool
WriteIfChanged bool
StoreSymlinks bool
+ IgnoreMissingFiles bool
+ Stderr io.Writer
Filesystem pathtools.FileSystem
}
@@ -271,19 +275,25 @@
followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
z := &ZipWriter{
- time: jar.DefaultTime,
- createdDirs: make(map[string]string),
- createdFiles: make(map[string]string),
- directories: args.AddDirectoryEntriesToZip,
- compLevel: args.CompressionLevel,
- followSymlinks: followSymlinks,
- fs: args.Filesystem,
+ time: jar.DefaultTime,
+ createdDirs: make(map[string]string),
+ createdFiles: make(map[string]string),
+ directories: args.AddDirectoryEntriesToZip,
+ compLevel: args.CompressionLevel,
+ followSymlinks: followSymlinks,
+ ignoreMissingFiles: args.IgnoreMissingFiles,
+ stderr: args.Stderr,
+ fs: args.Filesystem,
}
if z.fs == nil {
z.fs = pathtools.OsFs
}
+ if z.stderr == nil {
+ z.stderr = os.Stderr
+ }
+
pathMappings := []pathMapping{}
noCompression := args.CompressionLevel == 0
@@ -301,29 +311,44 @@
return err
}
if len(globbed) == 0 {
- return &os.PathError{
- Op: "stat",
+ err := &os.PathError{
+ Op: "lstat",
Path: s,
Err: os.ErrNotExist,
}
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
}
srcs = append(srcs, globbed...)
}
if fa.GlobDir != "" {
if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
return err
- } else if !exists {
- return &os.PathError{
- Op: "stat",
+ } else if !exists && !args.IgnoreMissingFiles {
+ err := &os.PathError{
+ Op: "lstat",
Path: fa.GlobDir,
Err: os.ErrNotExist,
}
- } else if !isDir {
- return &os.PathError{
- Op: "stat",
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
+ } else if !isDir && !args.IgnoreMissingFiles {
+ err := &os.PathError{
+ Op: "lstat",
Path: fa.GlobDir,
Err: syscall.ENOTDIR,
}
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
}
globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
if err != nil {
@@ -576,6 +601,10 @@
}
if err != nil {
+ if os.IsNotExist(err) && z.ignoreMissingFiles {
+ fmt.Fprintln(z.stderr, "warning:", err)
+ return nil
+ }
return err
} else if s.IsDir() {
if z.directories {
diff --git a/zip/zip_test.go b/zip/zip_test.go
index a08fb12..93c5f3d 100644
--- a/zip/zip_test.go
+++ b/zip/zip_test.go
@@ -98,14 +98,15 @@
func TestZip(t *testing.T) {
testCases := []struct {
- name string
- args *FileArgsBuilder
- compressionLevel int
- emulateJar bool
- nonDeflatedFiles map[string]bool
- dirEntries bool
- manifest string
- storeSymlinks bool
+ name string
+ args *FileArgsBuilder
+ compressionLevel int
+ emulateJar bool
+ nonDeflatedFiles map[string]bool
+ dirEntries bool
+ manifest string
+ storeSymlinks bool
+ ignoreMissingFiles bool
files []zip.FileHeader
err error
@@ -338,6 +339,20 @@
fh("a/a/b", fileB, zip.Deflate),
},
},
+ {
+ name: "ignore missing files",
+ args: fileArgsBuilder().
+ File("a/a/a").
+ File("a/a/b").
+ File("missing"),
+ compressionLevel: 9,
+ ignoreMissingFiles: true,
+
+ files: []zip.FileHeader{
+ fh("a/a/a", fileA, zip.Deflate),
+ fh("a/a/b", fileB, zip.Deflate),
+ },
+ },
// errors
{
@@ -381,7 +396,9 @@
args.NonDeflatedFiles = test.nonDeflatedFiles
args.ManifestSourcePath = test.manifest
args.StoreSymlinks = test.storeSymlinks
+ args.IgnoreMissingFiles = test.ignoreMissingFiles
args.Filesystem = mockFs
+ args.Stderr = &bytes.Buffer{}
buf := &bytes.Buffer{}
err := ZipTo(args, buf)