Merge changes from topic "glob_escapes"
* changes:
Use file glob for zipping classes*.dex files
soong_zip: add --ignore_missing_files flag
Add a --symlinks argument to soong_zip
soong_zip: support globs in -f and -D arguments
diff --git a/java/app_builder.go b/java/app_builder.go
index 954ca44..e27b1b7 100644
--- a/java/app_builder.go
+++ b/java/app_builder.go
@@ -103,10 +103,10 @@
`cp ${manifest} ${outDir}/AndroidManifest.xml && ` +
`cp ${classesJar} ${outDir}/classes.jar && ` +
`cp ${rTxt} ${outDir}/R.txt && ` +
- `${config.SoongZipCmd} -jar -o $out -C ${outDir} -D ${outDir} ${resArgs}`,
+ `${config.SoongZipCmd} -jar -o $out -C ${outDir} -D ${outDir}`,
CommandDeps: []string{"${config.SoongZipCmd}"},
},
- "manifest", "classesJar", "rTxt", "resArgs", "outDir")
+ "manifest", "classesJar", "rTxt", "outDir")
func BuildAAR(ctx android.ModuleContext, outputFile android.WritablePath,
classesJar, manifest, rTxt android.Path, res android.Paths) {
diff --git a/java/builder.go b/java/builder.go
index 48b5a7b..07af8eb 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -321,7 +321,7 @@
Output: outputFile,
Implicits: deps,
Args: map[string]string{
- "jarArgs": strings.Join(proptools.NinjaEscape(jarArgs), " "),
+ "jarArgs": strings.Join(proptools.NinjaAndShellEscape(jarArgs), " "),
},
})
}
diff --git a/java/dex.go b/java/dex.go
index 03316f6..ce0c18e 100644
--- a/java/dex.go
+++ b/java/dex.go
@@ -26,7 +26,7 @@
blueprint.RuleParams{
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
`${config.D8Cmd} --output $outDir $d8Flags $in && ` +
- `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -D $outDir && ` +
+ `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
CommandDeps: []string{
"${config.D8Cmd}",
@@ -46,7 +46,7 @@
`-printmapping $outDict ` +
`$r8Flags && ` +
`touch "$outDict" && ` +
- `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -D $outDir && ` +
+ `${config.SoongZipCmd} -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
CommandDeps: []string{
"${config.R8Cmd}",
diff --git a/java/java_resources.go b/java/java_resources.go
index fdc1590..6c1fd39 100644
--- a/java/java_resources.go
+++ b/java/java_resources.go
@@ -19,6 +19,8 @@
"path/filepath"
"strings"
+ "github.com/google/blueprint/pathtools"
+
"android/soong/android"
)
@@ -64,7 +66,7 @@
if !strings.HasPrefix(path, dir.String()) {
panic(fmt.Errorf("path %q does not start with %q", path, dir))
}
- args = append(args, "-f", path)
+ args = append(args, "-f", pathtools.MatchEscape(path))
}
}
}
@@ -107,7 +109,7 @@
if i == 0 || dir != lastDir {
args = append(args, "-C", dir)
}
- args = append(args, "-f", path)
+ args = append(args, "-f", pathtools.MatchEscape(path))
lastDir = dir
}
diff --git a/zip/cmd/main.go b/zip/cmd/main.go
index 1125602..b4f75f7 100644
--- a/zip/cmd/main.go
+++ b/zip/cmd/main.go
@@ -136,6 +136,8 @@
compLevel := flags.Int("L", 5, "deflate compression level (0-9)")
emulateJar := flags.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
+ ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
+ symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file")
@@ -197,9 +199,11 @@
NumParallelJobs: *parallelJobs,
NonDeflatedFiles: nonDeflatedFiles,
WriteIfChanged: *writeIfChanged,
+ StoreSymlinks: *symlinks,
+ IgnoreMissingFiles: *ignoreMissingFiles,
})
if err != nil {
- fmt.Fprintln(os.Stderr, err.Error())
+ fmt.Fprintln(os.Stderr, "error:", err.Error())
os.Exit(1)
}
}
diff --git a/zip/zip.go b/zip/zip.go
index e7de6f8..774966a 100644
--- a/zip/zip.go
+++ b/zip/zip.go
@@ -27,6 +27,7 @@
"sort"
"strings"
"sync"
+ "syscall"
"time"
"unicode"
@@ -163,6 +164,15 @@
return b.fileArgs
}
+type IncorrectRelativeRootError struct {
+ RelativeRoot string
+ Path string
+}
+
+func (x IncorrectRelativeRootError) Error() string {
+ return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
+}
+
type ZipWriter struct {
time time.Time
createdFiles map[string]string
@@ -178,7 +188,11 @@
compressorPool sync.Pool
compLevel int
- fs pathtools.FileSystem
+ followSymlinks pathtools.ShouldFollowSymlinks
+ ignoreMissingFiles bool
+
+ stderr io.Writer
+ fs pathtools.FileSystem
}
type zipEntry struct {
@@ -202,7 +216,11 @@
NumParallelJobs int
NonDeflatedFiles map[string]bool
WriteIfChanged bool
- Filesystem pathtools.FileSystem
+ StoreSymlinks bool
+ IgnoreMissingFiles bool
+
+ Stderr io.Writer
+ Filesystem pathtools.FileSystem
}
const NOQUOTE = '\x00'
@@ -253,27 +271,90 @@
args.AddDirectoryEntriesToZip = true
}
+ // Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
+ followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
+
z := &ZipWriter{
- time: jar.DefaultTime,
- createdDirs: make(map[string]string),
- createdFiles: make(map[string]string),
- directories: args.AddDirectoryEntriesToZip,
- compLevel: args.CompressionLevel,
- fs: args.Filesystem,
+ time: jar.DefaultTime,
+ createdDirs: make(map[string]string),
+ createdFiles: make(map[string]string),
+ directories: args.AddDirectoryEntriesToZip,
+ compLevel: args.CompressionLevel,
+ followSymlinks: followSymlinks,
+ ignoreMissingFiles: args.IgnoreMissingFiles,
+ stderr: args.Stderr,
+ fs: args.Filesystem,
}
if z.fs == nil {
z.fs = pathtools.OsFs
}
+ if z.stderr == nil {
+ z.stderr = os.Stderr
+ }
+
pathMappings := []pathMapping{}
noCompression := args.CompressionLevel == 0
for _, fa := range args.FileArgs {
- srcs := fa.SourceFiles
+ var srcs []string
+ for _, s := range fa.SourceFiles {
+ s = strings.TrimSpace(s)
+ if s == "" {
+ continue
+ }
+
+ globbed, _, err := z.fs.Glob(s, nil, followSymlinks)
+ if err != nil {
+ return err
+ }
+ if len(globbed) == 0 {
+ err := &os.PathError{
+ Op: "lstat",
+ Path: s,
+ Err: os.ErrNotExist,
+ }
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
+ }
+ srcs = append(srcs, globbed...)
+ }
if fa.GlobDir != "" {
- srcs = append(srcs, recursiveGlobFiles(fa.GlobDir)...)
+ if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
+ return err
+ } else if !exists && !args.IgnoreMissingFiles {
+ err := &os.PathError{
+ Op: "lstat",
+ Path: fa.GlobDir,
+ Err: os.ErrNotExist,
+ }
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
+ } else if !isDir && !args.IgnoreMissingFiles {
+ err := &os.PathError{
+ Op: "lstat",
+ Path: fa.GlobDir,
+ Err: syscall.ENOTDIR,
+ }
+ if args.IgnoreMissingFiles {
+ fmt.Fprintln(args.Stderr, "warning:", err)
+ } else {
+ return err
+ }
+ }
+ globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
+ if err != nil {
+ return err
+ }
+ srcs = append(srcs, globbed...)
}
for _, src := range srcs {
err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
@@ -328,11 +409,6 @@
func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
nonDeflatedFiles map[string]bool, noCompression bool) error {
- src = strings.TrimSpace(src)
- if src == "" {
- return nil
- }
- src = filepath.Clean(src)
var dest string
if fa.JunkPaths {
@@ -343,6 +419,13 @@
if err != nil {
return err
}
+ if strings.HasPrefix(dest, "../") {
+ return IncorrectRelativeRootError{
+ Path: src,
+ RelativeRoot: fa.SourcePrefixToStrip,
+ }
+ }
+
}
dest = filepath.Join(fa.PathPrefixInZip, dest)
@@ -509,7 +592,19 @@
var fileSize int64
var executable bool
- if s, err := z.fs.Lstat(src); err != nil {
+ var s os.FileInfo
+ var err error
+ if z.followSymlinks {
+ s, err = z.fs.Stat(src)
+ } else {
+ s, err = z.fs.Lstat(src)
+ }
+
+ if err != nil {
+ if os.IsNotExist(err) && z.ignoreMissingFiles {
+ fmt.Fprintln(z.stderr, "warning:", err)
+ return nil
+ }
return err
} else if s.IsDir() {
if z.directories {
@@ -889,15 +984,3 @@
return nil
}
-
-func recursiveGlobFiles(path string) []string {
- var files []string
- filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
- if !info.IsDir() {
- files = append(files, path)
- }
- return nil
- })
-
- return files
-}
diff --git a/zip/zip_test.go b/zip/zip_test.go
index 0c2105c..93c5f3d 100644
--- a/zip/zip_test.go
+++ b/zip/zip_test.go
@@ -98,13 +98,15 @@
func TestZip(t *testing.T) {
testCases := []struct {
- name string
- args *FileArgsBuilder
- compressionLevel int
- emulateJar bool
- nonDeflatedFiles map[string]bool
- dirEntries bool
- manifest string
+ name string
+ args *FileArgsBuilder
+ compressionLevel int
+ emulateJar bool
+ nonDeflatedFiles map[string]bool
+ dirEntries bool
+ manifest string
+ storeSymlinks bool
+ ignoreMissingFiles bool
files []zip.FileHeader
err error
@@ -130,6 +132,36 @@
},
},
{
+ name: "files glob",
+ args: fileArgsBuilder().
+ SourcePrefixToStrip("a").
+ File("a/**/*"),
+ compressionLevel: 9,
+ storeSymlinks: true,
+
+ files: []zip.FileHeader{
+ fh("a/a", fileA, zip.Deflate),
+ fh("a/b", fileB, zip.Deflate),
+ fhLink("a/c", "../../c"),
+ fhLink("a/d", "b"),
+ },
+ },
+ {
+ name: "dir",
+ args: fileArgsBuilder().
+ SourcePrefixToStrip("a").
+ Dir("a"),
+ compressionLevel: 9,
+ storeSymlinks: true,
+
+ files: []zip.FileHeader{
+ fh("a/a", fileA, zip.Deflate),
+ fh("a/b", fileB, zip.Deflate),
+ fhLink("a/c", "../../c"),
+ fhLink("a/d", "b"),
+ },
+ },
+ {
name: "stored files",
args: fileArgsBuilder().
File("a/a/a").
@@ -151,6 +183,7 @@
File("a/a/c").
File("a/a/d"),
compressionLevel: 9,
+ storeSymlinks: true,
files: []zip.FileHeader{
fh("a/a/a", fileA, zip.Deflate),
@@ -160,6 +193,23 @@
},
},
{
+ name: "follow symlinks",
+ args: fileArgsBuilder().
+ File("a/a/a").
+ File("a/a/b").
+ File("a/a/c").
+ File("a/a/d"),
+ compressionLevel: 9,
+ storeSymlinks: false,
+
+ files: []zip.FileHeader{
+ fh("a/a/a", fileA, zip.Deflate),
+ fh("a/a/b", fileB, zip.Deflate),
+ fh("a/a/c", fileC, zip.Deflate),
+ fh("a/a/d", fileB, zip.Deflate),
+ },
+ },
+ {
name: "list",
args: fileArgsBuilder().
List("l"),
@@ -289,6 +339,20 @@
fh("a/a/b", fileB, zip.Deflate),
},
},
+ {
+ name: "ignore missing files",
+ args: fileArgsBuilder().
+ File("a/a/a").
+ File("a/a/b").
+ File("missing"),
+ compressionLevel: 9,
+ ignoreMissingFiles: true,
+
+ files: []zip.FileHeader{
+ fh("a/a/a", fileA, zip.Deflate),
+ fh("a/a/b", fileB, zip.Deflate),
+ },
+ },
// errors
{
@@ -298,11 +362,24 @@
err: os.ErrNotExist,
},
{
+ name: "error missing dir",
+ args: fileArgsBuilder().
+ Dir("missing"),
+ err: os.ErrNotExist,
+ },
+ {
name: "error missing file in list",
args: fileArgsBuilder().
List("l2"),
err: os.ErrNotExist,
},
+ {
+ name: "error incorrect relative root",
+ args: fileArgsBuilder().
+ SourcePrefixToStrip("b").
+ File("a/a/a"),
+ err: IncorrectRelativeRootError{},
+ },
}
for _, test := range testCases {
@@ -318,7 +395,10 @@
args.AddDirectoryEntriesToZip = test.dirEntries
args.NonDeflatedFiles = test.nonDeflatedFiles
args.ManifestSourcePath = test.manifest
+ args.StoreSymlinks = test.storeSymlinks
+ args.IgnoreMissingFiles = test.ignoreMissingFiles
args.Filesystem = mockFs
+ args.Stderr = &bytes.Buffer{}
buf := &bytes.Buffer{}
err := ZipTo(args, buf)
@@ -330,6 +410,10 @@
if !os.IsNotExist(test.err) {
t.Fatalf("want error %v, got %v", test.err, err)
}
+ } else if _, wantRelativeRootErr := test.err.(IncorrectRelativeRootError); wantRelativeRootErr {
+ if _, gotRelativeRootErr := err.(IncorrectRelativeRootError); !gotRelativeRootErr {
+ t.Fatalf("want error %v, got %v", test.err, err)
+ }
} else {
t.Fatalf("want error %v, got %v", test.err, err)
}