zip2zip: Support sorting globbed arguments, '**'
When '-s' is passed, any globbed arguments will have their results
sorted. When there are multiple arguments, the files will still be
inserted in argument order.
A bare '**' is now special cased to mean every file in the input zip.
This allows zip2zip to sort entire zip files efficiently by using
`zip2zip -s -i <> -o <> '**'`. This can be useful if your original zip
program used filesystem ordering which was not reproducible.
Test: m -j blueprint_tools (new tests pass)
Change-Id: Ic3512c5fe14c94c6f3e134296905121d2ff8b58a
diff --git a/cmd/zip2zip/Android.bp b/cmd/zip2zip/Android.bp
index 8cac003..476be4f 100644
--- a/cmd/zip2zip/Android.bp
+++ b/cmd/zip2zip/Android.bp
@@ -18,5 +18,6 @@
srcs: [
"zip2zip.go",
],
+ testSrcs: ["zip2zip_test.go"],
}
diff --git a/cmd/zip2zip/zip2zip.go b/cmd/zip2zip/zip2zip.go
index 8e7523f..48c36cc 100644
--- a/cmd/zip2zip/zip2zip.go
+++ b/cmd/zip2zip/zip2zip.go
@@ -17,49 +17,59 @@
import (
"flag"
"fmt"
+ "log"
"os"
"path/filepath"
+ "sort"
"strings"
+ "time"
"android/soong/third_party/zip"
)
var (
- input = flag.String("i", "", "zip file to read from")
- output = flag.String("o", "", "output file")
+ input = flag.String("i", "", "zip file to read from")
+ output = flag.String("o", "", "output file")
+ sortGlobs = flag.Bool("s", false, "sort matches from each glob (defaults to the order from the input zip file)")
+ setTime = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00")
+
+ staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
)
-func usage() {
- fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [filespec]...")
- flag.PrintDefaults()
- fmt.Fprintln(os.Stderr, " filespec:")
- fmt.Fprintln(os.Stderr, " <name>")
- fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
- fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
- fmt.Fprintln(os.Stderr, "")
- fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
- fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
- os.Exit(2)
-}
-
func main() {
+ flag.Usage = func() {
+ fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s] [-t] [filespec]...")
+ flag.PrintDefaults()
+ fmt.Fprintln(os.Stderr, " filespec:")
+ fmt.Fprintln(os.Stderr, " <name>")
+ fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
+ fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
+ fmt.Fprintln(os.Stderr, "")
+ fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://golang.org/pkg/path/filepath/#Match")
+ fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip")
+ fmt.Fprintln(os.Stderr, "")
+ fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
+ fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
+ }
+
flag.Parse()
if flag.NArg() == 0 || *input == "" || *output == "" {
- usage()
+ flag.Usage()
+ os.Exit(1)
}
+ log.SetFlags(log.Lshortfile)
+
reader, err := zip.OpenReader(*input)
if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(3)
+ log.Fatal(err)
}
defer reader.Close()
output, err := os.Create(*output)
if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(4)
+ log.Fatal(err)
}
defer output.Close()
@@ -67,20 +77,24 @@
defer func() {
err := writer.Close()
if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(5)
+ log.Fatal(err)
}
}()
- for _, arg := range flag.Args() {
+ if err := zip2zip(&reader.Reader, writer, *sortGlobs, *setTime, flag.Args()); err != nil {
+ log.Fatal(err)
+ }
+}
+
+func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, setTime bool, args []string) error {
+ for _, arg := range args {
var input string
var output string
// Reserve escaping for future implementation, so make sure no
// one is using \ and expecting a certain behavior.
if strings.Contains(arg, "\\") {
- fmt.Fprintln(os.Stderr, "\\ characters are not currently supported")
- os.Exit(6)
+ return fmt.Errorf("\\ characters are not currently supported")
}
args := strings.SplitN(arg, ":", 2)
@@ -89,25 +103,45 @@
output = args[1]
}
+ type pair struct {
+ *zip.File
+ newName string
+ }
+
+ matches := []pair{}
if strings.IndexAny(input, "*?[") >= 0 {
+ matchAll := input == "**"
+ if !matchAll && strings.Contains(input, "**") {
+ return fmt.Errorf("** is only supported on its own, not with other characters")
+ }
+
for _, file := range reader.File {
- if match, err := filepath.Match(input, file.Name); err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(7)
- } else if match {
- var newFileName string
- if output == "" {
- newFileName = file.Name
- } else {
- _, name := filepath.Split(file.Name)
- newFileName = filepath.Join(output, name)
- }
- err = writer.CopyFrom(file, newFileName)
+ match := matchAll
+
+ if !match {
+ var err error
+ match, err = filepath.Match(input, file.Name)
if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(8)
+ return err
}
}
+
+ if match {
+ var newName string
+ if output == "" {
+ newName = file.Name
+ } else {
+ _, name := filepath.Split(file.Name)
+ newName = filepath.Join(output, name)
+ }
+ matches = append(matches, pair{file, newName})
+ }
+ }
+
+ if sortGlobs {
+ sort.SliceStable(matches, func(i, j int) bool {
+ return matches[i].newName < matches[j].newName
+ })
}
} else {
if output == "" {
@@ -115,14 +149,21 @@
}
for _, file := range reader.File {
if input == file.Name {
- err = writer.CopyFrom(file, output)
- if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(8)
- }
+ matches = append(matches, pair{file, output})
break
}
}
}
+
+ for _, match := range matches {
+ if setTime {
+ match.File.SetModTime(staticTime)
+ }
+ if err := writer.CopyFrom(match.File, match.newName); err != nil {
+ return err
+ }
+ }
}
+
+ return nil
}
diff --git a/cmd/zip2zip/zip2zip_test.go b/cmd/zip2zip/zip2zip_test.go
new file mode 100644
index 0000000..7f2e31a
--- /dev/null
+++ b/cmd/zip2zip/zip2zip_test.go
@@ -0,0 +1,188 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "reflect"
+ "testing"
+
+ "android/soong/third_party/zip"
+)
+
+var testCases = []struct {
+ name string
+
+ inputFiles []string
+ sortGlobs bool
+ args []string
+
+ outputFiles []string
+ err error
+}{
+ {
+ name: "unsupported \\",
+
+ args: []string{"a\\b:b"},
+
+ err: fmt.Errorf("\\ characters are not currently supported"),
+ },
+ {
+ name: "unsupported **",
+
+ args: []string{"a/**:b"},
+
+ err: fmt.Errorf("** is only supported on its own, not with other characters"),
+ },
+ { // This is modelled after the update package build rules in build/make/core/Makefile
+ name: "filter globs",
+
+ inputFiles: []string{
+ "RADIO/a",
+ "IMAGES/system.img",
+ "IMAGES/b.txt",
+ "IMAGES/recovery.img",
+ "IMAGES/vendor.img",
+ "OTA/android-info.txt",
+ "OTA/b",
+ },
+ args: []string{"OTA/android-info.txt:android-info.txt", "IMAGES/*.img:."},
+
+ outputFiles: []string{
+ "android-info.txt",
+ "system.img",
+ "recovery.img",
+ "vendor.img",
+ },
+ },
+ {
+ name: "sorted filter globs",
+
+ inputFiles: []string{
+ "RADIO/a",
+ "IMAGES/system.img",
+ "IMAGES/b.txt",
+ "IMAGES/recovery.img",
+ "IMAGES/vendor.img",
+ "OTA/android-info.txt",
+ "OTA/b",
+ },
+ sortGlobs: true,
+ args: []string{"IMAGES/*.img:.", "OTA/android-info.txt:android-info.txt"},
+
+ outputFiles: []string{
+ "recovery.img",
+ "system.img",
+ "vendor.img",
+ "android-info.txt",
+ },
+ },
+ {
+ name: "sort all",
+
+ inputFiles: []string{
+ "RADIO/a",
+ "IMAGES/system.img",
+ "IMAGES/b.txt",
+ "IMAGES/recovery.img",
+ "IMAGES/vendor.img",
+ "OTA/b",
+ "OTA/android-info.txt",
+ },
+ sortGlobs: true,
+ args: []string{"**"},
+
+ outputFiles: []string{
+ "IMAGES/b.txt",
+ "IMAGES/recovery.img",
+ "IMAGES/system.img",
+ "IMAGES/vendor.img",
+ "OTA/android-info.txt",
+ "OTA/b",
+ "RADIO/a",
+ },
+ },
+ {
+ name: "double input",
+
+ inputFiles: []string{
+ "b",
+ "a",
+ },
+ args: []string{"a:a2", "**"},
+
+ outputFiles: []string{
+ "a2",
+ "b",
+ "a",
+ },
+ },
+}
+
+func errorString(e error) string {
+ if e == nil {
+ return ""
+ }
+ return e.Error()
+}
+
+func TestZip2Zip(t *testing.T) {
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ inputBuf := &bytes.Buffer{}
+ outputBuf := &bytes.Buffer{}
+
+ inputWriter := zip.NewWriter(inputBuf)
+ for _, file := range testCase.inputFiles {
+ w, err := inputWriter.Create(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fmt.Fprintln(w, "test")
+ }
+ inputWriter.Close()
+ inputBytes := inputBuf.Bytes()
+ inputReader, err := zip.NewReader(bytes.NewReader(inputBytes), int64(len(inputBytes)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ outputWriter := zip.NewWriter(outputBuf)
+ err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, false, testCase.args)
+ if errorString(testCase.err) != errorString(err) {
+ t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err))
+ }
+
+ outputWriter.Close()
+ outputBytes := outputBuf.Bytes()
+ outputReader, err := zip.NewReader(bytes.NewReader(outputBytes), int64(len(outputBytes)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var outputFiles []string
+ if len(outputReader.File) > 0 {
+ outputFiles = make([]string, len(outputReader.File))
+ for i, file := range outputReader.File {
+ outputFiles[i] = file.Name
+ }
+ }
+
+ if !reflect.DeepEqual(testCase.outputFiles, outputFiles) {
+ t.Fatalf("Output file list does not match:\n got: %v\nwant: %v", outputFiles, testCase.outputFiles)
+ }
+ })
+ }
+}