blob: a94392b64c99eb797a82608145e342eb9b2f39e4 [file] [log] [blame]
Jeff Gaston8bab5f22017-09-01 13:34:28 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
Colin Cross635acc92017-09-12 22:50:46 -070018 "errors"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070019 "flag"
20 "fmt"
Colin Cross635acc92017-09-12 22:50:46 -070021 "hash/crc32"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070022 "log"
23 "os"
Nan Zhang13f4cf52017-09-19 18:42:01 -070024 "path/filepath"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070025 "sort"
26 "strings"
27
28 "android/soong/jar"
29 "android/soong/third_party/zip"
30)
31
Nan Zhang13f4cf52017-09-19 18:42:01 -070032type stripDir struct{}
Nan Zhangd5998cc2017-09-13 13:17:43 -070033
Nan Zhang13f4cf52017-09-19 18:42:01 -070034func (s *stripDir) String() string {
Nan Zhangd5998cc2017-09-13 13:17:43 -070035 return `""`
36}
37
Nan Zhang13f4cf52017-09-19 18:42:01 -070038func (s *stripDir) Set(dir string) error {
39 stripDirs = append(stripDirs, filepath.Clean(dir))
40
41 return nil
42}
43
44type zipToNotStrip struct{}
45
46func (s *zipToNotStrip) String() string {
47 return `""`
48}
49
50func (s *zipToNotStrip) Set(zip_path string) error {
51 zipsToNotStrip[zip_path] = true
Nan Zhangd5998cc2017-09-13 13:17:43 -070052
53 return nil
54}
55
Jeff Gaston8bab5f22017-09-01 13:34:28 -070056var (
Colin Cross635acc92017-09-12 22:50:46 -070057 sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
58 emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
59 stripDirs []string
60 zipsToNotStrip = make(map[string]bool)
61 stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
62 manifest = flag.String("m", "", "manifest file to insert in jar")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070063)
64
Nan Zhangd5998cc2017-09-13 13:17:43 -070065func init() {
Nan Zhang13f4cf52017-09-19 18:42:01 -070066 flag.Var(&stripDir{}, "stripDir", "the prefix of file path to be excluded from the output zip")
67 flag.Var(&zipToNotStrip{}, "zipToNotStrip", "the input zip file which is not applicable for stripping")
Nan Zhangd5998cc2017-09-13 13:17:43 -070068}
69
Jeff Gaston8bab5f22017-09-01 13:34:28 -070070func main() {
71 flag.Usage = func() {
Colin Cross635acc92017-09-12 22:50:46 -070072 fmt.Fprintln(os.Stderr, "usage: merge_zips [-jsD] [-m manifest] output [inputs...]")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070073 flag.PrintDefaults()
74 }
75
76 // parse args
77 flag.Parse()
78 args := flag.Args()
79 if len(args) < 2 {
80 flag.Usage()
81 os.Exit(1)
82 }
83 outputPath := args[0]
84 inputs := args[1:]
85
86 log.SetFlags(log.Lshortfile)
87
88 // make writer
89 output, err := os.Create(outputPath)
90 if err != nil {
91 log.Fatal(err)
92 }
93 defer output.Close()
94 writer := zip.NewWriter(output)
95 defer func() {
96 err := writer.Close()
97 if err != nil {
98 log.Fatal(err)
99 }
100 }()
101
102 // make readers
103 readers := []namedZipReader{}
104 for _, input := range inputs {
105 reader, err := zip.OpenReader(input)
106 if err != nil {
107 log.Fatal(err)
108 }
109 defer reader.Close()
110 namedReader := namedZipReader{path: input, reader: reader}
111 readers = append(readers, namedReader)
112 }
113
Colin Cross635acc92017-09-12 22:50:46 -0700114 if *manifest != "" && !*emulateJar {
115 log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
116 }
117
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700118 // do merge
Colin Cross635acc92017-09-12 22:50:46 -0700119 err = mergeZips(readers, writer, *manifest, *sortEntries, *emulateJar, *stripDirEntries)
120 if err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700121 log.Fatal(err)
122 }
123}
124
125// a namedZipReader reads a .zip file and can say which file it's reading
126type namedZipReader struct {
127 path string
128 reader *zip.ReadCloser
129}
130
131// a zipEntryPath refers to a file contained in a zip
132type zipEntryPath struct {
133 zipName string
134 entryName string
135}
136
137func (p zipEntryPath) String() string {
138 return p.zipName + "/" + p.entryName
139}
140
Colin Cross635acc92017-09-12 22:50:46 -0700141// a zipEntry is a zipSource that pulls its content from another zip
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700142type zipEntry struct {
143 path zipEntryPath
144 content *zip.File
145}
146
Colin Cross635acc92017-09-12 22:50:46 -0700147func (ze zipEntry) String() string {
148 return ze.path.String()
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700149}
150
Colin Cross635acc92017-09-12 22:50:46 -0700151func (ze zipEntry) IsDir() bool {
152 return ze.content.FileInfo().IsDir()
153}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700154
Colin Cross635acc92017-09-12 22:50:46 -0700155func (ze zipEntry) CRC32() uint32 {
156 return ze.content.FileHeader.CRC32
157}
158
159func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error {
160 return zw.CopyFrom(ze.content, dest)
161}
162
163// a bufferEntry is a zipSource that pulls its content from a []byte
164type bufferEntry struct {
165 fh *zip.FileHeader
166 content []byte
167}
168
169func (be bufferEntry) String() string {
170 return "internal buffer"
171}
172
173func (be bufferEntry) IsDir() bool {
174 return be.fh.FileInfo().IsDir()
175}
176
177func (be bufferEntry) CRC32() uint32 {
178 return crc32.ChecksumIEEE(be.content)
179}
180
181func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error {
182 w, err := zw.CreateHeader(be.fh)
183 if err != nil {
184 return err
185 }
186
187 if !be.IsDir() {
188 _, err = w.Write(be.content)
189 if err != nil {
190 return err
191 }
192 }
193
194 return nil
195}
196
197type zipSource interface {
198 String() string
199 IsDir() bool
200 CRC32() uint32
201 WriteToZip(dest string, zw *zip.Writer) error
202}
203
204// a fileMapping specifies to copy a zip entry from one place to another
205type fileMapping struct {
206 dest string
207 source zipSource
208}
209
210func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest string,
211 sortEntries, emulateJar, stripDirEntries bool) error {
212
213 sourceByDest := make(map[string]zipSource, 0)
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700214 orderedMappings := []fileMapping{}
215
Colin Cross635acc92017-09-12 22:50:46 -0700216 // if dest already exists returns a non-null zipSource for the existing source
217 addMapping := func(dest string, source zipSource) zipSource {
218 mapKey := filepath.Clean(dest)
219 if existingSource, exists := sourceByDest[mapKey]; exists {
220 return existingSource
221 }
222
223 sourceByDest[mapKey] = source
224 orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest})
225 return nil
226 }
227
228 if manifest != "" {
229 if !stripDirEntries {
230 dirHeader := jar.MetaDirFileHeader()
231 dirSource := bufferEntry{dirHeader, nil}
232 addMapping(jar.MetaDir, dirSource)
233 }
234
235 fh, buf, err := jar.ManifestFileContents(manifest)
236 if err != nil {
237 return err
238 }
239
240 fileSource := bufferEntry{fh, buf}
241 addMapping(jar.ManifestFile, fileSource)
242 }
243
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700244 for _, namedReader := range readers {
Nan Zhang13f4cf52017-09-19 18:42:01 -0700245 _, skipStripThisZip := zipsToNotStrip[namedReader.path]
Nan Zhangd5998cc2017-09-13 13:17:43 -0700246 FileLoop:
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700247 for _, file := range namedReader.reader.File {
Nan Zhang13f4cf52017-09-19 18:42:01 -0700248 if !skipStripThisZip {
249 for _, dir := range stripDirs {
250 if strings.HasPrefix(file.Name, dir+"/") {
251 if emulateJar {
252 if file.Name != jar.MetaDir && file.Name != jar.ManifestFile {
253 continue FileLoop
254 }
255 } else {
256 continue FileLoop
257 }
258 }
Nan Zhangd5998cc2017-09-13 13:17:43 -0700259 }
260 }
Colin Cross635acc92017-09-12 22:50:46 -0700261
262 if stripDirEntries && file.FileInfo().IsDir() {
263 continue
264 }
265
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700266 // check for other files or directories destined for the same path
267 dest := file.Name
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700268
269 // make a new entry to add
270 source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700271
Colin Cross635acc92017-09-12 22:50:46 -0700272 if existingSource := addMapping(dest, source); existingSource != nil {
Colin Cross34540312017-09-06 12:52:37 -0700273 // handle duplicates
Colin Cross635acc92017-09-12 22:50:46 -0700274 if existingSource.IsDir() != source.IsDir() {
Colin Cross34540312017-09-06 12:52:37 -0700275 return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700276 dest, existingSource, source)
Colin Cross34540312017-09-06 12:52:37 -0700277 }
278 if emulateJar &&
279 file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass {
280 // Skip manifest and module info files that are not from the first input file
281 continue
282 }
Colin Cross635acc92017-09-12 22:50:46 -0700283 if !source.IsDir() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700284 if emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700285 if existingSource.CRC32() != source.CRC32() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700286 fmt.Fprintf(os.Stdout, "WARNING: Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700287 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700288 }
289 } else {
290 return fmt.Errorf("Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700291 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700292 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700293 }
294 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700295 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700296 }
297
Colin Cross34540312017-09-06 12:52:37 -0700298 if emulateJar {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700299 jarSort(orderedMappings)
300 } else if sortEntries {
301 alphanumericSort(orderedMappings)
302 }
303
304 for _, entry := range orderedMappings {
Colin Cross635acc92017-09-12 22:50:46 -0700305 if err := entry.source.WriteToZip(entry.dest, writer); err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700306 return err
307 }
308 }
309
310 return nil
311}
312
313func jarSort(files []fileMapping) {
314 sort.SliceStable(files, func(i, j int) bool {
315 return jar.EntryNamesLess(files[i].dest, files[j].dest)
316 })
317}
318
319func alphanumericSort(files []fileMapping) {
320 sort.SliceStable(files, func(i, j int) bool {
321 return files[i].dest < files[j].dest
322 })
323}