blob: f383de90a12e1f5c86140c9dd09d66e42d58d30d [file] [log] [blame]
Jeff Gaston8bab5f22017-09-01 13:34:28 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
Colin Cross635acc92017-09-12 22:50:46 -070018 "errors"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070019 "flag"
20 "fmt"
Colin Cross635acc92017-09-12 22:50:46 -070021 "hash/crc32"
Nan Zhang5925b0f2017-12-19 15:13:40 -080022 "io/ioutil"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070023 "log"
24 "os"
Nan Zhang13f4cf52017-09-19 18:42:01 -070025 "path/filepath"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070026 "sort"
Colin Cross4c03f682018-07-15 08:16:31 -070027
28 "github.com/google/blueprint/pathtools"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070029
30 "android/soong/jar"
31 "android/soong/third_party/zip"
32)
33
Colin Cross0cf45cd2017-10-04 17:04:16 -070034type fileList []string
Nan Zhangd5998cc2017-09-13 13:17:43 -070035
Colin Cross0cf45cd2017-10-04 17:04:16 -070036func (f *fileList) String() string {
Nan Zhangd5998cc2017-09-13 13:17:43 -070037 return `""`
38}
39
Colin Cross0cf45cd2017-10-04 17:04:16 -070040func (f *fileList) Set(name string) error {
41 *f = append(*f, filepath.Clean(name))
Nan Zhang13f4cf52017-09-19 18:42:01 -070042
43 return nil
44}
45
Colin Cross0cf45cd2017-10-04 17:04:16 -070046type zipsToNotStripSet map[string]bool
Nan Zhang13f4cf52017-09-19 18:42:01 -070047
Colin Cross0cf45cd2017-10-04 17:04:16 -070048func (s zipsToNotStripSet) String() string {
Nan Zhang13f4cf52017-09-19 18:42:01 -070049 return `""`
50}
51
Colin Cross0cf45cd2017-10-04 17:04:16 -070052func (s zipsToNotStripSet) Set(zip_path string) error {
53 s[zip_path] = true
Nan Zhangd5998cc2017-09-13 13:17:43 -070054
55 return nil
56}
57
Jeff Gaston8bab5f22017-09-01 13:34:28 -070058var (
Colin Crosse909e1e2017-11-22 14:09:40 -080059 sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
60 emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
Nan Zhang5925b0f2017-12-19 15:13:40 -080061 emulatePar = flag.Bool("p", false, "merge zip entries based on par format")
Colin Crosse909e1e2017-11-22 14:09:40 -080062 stripDirs fileList
63 stripFiles fileList
64 zipsToNotStrip = make(zipsToNotStripSet)
65 stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
66 manifest = flag.String("m", "", "manifest file to insert in jar")
Nan Zhang1db85402017-12-18 13:20:23 -080067 pyMain = flag.String("pm", "", "__main__.py file to insert in par")
Nan Zhang5925b0f2017-12-19 15:13:40 -080068 entrypoint = flag.String("e", "", "par entrypoint file to insert in par")
Colin Crosse909e1e2017-11-22 14:09:40 -080069 ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070070)
71
Nan Zhangd5998cc2017-09-13 13:17:43 -070072func init() {
Colin Cross4c03f682018-07-15 08:16:31 -070073 flag.Var(&stripDirs, "stripDir", "directories to be excluded from the output zip, accepts wildcards")
74 flag.Var(&stripFiles, "stripFile", "files to be excluded from the output zip, accepts wildcards")
Colin Cross0cf45cd2017-10-04 17:04:16 -070075 flag.Var(&zipsToNotStrip, "zipToNotStrip", "the input zip file which is not applicable for stripping")
Nan Zhangd5998cc2017-09-13 13:17:43 -070076}
77
Jeff Gaston8bab5f22017-09-01 13:34:28 -070078func main() {
79 flag.Usage = func() {
Nan Zhang1db85402017-12-18 13:20:23 -080080 fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [-e entrypoint] [-pm __main__.py] output [inputs...]")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070081 flag.PrintDefaults()
82 }
83
84 // parse args
85 flag.Parse()
86 args := flag.Args()
Colin Cross5c6ecc12017-10-23 18:12:27 -070087 if len(args) < 1 {
Jeff Gaston8bab5f22017-09-01 13:34:28 -070088 flag.Usage()
89 os.Exit(1)
90 }
91 outputPath := args[0]
92 inputs := args[1:]
93
94 log.SetFlags(log.Lshortfile)
95
96 // make writer
97 output, err := os.Create(outputPath)
98 if err != nil {
99 log.Fatal(err)
100 }
101 defer output.Close()
102 writer := zip.NewWriter(output)
103 defer func() {
104 err := writer.Close()
105 if err != nil {
106 log.Fatal(err)
107 }
108 }()
109
110 // make readers
111 readers := []namedZipReader{}
112 for _, input := range inputs {
113 reader, err := zip.OpenReader(input)
114 if err != nil {
115 log.Fatal(err)
116 }
117 defer reader.Close()
Colin Cross24860652018-07-14 22:19:14 -0700118 namedReader := namedZipReader{path: input, reader: &reader.Reader}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700119 readers = append(readers, namedReader)
120 }
121
Colin Cross635acc92017-09-12 22:50:46 -0700122 if *manifest != "" && !*emulateJar {
123 log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
124 }
125
Nan Zhang5925b0f2017-12-19 15:13:40 -0800126 if *entrypoint != "" && !*emulatePar {
127 log.Fatal(errors.New("must specify -p when specifying a entrypoint via -e"))
128 }
129
Nan Zhang1db85402017-12-18 13:20:23 -0800130 if *pyMain != "" && !*emulatePar {
131 log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm"))
132 }
133
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700134 // do merge
Nan Zhang1db85402017-12-18 13:20:23 -0800135 err = mergeZips(readers, writer, *manifest, *entrypoint, *pyMain, *sortEntries, *emulateJar, *emulatePar,
Colin Cross24860652018-07-14 22:19:14 -0700136 *stripDirEntries, *ignoreDuplicates, []string(stripFiles), []string(stripDirs), map[string]bool(zipsToNotStrip))
Colin Cross635acc92017-09-12 22:50:46 -0700137 if err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700138 log.Fatal(err)
139 }
140}
141
142// a namedZipReader reads a .zip file and can say which file it's reading
143type namedZipReader struct {
144 path string
Colin Cross24860652018-07-14 22:19:14 -0700145 reader *zip.Reader
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700146}
147
148// a zipEntryPath refers to a file contained in a zip
149type zipEntryPath struct {
150 zipName string
151 entryName string
152}
153
154func (p zipEntryPath) String() string {
155 return p.zipName + "/" + p.entryName
156}
157
Colin Cross635acc92017-09-12 22:50:46 -0700158// a zipEntry is a zipSource that pulls its content from another zip
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700159type zipEntry struct {
160 path zipEntryPath
161 content *zip.File
162}
163
Colin Cross635acc92017-09-12 22:50:46 -0700164func (ze zipEntry) String() string {
165 return ze.path.String()
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700166}
167
Colin Cross635acc92017-09-12 22:50:46 -0700168func (ze zipEntry) IsDir() bool {
169 return ze.content.FileInfo().IsDir()
170}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700171
Colin Cross635acc92017-09-12 22:50:46 -0700172func (ze zipEntry) CRC32() uint32 {
173 return ze.content.FileHeader.CRC32
174}
175
176func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error {
177 return zw.CopyFrom(ze.content, dest)
178}
179
180// a bufferEntry is a zipSource that pulls its content from a []byte
181type bufferEntry struct {
182 fh *zip.FileHeader
183 content []byte
184}
185
186func (be bufferEntry) String() string {
187 return "internal buffer"
188}
189
190func (be bufferEntry) IsDir() bool {
191 return be.fh.FileInfo().IsDir()
192}
193
194func (be bufferEntry) CRC32() uint32 {
195 return crc32.ChecksumIEEE(be.content)
196}
197
198func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error {
199 w, err := zw.CreateHeader(be.fh)
200 if err != nil {
201 return err
202 }
203
204 if !be.IsDir() {
205 _, err = w.Write(be.content)
206 if err != nil {
207 return err
208 }
209 }
210
211 return nil
212}
213
214type zipSource interface {
215 String() string
216 IsDir() bool
217 CRC32() uint32
218 WriteToZip(dest string, zw *zip.Writer) error
219}
220
221// a fileMapping specifies to copy a zip entry from one place to another
222type fileMapping struct {
223 dest string
224 source zipSource
225}
226
Nan Zhang1db85402017-12-18 13:20:23 -0800227func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoint, pyMain string,
Colin Cross24860652018-07-14 22:19:14 -0700228 sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool,
229 stripFiles, stripDirs []string, zipsToNotStrip map[string]bool) error {
Colin Cross635acc92017-09-12 22:50:46 -0700230
231 sourceByDest := make(map[string]zipSource, 0)
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700232 orderedMappings := []fileMapping{}
233
Colin Cross635acc92017-09-12 22:50:46 -0700234 // if dest already exists returns a non-null zipSource for the existing source
235 addMapping := func(dest string, source zipSource) zipSource {
236 mapKey := filepath.Clean(dest)
237 if existingSource, exists := sourceByDest[mapKey]; exists {
238 return existingSource
239 }
240
241 sourceByDest[mapKey] = source
242 orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest})
243 return nil
244 }
245
246 if manifest != "" {
247 if !stripDirEntries {
248 dirHeader := jar.MetaDirFileHeader()
249 dirSource := bufferEntry{dirHeader, nil}
250 addMapping(jar.MetaDir, dirSource)
251 }
252
Colin Cross05518bc2018-09-27 15:06:19 -0700253 contents, err := ioutil.ReadFile(manifest)
254 if err != nil {
255 return err
256 }
257
258 fh, buf, err := jar.ManifestFileContents(contents)
Colin Cross635acc92017-09-12 22:50:46 -0700259 if err != nil {
260 return err
261 }
262
263 fileSource := bufferEntry{fh, buf}
264 addMapping(jar.ManifestFile, fileSource)
265 }
266
Nan Zhang5925b0f2017-12-19 15:13:40 -0800267 if entrypoint != "" {
268 buf, err := ioutil.ReadFile(entrypoint)
269 if err != nil {
270 return err
271 }
272 fh := &zip.FileHeader{
273 Name: "entry_point.txt",
274 Method: zip.Store,
275 UncompressedSize64: uint64(len(buf)),
276 }
277 fh.SetMode(0700)
278 fh.SetModTime(jar.DefaultTime)
279 fileSource := bufferEntry{fh, buf}
280 addMapping("entry_point.txt", fileSource)
281 }
282
Nan Zhang1db85402017-12-18 13:20:23 -0800283 if pyMain != "" {
284 buf, err := ioutil.ReadFile(pyMain)
285 if err != nil {
286 return err
287 }
288 fh := &zip.FileHeader{
289 Name: "__main__.py",
290 Method: zip.Store,
291 UncompressedSize64: uint64(len(buf)),
292 }
293 fh.SetMode(0700)
294 fh.SetModTime(jar.DefaultTime)
295 fileSource := bufferEntry{fh, buf}
296 addMapping("__main__.py", fileSource)
297 }
298
Nan Zhang5925b0f2017-12-19 15:13:40 -0800299 if emulatePar {
300 // the runfiles packages needs to be populated with "__init__.py".
301 newPyPkgs := []string{}
302 // the runfiles dirs have been treated as packages.
303 existingPyPkgSet := make(map[string]bool)
304 // put existing __init__.py files to a set first. This set is used for preventing
305 // generated __init__.py files from overwriting existing ones.
306 for _, namedReader := range readers {
307 for _, file := range namedReader.reader.File {
308 if filepath.Base(file.Name) != "__init__.py" {
309 continue
310 }
311 pyPkg := pathBeforeLastSlash(file.Name)
312 if _, found := existingPyPkgSet[pyPkg]; found {
313 panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q.", file.Name))
314 } else {
315 existingPyPkgSet[pyPkg] = true
316 }
317 }
318 }
319 for _, namedReader := range readers {
320 for _, file := range namedReader.reader.File {
321 var parentPath string /* the path after trimming last "/" */
322 if filepath.Base(file.Name) == "__init__.py" {
323 // for existing __init__.py files, we should trim last "/" for twice.
324 // eg. a/b/c/__init__.py ---> a/b
325 parentPath = pathBeforeLastSlash(pathBeforeLastSlash(file.Name))
326 } else {
327 parentPath = pathBeforeLastSlash(file.Name)
328 }
329 populateNewPyPkgs(parentPath, existingPyPkgSet, &newPyPkgs)
330 }
331 }
332 for _, pkg := range newPyPkgs {
333 var emptyBuf []byte
334 fh := &zip.FileHeader{
335 Name: filepath.Join(pkg, "__init__.py"),
336 Method: zip.Store,
337 UncompressedSize64: uint64(len(emptyBuf)),
338 }
339 fh.SetMode(0700)
340 fh.SetModTime(jar.DefaultTime)
341 fileSource := bufferEntry{fh, emptyBuf}
342 addMapping(filepath.Join(pkg, "__init__.py"), fileSource)
343 }
344 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700345 for _, namedReader := range readers {
Nan Zhang13f4cf52017-09-19 18:42:01 -0700346 _, skipStripThisZip := zipsToNotStrip[namedReader.path]
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700347 for _, file := range namedReader.reader.File {
Colin Cross4c03f682018-07-15 08:16:31 -0700348 if !skipStripThisZip {
349 if skip, err := shouldStripEntry(emulateJar, stripFiles, stripDirs, file.Name); err != nil {
350 return err
351 } else if skip {
352 continue
353 }
Nan Zhangd5998cc2017-09-13 13:17:43 -0700354 }
Colin Cross635acc92017-09-12 22:50:46 -0700355
356 if stripDirEntries && file.FileInfo().IsDir() {
357 continue
358 }
359
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700360 // check for other files or directories destined for the same path
361 dest := file.Name
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700362
363 // make a new entry to add
364 source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700365
Colin Cross635acc92017-09-12 22:50:46 -0700366 if existingSource := addMapping(dest, source); existingSource != nil {
Colin Cross34540312017-09-06 12:52:37 -0700367 // handle duplicates
Colin Cross635acc92017-09-12 22:50:46 -0700368 if existingSource.IsDir() != source.IsDir() {
Colin Cross34540312017-09-06 12:52:37 -0700369 return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700370 dest, existingSource, source)
Colin Cross34540312017-09-06 12:52:37 -0700371 }
Colin Crosse909e1e2017-11-22 14:09:40 -0800372 if ignoreDuplicates {
373 continue
374 }
Colin Cross34540312017-09-06 12:52:37 -0700375 if emulateJar &&
376 file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass {
377 // Skip manifest and module info files that are not from the first input file
378 continue
379 }
Colin Cross635acc92017-09-12 22:50:46 -0700380 if !source.IsDir() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700381 if emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700382 if existingSource.CRC32() != source.CRC32() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700383 fmt.Fprintf(os.Stdout, "WARNING: Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700384 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700385 }
386 } else {
387 return fmt.Errorf("Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700388 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700389 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700390 }
391 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700392 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700393 }
394
Colin Cross34540312017-09-06 12:52:37 -0700395 if emulateJar {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700396 jarSort(orderedMappings)
397 } else if sortEntries {
398 alphanumericSort(orderedMappings)
399 }
400
401 for _, entry := range orderedMappings {
Colin Cross635acc92017-09-12 22:50:46 -0700402 if err := entry.source.WriteToZip(entry.dest, writer); err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700403 return err
404 }
405 }
406
407 return nil
408}
409
Nan Zhang5925b0f2017-12-19 15:13:40 -0800410// Sets the given directory and all its ancestor directories as Python packages.
411func populateNewPyPkgs(pkgPath string, existingPyPkgSet map[string]bool, newPyPkgs *[]string) {
412 for pkgPath != "" {
413 if _, found := existingPyPkgSet[pkgPath]; !found {
414 existingPyPkgSet[pkgPath] = true
415 *newPyPkgs = append(*newPyPkgs, pkgPath)
416 // Gets its ancestor directory by trimming last slash.
417 pkgPath = pathBeforeLastSlash(pkgPath)
418 } else {
419 break
420 }
421 }
422}
423
424func pathBeforeLastSlash(path string) string {
425 ret := filepath.Dir(path)
426 // filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/".
427 if ret == "." || ret == "/" {
428 return ""
429 }
430 return ret
431}
432
Colin Cross4c03f682018-07-15 08:16:31 -0700433func shouldStripEntry(emulateJar bool, stripFiles, stripDirs []string, name string) (bool, error) {
Colin Cross0cf45cd2017-10-04 17:04:16 -0700434 for _, dir := range stripDirs {
Colin Cross4c03f682018-07-15 08:16:31 -0700435 dir = filepath.Clean(dir)
436 patterns := []string{
437 dir + "/", // the directory itself
438 dir + "/**/*", // files recursively in the directory
439 dir + "/**/*/", // directories recursively in the directory
440 }
441
442 for _, pattern := range patterns {
443 match, err := pathtools.Match(pattern, name)
444 if err != nil {
445 return false, fmt.Errorf("%s: %s", err.Error(), pattern)
446 } else if match {
447 if emulateJar {
448 // When merging jar files, don't strip META-INF/MANIFEST.MF even if stripping META-INF is
449 // requested.
450 // TODO(ccross): which files does this affect?
451 if name != jar.MetaDir && name != jar.ManifestFile {
452 return true, nil
453 }
Colin Cross0cf45cd2017-10-04 17:04:16 -0700454 }
Colin Cross4c03f682018-07-15 08:16:31 -0700455 return true, nil
Colin Cross0cf45cd2017-10-04 17:04:16 -0700456 }
457 }
458 }
Colin Cross4c03f682018-07-15 08:16:31 -0700459
Colin Cross0cf45cd2017-10-04 17:04:16 -0700460 for _, pattern := range stripFiles {
Colin Cross4c03f682018-07-15 08:16:31 -0700461 if match, err := pathtools.Match(pattern, name); err != nil {
462 return false, fmt.Errorf("%s: %s", err.Error(), pattern)
Colin Cross0cf45cd2017-10-04 17:04:16 -0700463 } else if match {
Colin Cross4c03f682018-07-15 08:16:31 -0700464 return true, nil
Colin Cross0cf45cd2017-10-04 17:04:16 -0700465 }
466 }
Colin Cross4c03f682018-07-15 08:16:31 -0700467 return false, nil
Colin Cross0cf45cd2017-10-04 17:04:16 -0700468}
469
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700470func jarSort(files []fileMapping) {
471 sort.SliceStable(files, func(i, j int) bool {
472 return jar.EntryNamesLess(files[i].dest, files[j].dest)
473 })
474}
475
476func alphanumericSort(files []fileMapping) {
477 sort.SliceStable(files, func(i, j int) bool {
478 return files[i].dest < files[j].dest
479 })
480}