blob: 105765521835154662da8c3a55b9ec8f9172c377 [file] [log] [blame]
Jeff Gaston8bab5f22017-09-01 13:34:28 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
Colin Cross635acc92017-09-12 22:50:46 -070018 "errors"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070019 "flag"
20 "fmt"
Colin Cross635acc92017-09-12 22:50:46 -070021 "hash/crc32"
Nan Zhang5925b0f2017-12-19 15:13:40 -080022 "io/ioutil"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070023 "log"
24 "os"
Nan Zhang13f4cf52017-09-19 18:42:01 -070025 "path/filepath"
Jeff Gaston8bab5f22017-09-01 13:34:28 -070026 "sort"
27 "strings"
28
29 "android/soong/jar"
30 "android/soong/third_party/zip"
31)
32
Colin Cross0cf45cd2017-10-04 17:04:16 -070033type fileList []string
Nan Zhangd5998cc2017-09-13 13:17:43 -070034
Colin Cross0cf45cd2017-10-04 17:04:16 -070035func (f *fileList) String() string {
Nan Zhangd5998cc2017-09-13 13:17:43 -070036 return `""`
37}
38
Colin Cross0cf45cd2017-10-04 17:04:16 -070039func (f *fileList) Set(name string) error {
40 *f = append(*f, filepath.Clean(name))
Nan Zhang13f4cf52017-09-19 18:42:01 -070041
42 return nil
43}
44
Colin Cross0cf45cd2017-10-04 17:04:16 -070045type zipsToNotStripSet map[string]bool
Nan Zhang13f4cf52017-09-19 18:42:01 -070046
Colin Cross0cf45cd2017-10-04 17:04:16 -070047func (s zipsToNotStripSet) String() string {
Nan Zhang13f4cf52017-09-19 18:42:01 -070048 return `""`
49}
50
Colin Cross0cf45cd2017-10-04 17:04:16 -070051func (s zipsToNotStripSet) Set(zip_path string) error {
52 s[zip_path] = true
Nan Zhangd5998cc2017-09-13 13:17:43 -070053
54 return nil
55}
56
Jeff Gaston8bab5f22017-09-01 13:34:28 -070057var (
Colin Crosse909e1e2017-11-22 14:09:40 -080058 sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
59 emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
Nan Zhang5925b0f2017-12-19 15:13:40 -080060 emulatePar = flag.Bool("p", false, "merge zip entries based on par format")
Colin Crosse909e1e2017-11-22 14:09:40 -080061 stripDirs fileList
62 stripFiles fileList
63 zipsToNotStrip = make(zipsToNotStripSet)
64 stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
65 manifest = flag.String("m", "", "manifest file to insert in jar")
Nan Zhang1db85402017-12-18 13:20:23 -080066 pyMain = flag.String("pm", "", "__main__.py file to insert in par")
Nan Zhang5925b0f2017-12-19 15:13:40 -080067 entrypoint = flag.String("e", "", "par entrypoint file to insert in par")
Colin Crosse909e1e2017-11-22 14:09:40 -080068 ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070069)
70
Nan Zhangd5998cc2017-09-13 13:17:43 -070071func init() {
Colin Cross0cf45cd2017-10-04 17:04:16 -070072 flag.Var(&stripDirs, "stripDir", "the prefix of file path to be excluded from the output zip")
73 flag.Var(&stripFiles, "stripFile", "filenames to be excluded from the output zip, accepts wildcards")
74 flag.Var(&zipsToNotStrip, "zipToNotStrip", "the input zip file which is not applicable for stripping")
Nan Zhangd5998cc2017-09-13 13:17:43 -070075}
76
Jeff Gaston8bab5f22017-09-01 13:34:28 -070077func main() {
78 flag.Usage = func() {
Nan Zhang1db85402017-12-18 13:20:23 -080079 fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [-e entrypoint] [-pm __main__.py] output [inputs...]")
Jeff Gaston8bab5f22017-09-01 13:34:28 -070080 flag.PrintDefaults()
81 }
82
83 // parse args
84 flag.Parse()
85 args := flag.Args()
Colin Cross5c6ecc12017-10-23 18:12:27 -070086 if len(args) < 1 {
Jeff Gaston8bab5f22017-09-01 13:34:28 -070087 flag.Usage()
88 os.Exit(1)
89 }
90 outputPath := args[0]
91 inputs := args[1:]
92
93 log.SetFlags(log.Lshortfile)
94
95 // make writer
96 output, err := os.Create(outputPath)
97 if err != nil {
98 log.Fatal(err)
99 }
100 defer output.Close()
101 writer := zip.NewWriter(output)
102 defer func() {
103 err := writer.Close()
104 if err != nil {
105 log.Fatal(err)
106 }
107 }()
108
109 // make readers
110 readers := []namedZipReader{}
111 for _, input := range inputs {
112 reader, err := zip.OpenReader(input)
113 if err != nil {
114 log.Fatal(err)
115 }
116 defer reader.Close()
117 namedReader := namedZipReader{path: input, reader: reader}
118 readers = append(readers, namedReader)
119 }
120
Colin Cross635acc92017-09-12 22:50:46 -0700121 if *manifest != "" && !*emulateJar {
122 log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
123 }
124
Nan Zhang5925b0f2017-12-19 15:13:40 -0800125 if *entrypoint != "" && !*emulatePar {
126 log.Fatal(errors.New("must specify -p when specifying a entrypoint via -e"))
127 }
128
Nan Zhang1db85402017-12-18 13:20:23 -0800129 if *pyMain != "" && !*emulatePar {
130 log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm"))
131 }
132
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700133 // do merge
Nan Zhang1db85402017-12-18 13:20:23 -0800134 err = mergeZips(readers, writer, *manifest, *entrypoint, *pyMain, *sortEntries, *emulateJar, *emulatePar,
Nan Zhang5925b0f2017-12-19 15:13:40 -0800135 *stripDirEntries, *ignoreDuplicates)
Colin Cross635acc92017-09-12 22:50:46 -0700136 if err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700137 log.Fatal(err)
138 }
139}
140
141// a namedZipReader reads a .zip file and can say which file it's reading
142type namedZipReader struct {
143 path string
144 reader *zip.ReadCloser
145}
146
147// a zipEntryPath refers to a file contained in a zip
148type zipEntryPath struct {
149 zipName string
150 entryName string
151}
152
153func (p zipEntryPath) String() string {
154 return p.zipName + "/" + p.entryName
155}
156
Colin Cross635acc92017-09-12 22:50:46 -0700157// a zipEntry is a zipSource that pulls its content from another zip
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700158type zipEntry struct {
159 path zipEntryPath
160 content *zip.File
161}
162
Colin Cross635acc92017-09-12 22:50:46 -0700163func (ze zipEntry) String() string {
164 return ze.path.String()
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700165}
166
Colin Cross635acc92017-09-12 22:50:46 -0700167func (ze zipEntry) IsDir() bool {
168 return ze.content.FileInfo().IsDir()
169}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700170
Colin Cross635acc92017-09-12 22:50:46 -0700171func (ze zipEntry) CRC32() uint32 {
172 return ze.content.FileHeader.CRC32
173}
174
175func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error {
176 return zw.CopyFrom(ze.content, dest)
177}
178
179// a bufferEntry is a zipSource that pulls its content from a []byte
180type bufferEntry struct {
181 fh *zip.FileHeader
182 content []byte
183}
184
185func (be bufferEntry) String() string {
186 return "internal buffer"
187}
188
189func (be bufferEntry) IsDir() bool {
190 return be.fh.FileInfo().IsDir()
191}
192
193func (be bufferEntry) CRC32() uint32 {
194 return crc32.ChecksumIEEE(be.content)
195}
196
197func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error {
198 w, err := zw.CreateHeader(be.fh)
199 if err != nil {
200 return err
201 }
202
203 if !be.IsDir() {
204 _, err = w.Write(be.content)
205 if err != nil {
206 return err
207 }
208 }
209
210 return nil
211}
212
213type zipSource interface {
214 String() string
215 IsDir() bool
216 CRC32() uint32
217 WriteToZip(dest string, zw *zip.Writer) error
218}
219
220// a fileMapping specifies to copy a zip entry from one place to another
221type fileMapping struct {
222 dest string
223 source zipSource
224}
225
Nan Zhang1db85402017-12-18 13:20:23 -0800226func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoint, pyMain string,
Nan Zhang5925b0f2017-12-19 15:13:40 -0800227 sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool) error {
Colin Cross635acc92017-09-12 22:50:46 -0700228
229 sourceByDest := make(map[string]zipSource, 0)
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700230 orderedMappings := []fileMapping{}
231
Colin Cross635acc92017-09-12 22:50:46 -0700232 // if dest already exists returns a non-null zipSource for the existing source
233 addMapping := func(dest string, source zipSource) zipSource {
234 mapKey := filepath.Clean(dest)
235 if existingSource, exists := sourceByDest[mapKey]; exists {
236 return existingSource
237 }
238
239 sourceByDest[mapKey] = source
240 orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest})
241 return nil
242 }
243
244 if manifest != "" {
245 if !stripDirEntries {
246 dirHeader := jar.MetaDirFileHeader()
247 dirSource := bufferEntry{dirHeader, nil}
248 addMapping(jar.MetaDir, dirSource)
249 }
250
251 fh, buf, err := jar.ManifestFileContents(manifest)
252 if err != nil {
253 return err
254 }
255
256 fileSource := bufferEntry{fh, buf}
257 addMapping(jar.ManifestFile, fileSource)
258 }
259
Nan Zhang5925b0f2017-12-19 15:13:40 -0800260 if entrypoint != "" {
261 buf, err := ioutil.ReadFile(entrypoint)
262 if err != nil {
263 return err
264 }
265 fh := &zip.FileHeader{
266 Name: "entry_point.txt",
267 Method: zip.Store,
268 UncompressedSize64: uint64(len(buf)),
269 }
270 fh.SetMode(0700)
271 fh.SetModTime(jar.DefaultTime)
272 fileSource := bufferEntry{fh, buf}
273 addMapping("entry_point.txt", fileSource)
274 }
275
Nan Zhang1db85402017-12-18 13:20:23 -0800276 if pyMain != "" {
277 buf, err := ioutil.ReadFile(pyMain)
278 if err != nil {
279 return err
280 }
281 fh := &zip.FileHeader{
282 Name: "__main__.py",
283 Method: zip.Store,
284 UncompressedSize64: uint64(len(buf)),
285 }
286 fh.SetMode(0700)
287 fh.SetModTime(jar.DefaultTime)
288 fileSource := bufferEntry{fh, buf}
289 addMapping("__main__.py", fileSource)
290 }
291
Nan Zhang5925b0f2017-12-19 15:13:40 -0800292 if emulatePar {
293 // the runfiles packages needs to be populated with "__init__.py".
294 newPyPkgs := []string{}
295 // the runfiles dirs have been treated as packages.
296 existingPyPkgSet := make(map[string]bool)
297 // put existing __init__.py files to a set first. This set is used for preventing
298 // generated __init__.py files from overwriting existing ones.
299 for _, namedReader := range readers {
300 for _, file := range namedReader.reader.File {
301 if filepath.Base(file.Name) != "__init__.py" {
302 continue
303 }
304 pyPkg := pathBeforeLastSlash(file.Name)
305 if _, found := existingPyPkgSet[pyPkg]; found {
306 panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q.", file.Name))
307 } else {
308 existingPyPkgSet[pyPkg] = true
309 }
310 }
311 }
312 for _, namedReader := range readers {
313 for _, file := range namedReader.reader.File {
314 var parentPath string /* the path after trimming last "/" */
315 if filepath.Base(file.Name) == "__init__.py" {
316 // for existing __init__.py files, we should trim last "/" for twice.
317 // eg. a/b/c/__init__.py ---> a/b
318 parentPath = pathBeforeLastSlash(pathBeforeLastSlash(file.Name))
319 } else {
320 parentPath = pathBeforeLastSlash(file.Name)
321 }
322 populateNewPyPkgs(parentPath, existingPyPkgSet, &newPyPkgs)
323 }
324 }
325 for _, pkg := range newPyPkgs {
326 var emptyBuf []byte
327 fh := &zip.FileHeader{
328 Name: filepath.Join(pkg, "__init__.py"),
329 Method: zip.Store,
330 UncompressedSize64: uint64(len(emptyBuf)),
331 }
332 fh.SetMode(0700)
333 fh.SetModTime(jar.DefaultTime)
334 fileSource := bufferEntry{fh, emptyBuf}
335 addMapping(filepath.Join(pkg, "__init__.py"), fileSource)
336 }
337 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700338 for _, namedReader := range readers {
Nan Zhang13f4cf52017-09-19 18:42:01 -0700339 _, skipStripThisZip := zipsToNotStrip[namedReader.path]
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700340 for _, file := range namedReader.reader.File {
Colin Cross0cf45cd2017-10-04 17:04:16 -0700341 if !skipStripThisZip && shouldStripFile(emulateJar, file.Name) {
342 continue
Nan Zhangd5998cc2017-09-13 13:17:43 -0700343 }
Colin Cross635acc92017-09-12 22:50:46 -0700344
345 if stripDirEntries && file.FileInfo().IsDir() {
346 continue
347 }
348
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700349 // check for other files or directories destined for the same path
350 dest := file.Name
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700351
352 // make a new entry to add
353 source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file}
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700354
Colin Cross635acc92017-09-12 22:50:46 -0700355 if existingSource := addMapping(dest, source); existingSource != nil {
Colin Cross34540312017-09-06 12:52:37 -0700356 // handle duplicates
Colin Cross635acc92017-09-12 22:50:46 -0700357 if existingSource.IsDir() != source.IsDir() {
Colin Cross34540312017-09-06 12:52:37 -0700358 return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700359 dest, existingSource, source)
Colin Cross34540312017-09-06 12:52:37 -0700360 }
Colin Crosse909e1e2017-11-22 14:09:40 -0800361 if ignoreDuplicates {
362 continue
363 }
Colin Cross34540312017-09-06 12:52:37 -0700364 if emulateJar &&
365 file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass {
366 // Skip manifest and module info files that are not from the first input file
367 continue
368 }
Colin Cross635acc92017-09-12 22:50:46 -0700369 if !source.IsDir() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700370 if emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700371 if existingSource.CRC32() != source.CRC32() {
Nan Zhangd5998cc2017-09-13 13:17:43 -0700372 fmt.Fprintf(os.Stdout, "WARNING: Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700373 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700374 }
375 } else {
376 return fmt.Errorf("Duplicate path %v found in %v and %v\n",
Colin Cross635acc92017-09-12 22:50:46 -0700377 dest, existingSource, source)
Nan Zhangd5998cc2017-09-13 13:17:43 -0700378 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700379 }
380 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700381 }
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700382 }
383
Colin Cross34540312017-09-06 12:52:37 -0700384 if emulateJar {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700385 jarSort(orderedMappings)
386 } else if sortEntries {
387 alphanumericSort(orderedMappings)
388 }
389
390 for _, entry := range orderedMappings {
Colin Cross635acc92017-09-12 22:50:46 -0700391 if err := entry.source.WriteToZip(entry.dest, writer); err != nil {
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700392 return err
393 }
394 }
395
396 return nil
397}
398
Nan Zhang5925b0f2017-12-19 15:13:40 -0800399// Sets the given directory and all its ancestor directories as Python packages.
400func populateNewPyPkgs(pkgPath string, existingPyPkgSet map[string]bool, newPyPkgs *[]string) {
401 for pkgPath != "" {
402 if _, found := existingPyPkgSet[pkgPath]; !found {
403 existingPyPkgSet[pkgPath] = true
404 *newPyPkgs = append(*newPyPkgs, pkgPath)
405 // Gets its ancestor directory by trimming last slash.
406 pkgPath = pathBeforeLastSlash(pkgPath)
407 } else {
408 break
409 }
410 }
411}
412
413func pathBeforeLastSlash(path string) string {
414 ret := filepath.Dir(path)
415 // filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/".
416 if ret == "." || ret == "/" {
417 return ""
418 }
419 return ret
420}
421
Colin Cross0cf45cd2017-10-04 17:04:16 -0700422func shouldStripFile(emulateJar bool, name string) bool {
423 for _, dir := range stripDirs {
424 if strings.HasPrefix(name, dir+"/") {
425 if emulateJar {
426 if name != jar.MetaDir && name != jar.ManifestFile {
427 return true
428 }
429 } else {
430 return true
431 }
432 }
433 }
434 for _, pattern := range stripFiles {
435 if match, err := filepath.Match(pattern, filepath.Base(name)); err != nil {
436 panic(fmt.Errorf("%s: %s", err.Error(), pattern))
437 } else if match {
438 return true
439 }
440 }
441 return false
442}
443
Jeff Gaston8bab5f22017-09-01 13:34:28 -0700444func jarSort(files []fileMapping) {
445 sort.SliceStable(files, func(i, j int) bool {
446 return jar.EntryNamesLess(files[i].dest, files[j].dest)
447 })
448}
449
450func alphanumericSort(files []fileMapping) {
451 sort.SliceStable(files, func(i, j int) bool {
452 return files[i].dest < files[j].dest
453 })
454}