blob: 3c710a782d66d97d85dd61cd488e8eebf0abba11 [file] [log] [blame]
Colin Cross2fe66872015-03-30 17:20:39 -07001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Jeff Gaston11b5c512017-10-12 12:19:14 -070015package zip
Colin Cross2fe66872015-03-30 17:20:39 -070016
17import (
Dan Willemsen017d8932016-08-04 15:43:03 -070018 "bytes"
19 "compress/flate"
Jeff Gastona2976952017-08-22 17:51:25 -070020 "errors"
Colin Cross2fe66872015-03-30 17:20:39 -070021 "fmt"
Dan Willemsen017d8932016-08-04 15:43:03 -070022 "hash/crc32"
Colin Cross2fe66872015-03-30 17:20:39 -070023 "io"
24 "io/ioutil"
25 "os"
26 "path/filepath"
Jeff Gastona2976952017-08-22 17:51:25 -070027 "sort"
Colin Cross2fe66872015-03-30 17:20:39 -070028 "strings"
Dan Willemsen017d8932016-08-04 15:43:03 -070029 "sync"
Colin Cross1d98ee22018-09-18 17:05:15 -070030 "syscall"
Colin Cross2fe66872015-03-30 17:20:39 -070031 "time"
Nan Zhang674dd932018-01-26 18:30:36 -080032 "unicode"
Dan Willemsen017d8932016-08-04 15:43:03 -070033
Colin Crossf83c1502017-11-10 13:11:02 -080034 "github.com/google/blueprint/pathtools"
35
Jeff Gastona2976952017-08-22 17:51:25 -070036 "android/soong/jar"
Dan Willemsen017d8932016-08-04 15:43:03 -070037 "android/soong/third_party/zip"
Colin Cross2fe66872015-03-30 17:20:39 -070038)
39
Dan Willemsen017d8932016-08-04 15:43:03 -070040// Block size used during parallel compression of a single file.
41const parallelBlockSize = 1 * 1024 * 1024 // 1MB
42
43// Minimum file size to use parallel compression. It requires more
44// flate.Writer allocations, since we can't change the dictionary
45// during Reset
46const minParallelFileSize = parallelBlockSize * 6
47
48// Size of the ZIP compression window (32KB)
49const windowSize = 32 * 1024
50
51type nopCloser struct {
52 io.Writer
53}
54
55func (nopCloser) Close() error {
56 return nil
57}
58
Jeff Gastoncef50b92017-08-23 15:41:35 -070059type byteReaderCloser struct {
Colin Cross635acc92017-09-12 22:50:46 -070060 *bytes.Reader
Jeff Gastoncef50b92017-08-23 15:41:35 -070061 io.Closer
62}
63
Nan Zhang9067b042017-03-17 14:04:43 -070064type pathMapping struct {
65 dest, src string
Nan Zhangf281bd82017-04-25 16:47:45 -070066 zipMethod uint16
67}
68
Jeff Gastonc3bdc972017-10-12 12:18:19 -070069type FileArg struct {
70 PathPrefixInZip, SourcePrefixToStrip string
71 SourceFiles []string
Colin Crossb7c69112018-09-18 16:51:43 -070072 JunkPaths bool
Jeff Gastonc3bdc972017-10-12 12:18:19 -070073 GlobDir string
74}
75
Colin Crossfe945b42018-09-27 15:00:07 -070076type FileArgsBuilder struct {
77 state FileArg
78 err error
79 fs pathtools.FileSystem
80
81 fileArgs []FileArg
82}
83
84func NewFileArgsBuilder() *FileArgsBuilder {
85 return &FileArgsBuilder{
86 fs: pathtools.OsFs,
87 }
88}
89
90func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
91 b.state.JunkPaths = v
92 b.state.SourcePrefixToStrip = ""
93 return b
94}
95
96func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
97 b.state.JunkPaths = false
98 b.state.SourcePrefixToStrip = prefixToStrip
99 return b
100}
101
102func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
103 b.state.PathPrefixInZip = rootPrefix
104 return b
105}
106
107func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
108 if b.err != nil {
109 return b
110 }
111
112 arg := b.state
113 arg.SourceFiles = []string{name}
114 b.fileArgs = append(b.fileArgs, arg)
115 return b
116}
117
118func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
119 if b.err != nil {
120 return b
121 }
122
123 arg := b.state
124 arg.GlobDir = name
125 b.fileArgs = append(b.fileArgs, arg)
126 return b
127}
128
129func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
130 if b.err != nil {
131 return b
132 }
133
134 f, err := b.fs.Open(name)
135 if err != nil {
136 b.err = err
137 return b
138 }
139 defer f.Close()
140
141 list, err := ioutil.ReadAll(f)
142 if err != nil {
143 b.err = err
144 return b
145 }
146
147 arg := b.state
Jiyong Park04bbf982019-11-04 13:18:41 +0900148 arg.SourceFiles = strings.Fields(string(list))
Colin Crossfe945b42018-09-27 15:00:07 -0700149 b.fileArgs = append(b.fileArgs, arg)
150 return b
151}
152
153func (b *FileArgsBuilder) Error() error {
154 if b == nil {
155 return nil
156 }
157 return b.err
158}
159
160func (b *FileArgsBuilder) FileArgs() []FileArg {
161 if b == nil {
162 return nil
163 }
164 return b.fileArgs
165}
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700166
Colin Cross1d98ee22018-09-18 17:05:15 -0700167type IncorrectRelativeRootError struct {
168 RelativeRoot string
169 Path string
170}
171
172func (x IncorrectRelativeRootError) Error() string {
173 return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
174}
175
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700176type ZipWriter struct {
Colin Crosse5580972017-08-30 17:40:21 -0700177 time time.Time
178 createdFiles map[string]string
179 createdDirs map[string]string
180 directories bool
Colin Crosse19c7932015-04-24 15:08:38 -0700181
Dan Willemsen017d8932016-08-04 15:43:03 -0700182 errors chan error
183 writeOps chan chan *zipEntry
184
Jeff Gaston175f34c2017-08-17 21:43:21 -0700185 cpuRateLimiter *CPURateLimiter
186 memoryRateLimiter *MemoryRateLimiter
Dan Willemsen017d8932016-08-04 15:43:03 -0700187
188 compressorPool sync.Pool
189 compLevel int
Colin Cross05518bc2018-09-27 15:06:19 -0700190
Colin Cross4be8f9e2018-09-28 15:16:48 -0700191 followSymlinks pathtools.ShouldFollowSymlinks
192 ignoreMissingFiles bool
Colin Cross09f11052018-09-21 15:12:39 -0700193
Colin Cross4be8f9e2018-09-28 15:16:48 -0700194 stderr io.Writer
195 fs pathtools.FileSystem
Dan Willemsen017d8932016-08-04 15:43:03 -0700196}
197
198type zipEntry struct {
199 fh *zip.FileHeader
200
201 // List of delayed io.Reader
202 futureReaders chan chan io.Reader
Jeff Gaston175f34c2017-08-17 21:43:21 -0700203
204 // Only used for passing into the MemoryRateLimiter to ensure we
205 // release as much memory as much as we request
206 allocatedSize int64
Colin Cross2fe66872015-03-30 17:20:39 -0700207}
208
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700209type ZipArgs struct {
Colin Crossfe945b42018-09-27 15:00:07 -0700210 FileArgs []FileArg
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700211 OutputFilePath string
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700212 EmulateJar bool
Colin Cross9cb51db2019-06-17 14:12:41 -0700213 SrcJar bool
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700214 AddDirectoryEntriesToZip bool
215 CompressionLevel int
216 ManifestSourcePath string
217 NumParallelJobs int
218 NonDeflatedFiles map[string]bool
Colin Crossf83c1502017-11-10 13:11:02 -0800219 WriteIfChanged bool
Colin Cross09f11052018-09-21 15:12:39 -0700220 StoreSymlinks bool
Colin Cross4be8f9e2018-09-28 15:16:48 -0700221 IgnoreMissingFiles bool
Colin Cross09f11052018-09-21 15:12:39 -0700222
Colin Cross4be8f9e2018-09-28 15:16:48 -0700223 Stderr io.Writer
Colin Cross09f11052018-09-21 15:12:39 -0700224 Filesystem pathtools.FileSystem
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700225}
Colin Cross2fe66872015-03-30 17:20:39 -0700226
Nan Zhang674dd932018-01-26 18:30:36 -0800227const NOQUOTE = '\x00'
228
229func ReadRespFile(bytes []byte) []string {
230 var args []string
231 var arg []rune
232
233 isEscaping := false
234 quotingStart := NOQUOTE
235 for _, c := range string(bytes) {
236 switch {
237 case isEscaping:
238 if quotingStart == '"' {
239 if !(c == '"' || c == '\\') {
240 // '\"' or '\\' will be escaped under double quoting.
241 arg = append(arg, '\\')
242 }
243 }
244 arg = append(arg, c)
245 isEscaping = false
246 case c == '\\' && quotingStart != '\'':
247 isEscaping = true
248 case quotingStart == NOQUOTE && (c == '\'' || c == '"'):
249 quotingStart = c
250 case quotingStart != NOQUOTE && c == quotingStart:
251 quotingStart = NOQUOTE
252 case quotingStart == NOQUOTE && unicode.IsSpace(c):
253 // Current character is a space outside quotes
254 if len(arg) != 0 {
255 args = append(args, string(arg))
256 }
257 arg = arg[:0]
258 default:
259 arg = append(arg, c)
260 }
261 }
262
263 if len(arg) != 0 {
264 args = append(args, string(arg))
265 }
266
267 return args
268}
269
Colin Cross05518bc2018-09-27 15:06:19 -0700270func ZipTo(args ZipArgs, w io.Writer) error {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700271 if args.EmulateJar {
272 args.AddDirectoryEntriesToZip = true
Jeff Gaston8edbb3a2017-08-22 20:05:28 -0700273 }
274
Colin Cross09f11052018-09-21 15:12:39 -0700275 // Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
276 followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
277
Colin Cross05518bc2018-09-27 15:06:19 -0700278 z := &ZipWriter{
Colin Cross4be8f9e2018-09-28 15:16:48 -0700279 time: jar.DefaultTime,
280 createdDirs: make(map[string]string),
281 createdFiles: make(map[string]string),
282 directories: args.AddDirectoryEntriesToZip,
283 compLevel: args.CompressionLevel,
284 followSymlinks: followSymlinks,
285 ignoreMissingFiles: args.IgnoreMissingFiles,
286 stderr: args.Stderr,
287 fs: args.Filesystem,
Colin Cross2fe66872015-03-30 17:20:39 -0700288 }
Colin Cross05518bc2018-09-27 15:06:19 -0700289
290 if z.fs == nil {
291 z.fs = pathtools.OsFs
292 }
293
Colin Cross4be8f9e2018-09-28 15:16:48 -0700294 if z.stderr == nil {
295 z.stderr = os.Stderr
296 }
297
Nan Zhang9067b042017-03-17 14:04:43 -0700298 pathMappings := []pathMapping{}
Nan Zhang9067b042017-03-17 14:04:43 -0700299
Colin Crossd3216292018-09-14 15:06:31 -0700300 noCompression := args.CompressionLevel == 0
301
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700302 for _, fa := range args.FileArgs {
Colin Cross1d98ee22018-09-18 17:05:15 -0700303 var srcs []string
304 for _, s := range fa.SourceFiles {
305 s = strings.TrimSpace(s)
306 if s == "" {
307 continue
308 }
309
Colin Cross09f11052018-09-21 15:12:39 -0700310 globbed, _, err := z.fs.Glob(s, nil, followSymlinks)
Colin Cross1d98ee22018-09-18 17:05:15 -0700311 if err != nil {
312 return err
313 }
314 if len(globbed) == 0 {
Colin Cross4be8f9e2018-09-28 15:16:48 -0700315 err := &os.PathError{
316 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700317 Path: s,
318 Err: os.ErrNotExist,
319 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700320 if args.IgnoreMissingFiles {
Dan Willemsenedc934c2018-12-27 12:41:25 -0800321 fmt.Fprintln(z.stderr, "warning:", err)
Colin Cross4be8f9e2018-09-28 15:16:48 -0700322 } else {
323 return err
324 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700325 }
326 srcs = append(srcs, globbed...)
327 }
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700328 if fa.GlobDir != "" {
Colin Cross1d98ee22018-09-18 17:05:15 -0700329 if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
330 return err
Colin Cross4be8f9e2018-09-28 15:16:48 -0700331 } else if !exists && !args.IgnoreMissingFiles {
332 err := &os.PathError{
333 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700334 Path: fa.GlobDir,
335 Err: os.ErrNotExist,
336 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700337 if args.IgnoreMissingFiles {
Dan Willemsenedc934c2018-12-27 12:41:25 -0800338 fmt.Fprintln(z.stderr, "warning:", err)
Colin Cross4be8f9e2018-09-28 15:16:48 -0700339 } else {
340 return err
341 }
342 } else if !isDir && !args.IgnoreMissingFiles {
343 err := &os.PathError{
344 Op: "lstat",
Colin Cross1d98ee22018-09-18 17:05:15 -0700345 Path: fa.GlobDir,
346 Err: syscall.ENOTDIR,
347 }
Colin Cross4be8f9e2018-09-28 15:16:48 -0700348 if args.IgnoreMissingFiles {
Dan Willemsenedc934c2018-12-27 12:41:25 -0800349 fmt.Fprintln(z.stderr, "warning:", err)
Colin Cross4be8f9e2018-09-28 15:16:48 -0700350 } else {
351 return err
352 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700353 }
Colin Cross09f11052018-09-21 15:12:39 -0700354 globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
Colin Cross1d98ee22018-09-18 17:05:15 -0700355 if err != nil {
356 return err
357 }
358 srcs = append(srcs, globbed...)
Colin Cross7b10cf12017-08-30 14:12:21 -0700359 }
360 for _, src := range srcs {
Colin Crossb7c69112018-09-18 16:51:43 -0700361 err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
Colin Crossd3216292018-09-14 15:06:31 -0700362 if err != nil {
Colin Cross05518bc2018-09-27 15:06:19 -0700363 return err
Nan Zhang9067b042017-03-17 14:04:43 -0700364 }
365 }
366 }
367
Colin Cross9cb51db2019-06-17 14:12:41 -0700368 return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
Colin Cross05518bc2018-09-27 15:06:19 -0700369}
370
371func Zip(args ZipArgs) error {
372 if args.OutputFilePath == "" {
373 return fmt.Errorf("output file path must be nonempty")
374 }
375
Colin Crossf83c1502017-11-10 13:11:02 -0800376 buf := &bytes.Buffer{}
377 var out io.Writer = buf
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700378
Colin Crossf83c1502017-11-10 13:11:02 -0800379 if !args.WriteIfChanged {
380 f, err := os.Create(args.OutputFilePath)
381 if err != nil {
382 return err
383 }
384
385 defer f.Close()
386 defer func() {
387 if err != nil {
388 os.Remove(args.OutputFilePath)
389 }
390 }()
391
392 out = f
393 }
394
Colin Cross05518bc2018-09-27 15:06:19 -0700395 err := ZipTo(args, out)
Colin Crossf83c1502017-11-10 13:11:02 -0800396 if err != nil {
397 return err
398 }
399
400 if args.WriteIfChanged {
401 err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
402 if err != nil {
403 return err
404 }
405 }
406
407 return nil
Colin Cross2fe66872015-03-30 17:20:39 -0700408}
409
Colin Crossb7c69112018-09-18 16:51:43 -0700410func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
Colin Crossd3216292018-09-14 15:06:31 -0700411 nonDeflatedFiles map[string]bool, noCompression bool) error {
412
Colin Crossb7c69112018-09-18 16:51:43 -0700413 var dest string
414
415 if fa.JunkPaths {
416 dest = filepath.Base(src)
417 } else {
418 var err error
419 dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
420 if err != nil {
421 return err
422 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700423 if strings.HasPrefix(dest, "../") {
424 return IncorrectRelativeRootError{
425 Path: src,
426 RelativeRoot: fa.SourcePrefixToStrip,
427 }
428 }
429
Nan Zhang9067b042017-03-17 14:04:43 -0700430 }
Colin Crossb7c69112018-09-18 16:51:43 -0700431 dest = filepath.Join(fa.PathPrefixInZip, dest)
Nan Zhang9067b042017-03-17 14:04:43 -0700432
Nan Zhangf281bd82017-04-25 16:47:45 -0700433 zipMethod := zip.Deflate
Colin Crossd3216292018-09-14 15:06:31 -0700434 if _, found := nonDeflatedFiles[dest]; found || noCompression {
Nan Zhangf281bd82017-04-25 16:47:45 -0700435 zipMethod = zip.Store
436 }
437 *pathMappings = append(*pathMappings,
438 pathMapping{dest: dest, src: src, zipMethod: zipMethod})
Nan Zhang9067b042017-03-17 14:04:43 -0700439
440 return nil
441}
442
Jeff Gastona2976952017-08-22 17:51:25 -0700443func jarSort(mappings []pathMapping) {
444 less := func(i int, j int) (smaller bool) {
445 return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
446 }
447 sort.SliceStable(mappings, less)
448}
449
Colin Cross9cb51db2019-06-17 14:12:41 -0700450func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
451 parallelJobs int) error {
452
Dan Willemsen017d8932016-08-04 15:43:03 -0700453 z.errors = make(chan error)
454 defer close(z.errors)
Colin Cross2fe66872015-03-30 17:20:39 -0700455
Dan Willemsen017d8932016-08-04 15:43:03 -0700456 // This channel size can be essentially unlimited -- it's used as a fifo
457 // queue decouple the CPU and IO loads. Directories don't require any
458 // compression time, but still cost some IO. Similar with small files that
459 // can be very fast to compress. Some files that are more difficult to
460 // compress won't take a corresponding longer time writing out.
461 //
462 // The optimum size here depends on your CPU and IO characteristics, and
463 // the the layout of your zip file. 1000 was chosen mostly at random as
464 // something that worked reasonably well for a test file.
465 //
466 // The RateLimit object will put the upper bounds on the number of
467 // parallel compressions and outstanding buffers.
468 z.writeOps = make(chan chan *zipEntry, 1000)
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700469 z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
Jeff Gaston175f34c2017-08-17 21:43:21 -0700470 z.memoryRateLimiter = NewMemoryRateLimiter(0)
471 defer func() {
472 z.cpuRateLimiter.Stop()
473 z.memoryRateLimiter.Stop()
474 }()
Jeff Gastona2976952017-08-22 17:51:25 -0700475
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700476 if manifest != "" && !emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700477 return errors.New("must specify --jar when specifying a manifest via -m")
Jeff Gastona2976952017-08-22 17:51:25 -0700478 }
479
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700480 if emulateJar {
Colin Cross635acc92017-09-12 22:50:46 -0700481 // manifest may be empty, in which case addManifest will fill in a default
482 pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
483
Jeff Gastona2976952017-08-22 17:51:25 -0700484 jarSort(pathMappings)
485 }
486
Dan Willemsen017d8932016-08-04 15:43:03 -0700487 go func() {
488 var err error
489 defer close(z.writeOps)
490
Nan Zhang9067b042017-03-17 14:04:43 -0700491 for _, ele := range pathMappings {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700492 if emulateJar && ele.dest == jar.ManifestFile {
Jeff Gastoncef50b92017-08-23 15:41:35 -0700493 err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
494 } else {
Colin Cross9cb51db2019-06-17 14:12:41 -0700495 err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
Jeff Gastoncef50b92017-08-23 15:41:35 -0700496 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700497 if err != nil {
498 z.errors <- err
499 return
500 }
501 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700502 }()
503
504 zipw := zip.NewWriter(f)
505
506 var currentWriteOpChan chan *zipEntry
507 var currentWriter io.WriteCloser
508 var currentReaders chan chan io.Reader
509 var currentReader chan io.Reader
510 var done bool
511
512 for !done {
513 var writeOpsChan chan chan *zipEntry
514 var writeOpChan chan *zipEntry
515 var readersChan chan chan io.Reader
516
517 if currentReader != nil {
518 // Only read and process errors
519 } else if currentReaders != nil {
520 readersChan = currentReaders
521 } else if currentWriteOpChan != nil {
522 writeOpChan = currentWriteOpChan
523 } else {
524 writeOpsChan = z.writeOps
525 }
526
527 select {
528 case writeOp, ok := <-writeOpsChan:
529 if !ok {
530 done = true
531 }
532
533 currentWriteOpChan = writeOp
534
535 case op := <-writeOpChan:
536 currentWriteOpChan = nil
537
Colin Crossf83c1502017-11-10 13:11:02 -0800538 var err error
Dan Willemsen017d8932016-08-04 15:43:03 -0700539 if op.fh.Method == zip.Deflate {
540 currentWriter, err = zipw.CreateCompressedHeader(op.fh)
541 } else {
542 var zw io.Writer
Jeff Gastonc5eb66d2017-08-24 14:11:27 -0700543
544 op.fh.CompressedSize64 = op.fh.UncompressedSize64
545
546 zw, err = zipw.CreateHeaderAndroid(op.fh)
Dan Willemsen017d8932016-08-04 15:43:03 -0700547 currentWriter = nopCloser{zw}
548 }
549 if err != nil {
550 return err
551 }
552
553 currentReaders = op.futureReaders
554 if op.futureReaders == nil {
555 currentWriter.Close()
556 currentWriter = nil
557 }
Jeff Gaston175f34c2017-08-17 21:43:21 -0700558 z.memoryRateLimiter.Finish(op.allocatedSize)
Dan Willemsen017d8932016-08-04 15:43:03 -0700559
560 case futureReader, ok := <-readersChan:
561 if !ok {
562 // Done with reading
563 currentWriter.Close()
564 currentWriter = nil
565 currentReaders = nil
566 }
567
568 currentReader = futureReader
569
570 case reader := <-currentReader:
Colin Crossf83c1502017-11-10 13:11:02 -0800571 _, err := io.Copy(currentWriter, reader)
Dan Willemsen017d8932016-08-04 15:43:03 -0700572 if err != nil {
573 return err
574 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700575
576 currentReader = nil
577
Colin Crossf83c1502017-11-10 13:11:02 -0800578 case err := <-z.errors:
Colin Cross2fe66872015-03-30 17:20:39 -0700579 return err
580 }
581 }
582
Dan Willemsen017d8932016-08-04 15:43:03 -0700583 // One last chance to catch an error
584 select {
Colin Crossf83c1502017-11-10 13:11:02 -0800585 case err := <-z.errors:
Dan Willemsen017d8932016-08-04 15:43:03 -0700586 return err
587 default:
588 zipw.Close()
589 return nil
Colin Cross2fe66872015-03-30 17:20:39 -0700590 }
Colin Cross2fe66872015-03-30 17:20:39 -0700591}
592
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700593// imports (possibly with compression) <src> into the zip at sub-path <dest>
Colin Cross9cb51db2019-06-17 14:12:41 -0700594func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
Dan Willemsen017d8932016-08-04 15:43:03 -0700595 var fileSize int64
Dan Willemsen10462b32017-03-15 19:02:51 -0700596 var executable bool
Dan Willemsen017d8932016-08-04 15:43:03 -0700597
Colin Cross09f11052018-09-21 15:12:39 -0700598 var s os.FileInfo
599 var err error
600 if z.followSymlinks {
601 s, err = z.fs.Stat(src)
602 } else {
603 s, err = z.fs.Lstat(src)
604 }
605
606 if err != nil {
Colin Cross4be8f9e2018-09-28 15:16:48 -0700607 if os.IsNotExist(err) && z.ignoreMissingFiles {
608 fmt.Fprintln(z.stderr, "warning:", err)
609 return nil
610 }
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700611 return err
Colin Cross9cb51db2019-06-17 14:12:41 -0700612 }
613
614 createParentDirs := func(dest, src string) error {
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700615 if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
Colin Crosse5580972017-08-30 17:40:21 -0700616 return err
617 }
618
619 if prev, exists := z.createdDirs[dest]; exists {
620 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
621 }
622 if prev, exists := z.createdFiles[dest]; exists {
623 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
624 }
625
626 z.createdFiles[dest] = src
627
Colin Cross9cb51db2019-06-17 14:12:41 -0700628 return nil
629 }
630
631 if s.IsDir() {
632 if z.directories {
633 return z.writeDirectory(dest, src, emulateJar)
634 }
635 return nil
636 } else if s.Mode()&os.ModeSymlink != 0 {
637 err = createParentDirs(dest, src)
638 if err != nil {
639 return err
640 }
641
642 return z.writeSymlink(dest, src)
643 } else if s.Mode().IsRegular() {
644 r, err := z.fs.Open(src)
645 if err != nil {
646 return err
647 }
648
649 if srcJar && filepath.Ext(src) == ".java" {
650 // rewrite the destination using the package path if it can be determined
651 pkg, err := jar.JavaPackage(r, src)
652 if err != nil {
653 // ignore errors for now, leaving the file at in its original location in the zip
654 } else {
655 dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
656 }
657
658 _, err = r.Seek(0, io.SeekStart)
659 if err != nil {
660 return err
661 }
Colin Crosse5580972017-08-30 17:40:21 -0700662 }
663
Dan Willemsen017d8932016-08-04 15:43:03 -0700664 fileSize = s.Size()
Dan Willemsen10462b32017-03-15 19:02:51 -0700665 executable = s.Mode()&0100 != 0
Colin Cross957cc4e2015-04-24 15:10:32 -0700666
Colin Cross9cb51db2019-06-17 14:12:41 -0700667 header := &zip.FileHeader{
668 Name: dest,
669 Method: method,
670 UncompressedSize64: uint64(fileSize),
671 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700672
Colin Cross9cb51db2019-06-17 14:12:41 -0700673 if executable {
674 header.SetMode(0700)
675 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700676
Colin Cross9cb51db2019-06-17 14:12:41 -0700677 err = createParentDirs(dest, src)
678 if err != nil {
679 return err
680 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700681
Colin Cross9cb51db2019-06-17 14:12:41 -0700682 return z.writeFileContents(header, r)
683 } else {
684 return fmt.Errorf("%s is not a file, directory, or symlink", src)
685 }
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700686}
687
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700688func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
Colin Crosse5580972017-08-30 17:40:21 -0700689 if prev, exists := z.createdDirs[dest]; exists {
690 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
691 }
692 if prev, exists := z.createdFiles[dest]; exists {
693 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
694 }
695
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700696 if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
Colin Cross635acc92017-09-12 22:50:46 -0700697 return err
Jeff Gastoncef50b92017-08-23 15:41:35 -0700698 }
699
Colin Cross05518bc2018-09-27 15:06:19 -0700700 var contents []byte
701 if src != "" {
702 f, err := z.fs.Open(src)
703 if err != nil {
704 return err
705 }
706
707 contents, err = ioutil.ReadAll(f)
708 f.Close()
709 if err != nil {
710 return err
711 }
712 }
713
714 fh, buf, err := jar.ManifestFileContents(contents)
Colin Cross635acc92017-09-12 22:50:46 -0700715 if err != nil {
716 return err
Jeff Gastoncef50b92017-08-23 15:41:35 -0700717 }
718
Colin Cross635acc92017-09-12 22:50:46 -0700719 reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
720
721 return z.writeFileContents(fh, reader)
Jeff Gastoncef50b92017-08-23 15:41:35 -0700722}
723
Colin Cross05518bc2018-09-27 15:06:19 -0700724func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700725
726 header.SetModTime(z.time)
727
Dan Willemsen017d8932016-08-04 15:43:03 -0700728 compressChan := make(chan *zipEntry, 1)
729 z.writeOps <- compressChan
730
731 // Pre-fill a zipEntry, it will be sent in the compressChan once
732 // we're sure about the Method and CRC.
733 ze := &zipEntry{
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700734 fh: header,
Dan Willemsen10462b32017-03-15 19:02:51 -0700735 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700736
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700737 ze.allocatedSize = int64(header.UncompressedSize64)
Jeff Gaston175f34c2017-08-17 21:43:21 -0700738 z.cpuRateLimiter.Request()
739 z.memoryRateLimiter.Request(ze.allocatedSize)
Dan Willemsen017d8932016-08-04 15:43:03 -0700740
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700741 fileSize := int64(header.UncompressedSize64)
742 if fileSize == 0 {
743 fileSize = int64(header.UncompressedSize)
744 }
745
746 if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
Dan Willemsen017d8932016-08-04 15:43:03 -0700747 wg := new(sync.WaitGroup)
748
749 // Allocate enough buffer to hold all readers. We'll limit
750 // this based on actual buffer sizes in RateLimit.
751 ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
752
753 // Calculate the CRC in the background, since reading the entire
754 // file could take a while.
755 //
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700756 // We could split this up into chunks as well, but it's faster
Dan Willemsen017d8932016-08-04 15:43:03 -0700757 // than the compression. Due to the Go Zip API, we also need to
758 // know the result before we can begin writing the compressed
759 // data out to the zipfile.
760 wg.Add(1)
Jeff Gaston175f34c2017-08-17 21:43:21 -0700761 go z.crcFile(r, ze, compressChan, wg)
Dan Willemsen017d8932016-08-04 15:43:03 -0700762
763 for start := int64(0); start < fileSize; start += parallelBlockSize {
764 sr := io.NewSectionReader(r, start, parallelBlockSize)
765 resultChan := make(chan io.Reader, 1)
766 ze.futureReaders <- resultChan
767
Jeff Gaston175f34c2017-08-17 21:43:21 -0700768 z.cpuRateLimiter.Request()
Dan Willemsen017d8932016-08-04 15:43:03 -0700769
770 last := !(start+parallelBlockSize < fileSize)
771 var dict []byte
772 if start >= windowSize {
773 dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700774 if err != nil {
775 return err
776 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700777 }
778
779 wg.Add(1)
Jeff Gaston175f34c2017-08-17 21:43:21 -0700780 go z.compressPartialFile(sr, dict, last, resultChan, wg)
Dan Willemsen017d8932016-08-04 15:43:03 -0700781 }
782
783 close(ze.futureReaders)
784
785 // Close the file handle after all readers are done
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700786 go func(wg *sync.WaitGroup, closer io.Closer) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700787 wg.Wait()
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700788 closer.Close()
Dan Willemsen017d8932016-08-04 15:43:03 -0700789 }(wg, r)
790 } else {
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700791 go func() {
792 z.compressWholeFile(ze, r, compressChan)
793 r.Close()
794 }()
Dan Willemsen017d8932016-08-04 15:43:03 -0700795 }
796
797 return nil
798}
799
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700800func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700801 defer wg.Done()
Jeff Gaston175f34c2017-08-17 21:43:21 -0700802 defer z.cpuRateLimiter.Finish()
Dan Willemsen017d8932016-08-04 15:43:03 -0700803
804 crc := crc32.NewIEEE()
805 _, err := io.Copy(crc, r)
806 if err != nil {
807 z.errors <- err
808 return
809 }
810
811 ze.fh.CRC32 = crc.Sum32()
812 resultChan <- ze
813 close(resultChan)
814}
815
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700816func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700817 defer wg.Done()
818
819 result, err := z.compressBlock(r, dict, last)
820 if err != nil {
821 z.errors <- err
822 return
823 }
824
Jeff Gaston175f34c2017-08-17 21:43:21 -0700825 z.cpuRateLimiter.Finish()
826
Dan Willemsen017d8932016-08-04 15:43:03 -0700827 resultChan <- result
828}
829
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700830func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700831 buf := new(bytes.Buffer)
832 var fw *flate.Writer
833 var err error
834 if len(dict) > 0 {
835 // There's no way to Reset a Writer with a new dictionary, so
836 // don't use the Pool
837 fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
838 } else {
839 var ok bool
840 if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
841 fw.Reset(buf)
842 } else {
843 fw, err = flate.NewWriter(buf, z.compLevel)
844 }
845 defer z.compressorPool.Put(fw)
846 }
847 if err != nil {
848 return nil, err
849 }
850
851 _, err = io.Copy(fw, r)
852 if err != nil {
853 return nil, err
854 }
855 if last {
856 fw.Close()
857 } else {
858 fw.Flush()
859 }
860
861 return buf, nil
862}
863
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700864func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
Dan Willemsen017d8932016-08-04 15:43:03 -0700865
Dan Willemsen017d8932016-08-04 15:43:03 -0700866 crc := crc32.NewIEEE()
Dan Willemsena8b55022017-03-15 21:49:26 -0700867 _, err := io.Copy(crc, r)
Colin Cross2fe66872015-03-30 17:20:39 -0700868 if err != nil {
Dan Willemsen017d8932016-08-04 15:43:03 -0700869 z.errors <- err
870 return
Colin Cross2fe66872015-03-30 17:20:39 -0700871 }
872
Dan Willemsena8b55022017-03-15 21:49:26 -0700873 ze.fh.CRC32 = crc.Sum32()
Colin Cross2fe66872015-03-30 17:20:39 -0700874
Dan Willemsen017d8932016-08-04 15:43:03 -0700875 _, err = r.Seek(0, 0)
Colin Cross2fe66872015-03-30 17:20:39 -0700876 if err != nil {
Dan Willemsen017d8932016-08-04 15:43:03 -0700877 z.errors <- err
878 return
Colin Cross2fe66872015-03-30 17:20:39 -0700879 }
880
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700881 readFile := func(reader io.ReadSeeker) ([]byte, error) {
882 _, err := reader.Seek(0, 0)
Nan Zhangf281bd82017-04-25 16:47:45 -0700883 if err != nil {
884 return nil, err
885 }
886
Jeff Gaston66dd6e52017-08-23 15:12:48 -0700887 buf, err := ioutil.ReadAll(reader)
Nan Zhangf281bd82017-04-25 16:47:45 -0700888 if err != nil {
889 return nil, err
890 }
891
892 return buf, nil
893 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700894
Dan Willemsena8b55022017-03-15 21:49:26 -0700895 ze.futureReaders = make(chan chan io.Reader, 1)
Dan Willemsen017d8932016-08-04 15:43:03 -0700896 futureReader := make(chan io.Reader, 1)
897 ze.futureReaders <- futureReader
898 close(ze.futureReaders)
899
Nan Zhangf281bd82017-04-25 16:47:45 -0700900 if ze.fh.Method == zip.Deflate {
901 compressed, err := z.compressBlock(r, nil, true)
902 if err != nil {
903 z.errors <- err
904 return
905 }
906 if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
907 futureReader <- compressed
Nan Zhangf281bd82017-04-25 16:47:45 -0700908 } else {
909 buf, err := readFile(r)
910 if err != nil {
911 z.errors <- err
912 return
913 }
914 ze.fh.Method = zip.Store
915 futureReader <- bytes.NewReader(buf)
Nan Zhangf281bd82017-04-25 16:47:45 -0700916 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700917 } else {
Nan Zhangf281bd82017-04-25 16:47:45 -0700918 buf, err := readFile(r)
Dan Willemsen017d8932016-08-04 15:43:03 -0700919 if err != nil {
920 z.errors <- err
921 return
922 }
Dan Willemsen017d8932016-08-04 15:43:03 -0700923 ze.fh.Method = zip.Store
924 futureReader <- bytes.NewReader(buf)
Dan Willemsen017d8932016-08-04 15:43:03 -0700925 }
Nan Zhangf281bd82017-04-25 16:47:45 -0700926
Jeff Gaston175f34c2017-08-17 21:43:21 -0700927 z.cpuRateLimiter.Finish()
928
Dan Willemsen017d8932016-08-04 15:43:03 -0700929 close(futureReader)
930
931 compressChan <- ze
932 close(compressChan)
Colin Cross2fe66872015-03-30 17:20:39 -0700933}
Colin Crosse19c7932015-04-24 15:08:38 -0700934
Colin Crosse5580972017-08-30 17:40:21 -0700935// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
936// the directory entry to the zip file if directories are enabled.
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700937func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
Jeff Gaston2d174132017-08-15 18:05:56 -0700938 // clean the input
Colin Crosse5580972017-08-30 17:40:21 -0700939 dir = filepath.Clean(dir)
Jeff Gaston2d174132017-08-15 18:05:56 -0700940
941 // discover any uncreated directories in the path
942 zipDirs := []string{}
Colin Crosse5580972017-08-30 17:40:21 -0700943 for dir != "" && dir != "." {
944 if _, exists := z.createdDirs[dir]; exists {
945 break
946 }
Jeff Gaston2d174132017-08-15 18:05:56 -0700947
Colin Crosse5580972017-08-30 17:40:21 -0700948 if prev, exists := z.createdFiles[dir]; exists {
949 return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
950 }
951
952 z.createdDirs[dir] = src
Jeff Gaston2d174132017-08-15 18:05:56 -0700953 // parent directories precede their children
Colin Crosse5580972017-08-30 17:40:21 -0700954 zipDirs = append([]string{dir}, zipDirs...)
Jeff Gaston2d174132017-08-15 18:05:56 -0700955
Colin Crosse5580972017-08-30 17:40:21 -0700956 dir = filepath.Dir(dir)
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700957 }
958
Colin Crosse5580972017-08-30 17:40:21 -0700959 if z.directories {
960 // make a directory entry for each uncreated directory
961 for _, cleanDir := range zipDirs {
Colin Cross635acc92017-09-12 22:50:46 -0700962 var dirHeader *zip.FileHeader
Colin Crosse19c7932015-04-24 15:08:38 -0700963
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700964 if emulateJar && cleanDir+"/" == jar.MetaDir {
Colin Cross635acc92017-09-12 22:50:46 -0700965 dirHeader = jar.MetaDirFileHeader()
966 } else {
967 dirHeader = &zip.FileHeader{
968 Name: cleanDir + "/",
969 }
970 dirHeader.SetMode(0700 | os.ModeDir)
Colin Crosse5580972017-08-30 17:40:21 -0700971 }
Jeff Gaston8edbb3a2017-08-22 20:05:28 -0700972
Colin Cross635acc92017-09-12 22:50:46 -0700973 dirHeader.SetModTime(z.time)
974
Colin Crosse5580972017-08-30 17:40:21 -0700975 ze := make(chan *zipEntry, 1)
976 ze <- &zipEntry{
977 fh: dirHeader,
978 }
979 close(ze)
980 z.writeOps <- ze
Colin Crosse19c7932015-04-24 15:08:38 -0700981 }
Colin Crosse19c7932015-04-24 15:08:38 -0700982 }
983
984 return nil
985}
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700986
Jeff Gastonc3bdc972017-10-12 12:18:19 -0700987func (z *ZipWriter) writeSymlink(rel, file string) error {
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700988 fileHeader := &zip.FileHeader{
989 Name: rel,
990 }
991 fileHeader.SetModTime(z.time)
Colin Cross297d9bc2018-06-22 16:37:47 -0700992 fileHeader.SetMode(0777 | os.ModeSymlink)
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700993
Colin Cross05518bc2018-09-27 15:06:19 -0700994 dest, err := z.fs.Readlink(file)
Dan Willemsena59a3bc2016-08-03 17:47:23 -0700995 if err != nil {
996 return err
997 }
998
Colin Cross297d9bc2018-06-22 16:37:47 -0700999 fileHeader.UncompressedSize64 = uint64(len(dest))
1000 fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
1001
Dan Willemsen017d8932016-08-04 15:43:03 -07001002 ze := make(chan *zipEntry, 1)
1003 futureReaders := make(chan chan io.Reader, 1)
1004 futureReader := make(chan io.Reader, 1)
1005 futureReaders <- futureReader
1006 close(futureReaders)
1007 futureReader <- bytes.NewBufferString(dest)
1008 close(futureReader)
1009
Dan Willemsen017d8932016-08-04 15:43:03 -07001010 ze <- &zipEntry{
1011 fh: fileHeader,
1012 futureReaders: futureReaders,
1013 }
1014 close(ze)
1015 z.writeOps <- ze
1016
1017 return nil
Dan Willemsena59a3bc2016-08-03 17:47:23 -07001018}