blob: 9e1ef90fff0ec47ff8c615e9f9a8bf7190601b4c [file] [log] [blame]
Nan Zhang674dd932018-01-26 18:30:36 -08001// Copyright 2018 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package zip
16
17import (
Colin Cross05518bc2018-09-27 15:06:19 -070018 "bytes"
19 "hash/crc32"
20 "io"
21 "os"
Nan Zhang674dd932018-01-26 18:30:36 -080022 "reflect"
Colin Cross05518bc2018-09-27 15:06:19 -070023 "syscall"
Nan Zhang674dd932018-01-26 18:30:36 -080024 "testing"
Colin Cross05518bc2018-09-27 15:06:19 -070025
26 "android/soong/third_party/zip"
27
28 "github.com/google/blueprint/pathtools"
Nan Zhang674dd932018-01-26 18:30:36 -080029)
30
Colin Cross05518bc2018-09-27 15:06:19 -070031var (
32 fileA = []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
33 fileB = []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")
34 fileC = []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")
35 fileEmpty = []byte("")
36 fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n")
37
38 fileCustomManifest = []byte("Custom manifest: true\n")
39 customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n")
40)
41
42var mockFs = pathtools.MockFs(map[string][]byte{
Colin Cross9cb51db2019-06-17 14:12:41 -070043 "a/a/a": fileA,
44 "a/a/b": fileB,
45 "a/a/c -> ../../c": nil,
46 "dangling -> missing": nil,
47 "a/a/d -> b": nil,
48 "c": fileC,
Jiyong Park04bbf982019-11-04 13:18:41 +090049 "l_nl": []byte("a/a/a\na/a/b\nc\n"),
50 "l_sp": []byte("a/a/a a/a/b c"),
Colin Cross9cb51db2019-06-17 14:12:41 -070051 "l2": []byte("missing\n"),
Colin Cross053fca12020-08-19 13:51:47 -070052 "rsp": []byte("'a/a/a'\na/a/b\n'@'\n'foo'\\''bar'"),
53 "@ -> c": nil,
54 "foo'bar -> c": nil,
Colin Cross9cb51db2019-06-17 14:12:41 -070055 "manifest.txt": fileCustomManifest,
Colin Cross05518bc2018-09-27 15:06:19 -070056})
57
58func fh(name string, contents []byte, method uint16) zip.FileHeader {
59 return zip.FileHeader{
60 Name: name,
61 Method: method,
62 CRC32: crc32.ChecksumIEEE(contents),
63 UncompressedSize64: uint64(len(contents)),
64 ExternalAttrs: 0,
65 }
66}
67
68func fhManifest(contents []byte) zip.FileHeader {
69 return zip.FileHeader{
70 Name: "META-INF/MANIFEST.MF",
71 Method: zip.Store,
72 CRC32: crc32.ChecksumIEEE(contents),
73 UncompressedSize64: uint64(len(contents)),
74 ExternalAttrs: (syscall.S_IFREG | 0700) << 16,
75 }
76}
77
78func fhLink(name string, to string) zip.FileHeader {
79 return zip.FileHeader{
80 Name: name,
81 Method: zip.Store,
82 CRC32: crc32.ChecksumIEEE([]byte(to)),
83 UncompressedSize64: uint64(len(to)),
84 ExternalAttrs: (syscall.S_IFLNK | 0777) << 16,
85 }
86}
87
88func fhDir(name string) zip.FileHeader {
89 return zip.FileHeader{
90 Name: name,
91 Method: zip.Store,
92 CRC32: crc32.ChecksumIEEE(nil),
93 UncompressedSize64: 0,
94 ExternalAttrs: (syscall.S_IFDIR|0700)<<16 | 0x10,
95 }
96}
97
98func fileArgsBuilder() *FileArgsBuilder {
99 return &FileArgsBuilder{
100 fs: mockFs,
101 }
102}
103
104func TestZip(t *testing.T) {
Colin Cross323dc602020-09-18 14:25:31 -0700105 t.Parallel()
Colin Cross05518bc2018-09-27 15:06:19 -0700106 testCases := []struct {
Colin Cross4be8f9e2018-09-28 15:16:48 -0700107 name string
108 args *FileArgsBuilder
109 compressionLevel int
110 emulateJar bool
111 nonDeflatedFiles map[string]bool
112 dirEntries bool
113 manifest string
114 storeSymlinks bool
115 ignoreMissingFiles bool
Colin Cross05518bc2018-09-27 15:06:19 -0700116
117 files []zip.FileHeader
118 err error
119 }{
120 {
121 name: "empty args",
122 args: fileArgsBuilder(),
123
124 files: []zip.FileHeader{},
125 },
126 {
127 name: "files",
128 args: fileArgsBuilder().
129 File("a/a/a").
130 File("a/a/b").
131 File("c"),
132 compressionLevel: 9,
133
134 files: []zip.FileHeader{
135 fh("a/a/a", fileA, zip.Deflate),
136 fh("a/a/b", fileB, zip.Deflate),
137 fh("c", fileC, zip.Deflate),
138 },
139 },
140 {
Colin Cross1d98ee22018-09-18 17:05:15 -0700141 name: "files glob",
142 args: fileArgsBuilder().
143 SourcePrefixToStrip("a").
144 File("a/**/*"),
145 compressionLevel: 9,
Colin Cross09f11052018-09-21 15:12:39 -0700146 storeSymlinks: true,
Colin Cross1d98ee22018-09-18 17:05:15 -0700147
148 files: []zip.FileHeader{
149 fh("a/a", fileA, zip.Deflate),
150 fh("a/b", fileB, zip.Deflate),
151 fhLink("a/c", "../../c"),
152 fhLink("a/d", "b"),
153 },
154 },
155 {
156 name: "dir",
157 args: fileArgsBuilder().
158 SourcePrefixToStrip("a").
159 Dir("a"),
160 compressionLevel: 9,
Colin Cross09f11052018-09-21 15:12:39 -0700161 storeSymlinks: true,
Colin Cross1d98ee22018-09-18 17:05:15 -0700162
163 files: []zip.FileHeader{
164 fh("a/a", fileA, zip.Deflate),
165 fh("a/b", fileB, zip.Deflate),
166 fhLink("a/c", "../../c"),
167 fhLink("a/d", "b"),
168 },
169 },
170 {
Colin Cross05518bc2018-09-27 15:06:19 -0700171 name: "stored files",
172 args: fileArgsBuilder().
173 File("a/a/a").
174 File("a/a/b").
175 File("c"),
176 compressionLevel: 0,
177
178 files: []zip.FileHeader{
179 fh("a/a/a", fileA, zip.Store),
180 fh("a/a/b", fileB, zip.Store),
181 fh("c", fileC, zip.Store),
182 },
183 },
184 {
185 name: "symlinks in zip",
186 args: fileArgsBuilder().
187 File("a/a/a").
188 File("a/a/b").
189 File("a/a/c").
190 File("a/a/d"),
191 compressionLevel: 9,
Colin Cross09f11052018-09-21 15:12:39 -0700192 storeSymlinks: true,
Colin Cross05518bc2018-09-27 15:06:19 -0700193
194 files: []zip.FileHeader{
195 fh("a/a/a", fileA, zip.Deflate),
196 fh("a/a/b", fileB, zip.Deflate),
197 fhLink("a/a/c", "../../c"),
198 fhLink("a/a/d", "b"),
199 },
200 },
201 {
Colin Cross09f11052018-09-21 15:12:39 -0700202 name: "follow symlinks",
203 args: fileArgsBuilder().
204 File("a/a/a").
205 File("a/a/b").
206 File("a/a/c").
207 File("a/a/d"),
208 compressionLevel: 9,
209 storeSymlinks: false,
210
211 files: []zip.FileHeader{
212 fh("a/a/a", fileA, zip.Deflate),
213 fh("a/a/b", fileB, zip.Deflate),
214 fh("a/a/c", fileC, zip.Deflate),
215 fh("a/a/d", fileB, zip.Deflate),
216 },
217 },
218 {
Colin Cross9cb51db2019-06-17 14:12:41 -0700219 name: "dangling symlinks",
220 args: fileArgsBuilder().
221 File("dangling"),
222 compressionLevel: 9,
223 storeSymlinks: true,
224
225 files: []zip.FileHeader{
226 fhLink("dangling", "missing"),
227 },
228 },
229 {
Colin Cross05518bc2018-09-27 15:06:19 -0700230 name: "list",
231 args: fileArgsBuilder().
Jiyong Park04bbf982019-11-04 13:18:41 +0900232 List("l_nl"),
233 compressionLevel: 9,
234
235 files: []zip.FileHeader{
236 fh("a/a/a", fileA, zip.Deflate),
237 fh("a/a/b", fileB, zip.Deflate),
238 fh("c", fileC, zip.Deflate),
239 },
240 },
241 {
242 name: "list",
243 args: fileArgsBuilder().
244 List("l_sp"),
Colin Cross05518bc2018-09-27 15:06:19 -0700245 compressionLevel: 9,
246
247 files: []zip.FileHeader{
248 fh("a/a/a", fileA, zip.Deflate),
249 fh("a/a/b", fileB, zip.Deflate),
250 fh("c", fileC, zip.Deflate),
251 },
252 },
253 {
Colin Cross053fca12020-08-19 13:51:47 -0700254 name: "rsp",
255 args: fileArgsBuilder().
256 RspFile("rsp"),
257 compressionLevel: 9,
258
259 files: []zip.FileHeader{
260 fh("a/a/a", fileA, zip.Deflate),
261 fh("a/a/b", fileB, zip.Deflate),
262 fh("@", fileC, zip.Deflate),
263 fh("foo'bar", fileC, zip.Deflate),
264 },
265 },
266 {
Colin Cross05518bc2018-09-27 15:06:19 -0700267 name: "prefix in zip",
268 args: fileArgsBuilder().
269 PathPrefixInZip("foo").
270 File("a/a/a").
271 File("a/a/b").
272 File("c"),
273 compressionLevel: 9,
274
275 files: []zip.FileHeader{
276 fh("foo/a/a/a", fileA, zip.Deflate),
277 fh("foo/a/a/b", fileB, zip.Deflate),
278 fh("foo/c", fileC, zip.Deflate),
279 },
280 },
281 {
282 name: "relative root",
283 args: fileArgsBuilder().
284 SourcePrefixToStrip("a").
285 File("a/a/a").
286 File("a/a/b"),
287 compressionLevel: 9,
288
289 files: []zip.FileHeader{
290 fh("a/a", fileA, zip.Deflate),
291 fh("a/b", fileB, zip.Deflate),
292 },
293 },
294 {
295 name: "multiple relative root",
296 args: fileArgsBuilder().
297 SourcePrefixToStrip("a").
298 File("a/a/a").
299 SourcePrefixToStrip("a/a").
300 File("a/a/b"),
301 compressionLevel: 9,
302
303 files: []zip.FileHeader{
304 fh("a/a", fileA, zip.Deflate),
305 fh("b", fileB, zip.Deflate),
306 },
307 },
308 {
309 name: "emulate jar",
310 args: fileArgsBuilder().
311 File("a/a/a").
312 File("a/a/b"),
313 compressionLevel: 9,
314 emulateJar: true,
315
316 files: []zip.FileHeader{
317 fhDir("META-INF/"),
318 fhManifest(fileManifest),
319 fhDir("a/"),
320 fhDir("a/a/"),
321 fh("a/a/a", fileA, zip.Deflate),
322 fh("a/a/b", fileB, zip.Deflate),
323 },
324 },
325 {
326 name: "emulate jar with manifest",
327 args: fileArgsBuilder().
328 File("a/a/a").
329 File("a/a/b"),
330 compressionLevel: 9,
331 emulateJar: true,
332 manifest: "manifest.txt",
333
334 files: []zip.FileHeader{
335 fhDir("META-INF/"),
336 fhManifest(customManifestAfter),
337 fhDir("a/"),
338 fhDir("a/a/"),
339 fh("a/a/a", fileA, zip.Deflate),
340 fh("a/a/b", fileB, zip.Deflate),
341 },
342 },
343 {
344 name: "dir entries",
345 args: fileArgsBuilder().
346 File("a/a/a").
347 File("a/a/b"),
348 compressionLevel: 9,
349 dirEntries: true,
350
351 files: []zip.FileHeader{
352 fhDir("a/"),
353 fhDir("a/a/"),
354 fh("a/a/a", fileA, zip.Deflate),
355 fh("a/a/b", fileB, zip.Deflate),
356 },
357 },
358 {
359 name: "junk paths",
360 args: fileArgsBuilder().
361 JunkPaths(true).
362 File("a/a/a").
363 File("a/a/b"),
364 compressionLevel: 9,
365
366 files: []zip.FileHeader{
367 fh("a", fileA, zip.Deflate),
368 fh("b", fileB, zip.Deflate),
369 },
370 },
371 {
372 name: "non deflated files",
373 args: fileArgsBuilder().
374 File("a/a/a").
375 File("a/a/b"),
376 compressionLevel: 9,
377 nonDeflatedFiles: map[string]bool{"a/a/a": true},
378
379 files: []zip.FileHeader{
380 fh("a/a/a", fileA, zip.Store),
381 fh("a/a/b", fileB, zip.Deflate),
382 },
383 },
Colin Cross4be8f9e2018-09-28 15:16:48 -0700384 {
385 name: "ignore missing files",
386 args: fileArgsBuilder().
387 File("a/a/a").
388 File("a/a/b").
389 File("missing"),
390 compressionLevel: 9,
391 ignoreMissingFiles: true,
392
393 files: []zip.FileHeader{
394 fh("a/a/a", fileA, zip.Deflate),
395 fh("a/a/b", fileB, zip.Deflate),
396 },
397 },
Colin Cross05518bc2018-09-27 15:06:19 -0700398
399 // errors
400 {
401 name: "error missing file",
402 args: fileArgsBuilder().
403 File("missing"),
404 err: os.ErrNotExist,
405 },
406 {
Colin Cross1d98ee22018-09-18 17:05:15 -0700407 name: "error missing dir",
408 args: fileArgsBuilder().
409 Dir("missing"),
410 err: os.ErrNotExist,
411 },
412 {
Colin Cross05518bc2018-09-27 15:06:19 -0700413 name: "error missing file in list",
414 args: fileArgsBuilder().
415 List("l2"),
416 err: os.ErrNotExist,
417 },
Colin Cross1d98ee22018-09-18 17:05:15 -0700418 {
419 name: "error incorrect relative root",
420 args: fileArgsBuilder().
421 SourcePrefixToStrip("b").
422 File("a/a/a"),
423 err: IncorrectRelativeRootError{},
424 },
Colin Cross05518bc2018-09-27 15:06:19 -0700425 }
426
427 for _, test := range testCases {
428 t.Run(test.name, func(t *testing.T) {
429 if test.args.Error() != nil {
430 t.Fatal(test.args.Error())
431 }
432
433 args := ZipArgs{}
434 args.FileArgs = test.args.FileArgs()
435 args.CompressionLevel = test.compressionLevel
436 args.EmulateJar = test.emulateJar
437 args.AddDirectoryEntriesToZip = test.dirEntries
438 args.NonDeflatedFiles = test.nonDeflatedFiles
439 args.ManifestSourcePath = test.manifest
Colin Cross09f11052018-09-21 15:12:39 -0700440 args.StoreSymlinks = test.storeSymlinks
Colin Cross4be8f9e2018-09-28 15:16:48 -0700441 args.IgnoreMissingFiles = test.ignoreMissingFiles
Colin Cross05518bc2018-09-27 15:06:19 -0700442 args.Filesystem = mockFs
Colin Cross4be8f9e2018-09-28 15:16:48 -0700443 args.Stderr = &bytes.Buffer{}
Colin Cross05518bc2018-09-27 15:06:19 -0700444
445 buf := &bytes.Buffer{}
446 err := ZipTo(args, buf)
447
448 if (err != nil) != (test.err != nil) {
449 t.Fatalf("want error %v, got %v", test.err, err)
450 } else if test.err != nil {
451 if os.IsNotExist(test.err) {
452 if !os.IsNotExist(test.err) {
453 t.Fatalf("want error %v, got %v", test.err, err)
454 }
Colin Cross1d98ee22018-09-18 17:05:15 -0700455 } else if _, wantRelativeRootErr := test.err.(IncorrectRelativeRootError); wantRelativeRootErr {
456 if _, gotRelativeRootErr := err.(IncorrectRelativeRootError); !gotRelativeRootErr {
457 t.Fatalf("want error %v, got %v", test.err, err)
458 }
Colin Cross05518bc2018-09-27 15:06:19 -0700459 } else {
460 t.Fatalf("want error %v, got %v", test.err, err)
461 }
462 return
463 }
464
465 br := bytes.NewReader(buf.Bytes())
466 zr, err := zip.NewReader(br, int64(br.Len()))
467 if err != nil {
468 t.Fatal(err)
469 }
470
471 var files []zip.FileHeader
472 for _, f := range zr.File {
473 r, err := f.Open()
474 if err != nil {
475 t.Fatalf("error when opening %s: %s", f.Name, err)
476 }
477
478 crc := crc32.NewIEEE()
479 len, err := io.Copy(crc, r)
480 r.Close()
481 if err != nil {
482 t.Fatalf("error when reading %s: %s", f.Name, err)
483 }
484
485 if uint64(len) != f.UncompressedSize64 {
486 t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
487 }
488
489 if crc.Sum32() != f.CRC32 {
490 t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
491 }
492
493 files = append(files, f.FileHeader)
494 }
495
496 if len(files) != len(test.files) {
497 t.Fatalf("want %d files, got %d", len(test.files), len(files))
498 }
499
500 for i := range files {
501 want := test.files[i]
502 got := files[i]
503
504 if want.Name != got.Name {
505 t.Errorf("incorrect file %d want %q got %q", i, want.Name, got.Name)
506 continue
507 }
508
509 if want.UncompressedSize64 != got.UncompressedSize64 {
510 t.Errorf("incorrect file %s length want %v got %v", want.Name,
511 want.UncompressedSize64, got.UncompressedSize64)
512 }
513
514 if want.ExternalAttrs != got.ExternalAttrs {
515 t.Errorf("incorrect file %s attrs want %x got %x", want.Name,
516 want.ExternalAttrs, got.ExternalAttrs)
517 }
518
519 if want.CRC32 != got.CRC32 {
520 t.Errorf("incorrect file %s crc want %v got %v", want.Name,
521 want.CRC32, got.CRC32)
522 }
523
524 if want.Method != got.Method {
525 t.Errorf("incorrect file %s method want %v got %v", want.Name,
526 want.Method, got.Method)
527 }
528 }
529 })
530 }
531}
532
Nan Zhang674dd932018-01-26 18:30:36 -0800533func TestReadRespFile(t *testing.T) {
Colin Cross323dc602020-09-18 14:25:31 -0700534 t.Parallel()
Nan Zhang674dd932018-01-26 18:30:36 -0800535 testCases := []struct {
536 name, in string
537 out []string
538 }{
539 {
540 name: "single quoting test case 1",
541 in: `./cmd '"'-C`,
542 out: []string{"./cmd", `"-C`},
543 },
544 {
545 name: "single quoting test case 2",
546 in: `./cmd '-C`,
547 out: []string{"./cmd", `-C`},
548 },
549 {
550 name: "single quoting test case 3",
551 in: `./cmd '\"'-C`,
552 out: []string{"./cmd", `\"-C`},
553 },
554 {
555 name: "single quoting test case 4",
556 in: `./cmd '\\'-C`,
557 out: []string{"./cmd", `\\-C`},
558 },
559 {
560 name: "none quoting test case 1",
561 in: `./cmd \'-C`,
562 out: []string{"./cmd", `'-C`},
563 },
564 {
565 name: "none quoting test case 2",
566 in: `./cmd \\-C`,
567 out: []string{"./cmd", `\-C`},
568 },
569 {
570 name: "none quoting test case 3",
571 in: `./cmd \"-C`,
572 out: []string{"./cmd", `"-C`},
573 },
574 {
575 name: "double quoting test case 1",
576 in: `./cmd "'"-C`,
577 out: []string{"./cmd", `'-C`},
578 },
579 {
580 name: "double quoting test case 2",
581 in: `./cmd "\\"-C`,
582 out: []string{"./cmd", `\-C`},
583 },
584 {
585 name: "double quoting test case 3",
586 in: `./cmd "\""-C`,
587 out: []string{"./cmd", `"-C`},
588 },
Colin Cross053fca12020-08-19 13:51:47 -0700589 {
590 name: "ninja rsp file",
591 in: "'a'\nb\n'@'\n'foo'\\''bar'\n'foo\"bar'",
592 out: []string{"a", "b", "@", "foo'bar", `foo"bar`},
593 },
Nan Zhang674dd932018-01-26 18:30:36 -0800594 }
595
596 for _, testCase := range testCases {
597 t.Run(testCase.name, func(t *testing.T) {
598 got := ReadRespFile([]byte(testCase.in))
599 if !reflect.DeepEqual(got, testCase.out) {
600 t.Errorf("expected %q got %q", testCase.out, got)
601 }
602 })
603 }
604}
Colin Cross9cb51db2019-06-17 14:12:41 -0700605
606func TestSrcJar(t *testing.T) {
Colin Cross323dc602020-09-18 14:25:31 -0700607 t.Parallel()
Colin Cross9cb51db2019-06-17 14:12:41 -0700608 mockFs := pathtools.MockFs(map[string][]byte{
609 "wrong_package.java": []byte("package foo;"),
610 "foo/correct_package.java": []byte("package foo;"),
611 "src/no_package.java": nil,
612 "src2/parse_error.java": []byte("error"),
613 })
614
615 want := []string{
616 "foo/",
617 "foo/wrong_package.java",
618 "foo/correct_package.java",
619 "no_package.java",
620 "src2/",
621 "src2/parse_error.java",
622 }
623
624 args := ZipArgs{}
625 args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs()
626
627 args.SrcJar = true
628 args.AddDirectoryEntriesToZip = true
629 args.Filesystem = mockFs
630 args.Stderr = &bytes.Buffer{}
631
632 buf := &bytes.Buffer{}
633 err := ZipTo(args, buf)
634 if err != nil {
635 t.Fatalf("got error %v", err)
636 }
637
638 br := bytes.NewReader(buf.Bytes())
639 zr, err := zip.NewReader(br, int64(br.Len()))
640 if err != nil {
641 t.Fatal(err)
642 }
643
644 var got []string
645 for _, f := range zr.File {
646 r, err := f.Open()
647 if err != nil {
648 t.Fatalf("error when opening %s: %s", f.Name, err)
649 }
650
651 crc := crc32.NewIEEE()
652 len, err := io.Copy(crc, r)
653 r.Close()
654 if err != nil {
655 t.Fatalf("error when reading %s: %s", f.Name, err)
656 }
657
658 if uint64(len) != f.UncompressedSize64 {
659 t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
660 }
661
662 if crc.Sum32() != f.CRC32 {
663 t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
664 }
665
666 got = append(got, f.Name)
667 }
668
669 if !reflect.DeepEqual(want, got) {
670 t.Errorf("want files %q, got %q", want, got)
671 }
672}