blob: 17228c4e0ecb11a6f78ca11e3114b0861d802f80 [file] [log] [blame]
Colin Cross24860652018-07-14 22:19:14 -07001// Copyright 2018 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "bytes"
19 "fmt"
Colin Cross7592d5a2023-07-18 15:57:09 -070020 "hash/crc32"
Colin Cross24860652018-07-14 22:19:14 -070021 "os"
22 "strconv"
23 "strings"
24 "testing"
Colin Crossfa24df62023-11-01 11:18:45 -070025 "time"
Colin Cross24860652018-07-14 22:19:14 -070026
27 "android/soong/jar"
28 "android/soong/third_party/zip"
29)
30
31type testZipEntry struct {
Colin Crossfa24df62023-11-01 11:18:45 -070032 name string
33 mode os.FileMode
34 data []byte
35 method uint16
36 timestamp time.Time
Colin Cross24860652018-07-14 22:19:14 -070037}
38
39var (
Colin Crossfa24df62023-11-01 11:18:45 -070040 A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
41 a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
42 a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate, jar.DefaultTime}
43 a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate, jar.DefaultTime}
44 bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
45 bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
46 bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate, jar.DefaultTime}
47 ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
48 bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate, jar.DefaultTime}
49 bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate, jar.DefaultTime}
50 be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate, jar.DefaultTime}
Colin Cross24860652018-07-14 22:19:14 -070051
Colin Crossfa24df62023-11-01 11:18:45 -070052 withTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime.Add(time.Hour)}
53 withoutTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime}
Colin Cross7592d5a2023-07-18 15:57:09 -070054
Colin Crossfa24df62023-11-01 11:18:45 -070055 service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store, jar.DefaultTime}
56 service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate, jar.DefaultTime}
57 service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store, jar.DefaultTime}
58 service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate, jar.DefaultTime}
59
60 metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
61 manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate, jar.DefaultTime}
62 manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate, jar.DefaultTime}
63 moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate, jar.DefaultTime}
Colin Cross24860652018-07-14 22:19:14 -070064)
65
Sasha Smundak1459a922019-07-16 18:45:24 -070066type testInputZip struct {
67 name string
68 entries []testZipEntry
69 reader *zip.Reader
70}
71
72func (tiz *testInputZip) Name() string {
73 return tiz.name
74}
75
76func (tiz *testInputZip) Open() error {
77 if tiz.reader == nil {
78 tiz.reader = testZipEntriesToZipReader(tiz.entries)
79 }
80 return nil
81}
82
83func (tiz *testInputZip) Close() error {
84 tiz.reader = nil
85 return nil
86}
87
88func (tiz *testInputZip) Entries() []*zip.File {
89 if tiz.reader == nil {
90 panic(fmt.Errorf("%s: should be open to get entries", tiz.Name()))
91 }
92 return tiz.reader.File
93}
94
95func (tiz *testInputZip) IsOpen() bool {
96 return tiz.reader != nil
97}
98
Colin Cross24860652018-07-14 22:19:14 -070099func TestMergeZips(t *testing.T) {
100 testCases := []struct {
101 name string
102 in [][]testZipEntry
103 stripFiles []string
104 stripDirs []string
105 jar bool
Anas Sulaiman437e9472024-02-09 21:06:49 +0000106 par bool
Colin Cross24860652018-07-14 22:19:14 -0700107 sort bool
108 ignoreDuplicates bool
109 stripDirEntries bool
110 zipsToNotStrip map[string]bool
111
112 out []testZipEntry
113 err string
114 }{
115 {
116 name: "duplicates error",
117 in: [][]testZipEntry{
118 {a},
119 {a2},
120 {a3},
121 },
122 out: []testZipEntry{a},
123 err: "duplicate",
124 },
125 {
126 name: "duplicates take first",
127 in: [][]testZipEntry{
128 {a},
129 {a2},
130 {a3},
131 },
132 out: []testZipEntry{a},
133
134 ignoreDuplicates: true,
135 },
136 {
Colin Crossdc1e8292018-10-17 15:05:56 -0700137 name: "duplicates identical",
138 in: [][]testZipEntry{
139 {a},
140 {a},
141 },
142 out: []testZipEntry{a},
143 },
144 {
Colin Cross24860652018-07-14 22:19:14 -0700145 name: "sort",
146 in: [][]testZipEntry{
147 {be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile},
148 },
149 out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be},
150
151 sort: true,
152 },
153 {
154 name: "jar sort",
155 in: [][]testZipEntry{
156 {be, bc, bDir, A, metainfDir, manifestFile},
157 },
158 out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
159
160 jar: true,
161 },
162 {
163 name: "jar merge",
164 in: [][]testZipEntry{
165 {metainfDir, manifestFile, bDir, be},
166 {metainfDir, manifestFile2, bDir, bc},
167 {metainfDir, manifestFile2, A},
168 },
169 out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
170
171 jar: true,
172 },
173 {
174 name: "merge",
175 in: [][]testZipEntry{
176 {bDir, be},
177 {bDir, bc},
178 {A},
179 },
180 out: []testZipEntry{bDir, be, bc, A},
181 },
182 {
183 name: "strip dir entries",
184 in: [][]testZipEntry{
185 {a, bDir, bbDir, bbb, bc, bd, be},
186 },
187 out: []testZipEntry{a, bbb, bc, bd, be},
188
189 stripDirEntries: true,
190 },
191 {
Colin Cross4c03f682018-07-15 08:16:31 -0700192 name: "strip files",
193 in: [][]testZipEntry{
194 {a, bDir, bbDir, bbb, bc, bd, be},
195 },
196 out: []testZipEntry{a, bDir, bbDir, bbb, bc},
197
198 stripFiles: []string{"b/d", "b/e"},
199 },
200 {
201 // merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the
202 // root of the zip.
Colin Cross24860652018-07-14 22:19:14 -0700203 name: "strip file name",
204 in: [][]testZipEntry{
205 {a, bDir, ba},
206 },
Colin Cross4c03f682018-07-15 08:16:31 -0700207 out: []testZipEntry{bDir, ba},
208
209 stripFiles: []string{"a"},
210 },
211 {
212 name: "strip files glob",
213 in: [][]testZipEntry{
214 {a, bDir, ba},
215 },
Colin Cross24860652018-07-14 22:19:14 -0700216 out: []testZipEntry{bDir},
217
Colin Cross4c03f682018-07-15 08:16:31 -0700218 stripFiles: []string{"**/a"},
Colin Cross24860652018-07-14 22:19:14 -0700219 },
220 {
221 name: "strip dirs",
222 in: [][]testZipEntry{
223 {a, bDir, bbDir, bbb, bc, bd, be},
224 },
225 out: []testZipEntry{a},
226
227 stripDirs: []string{"b"},
228 },
229 {
Colin Cross4c03f682018-07-15 08:16:31 -0700230 name: "strip dirs glob",
231 in: [][]testZipEntry{
232 {a, bDir, bbDir, bbb, bc, bd, be},
233 },
234 out: []testZipEntry{a, bDir, bc, bd, be},
235
236 stripDirs: []string{"b/*"},
237 },
238 {
Colin Cross24860652018-07-14 22:19:14 -0700239 name: "zips to not strip",
240 in: [][]testZipEntry{
241 {a, bDir, bc},
242 {bDir, bd},
243 {bDir, be},
244 },
245 out: []testZipEntry{a, bDir, bd},
246
247 stripDirs: []string{"b"},
248 zipsToNotStrip: map[string]bool{
249 "in1": true,
250 },
251 },
Colin Cross7592d5a2023-07-18 15:57:09 -0700252 {
253 name: "services",
254 in: [][]testZipEntry{
255 {service1a, service2},
256 {service1b},
257 },
258 jar: true,
259 out: []testZipEntry{service1combined, service2},
260 },
Colin Crossfa24df62023-11-01 11:18:45 -0700261 {
262 name: "strip timestamps",
263 in: [][]testZipEntry{
264 {withTimestamp},
265 {a},
266 },
267 out: []testZipEntry{withoutTimestamp, a},
268 },
Anas Sulaiman437e9472024-02-09 21:06:49 +0000269 {
270 name: "emulate par",
271 in: [][]testZipEntry{
272 {
273 testZipEntry{name: "3/main.py"},
274 testZipEntry{name: "c/main.py"},
275 testZipEntry{name: "a/main.py"},
276 testZipEntry{name: "2/main.py"},
277 testZipEntry{name: "b/main.py"},
278 testZipEntry{name: "1/main.py"},
279 },
280 },
281 out: []testZipEntry{
282 testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
283 testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
284 testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
285 testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
286 testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
287 testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
288 testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime},
289 testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime},
290 testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime},
291 testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime},
292 testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime},
293 testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime},
294 },
295 par: true,
296 },
Colin Cross24860652018-07-14 22:19:14 -0700297 }
298
299 for _, test := range testCases {
300 t.Run(test.name, func(t *testing.T) {
Sasha Smundak1459a922019-07-16 18:45:24 -0700301 inputZips := make([]InputZip, len(test.in))
Colin Cross24860652018-07-14 22:19:14 -0700302 for i, in := range test.in {
Sasha Smundak1459a922019-07-16 18:45:24 -0700303 inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in}
Colin Cross24860652018-07-14 22:19:14 -0700304 }
305
306 want := testZipEntriesToBuf(test.out)
307
308 out := &bytes.Buffer{}
309 writer := zip.NewWriter(out)
310
Sasha Smundak1459a922019-07-16 18:45:24 -0700311 err := mergeZips(inputZips, writer, "", "",
Anas Sulaiman437e9472024-02-09 21:06:49 +0000312 test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates,
Colin Cross24860652018-07-14 22:19:14 -0700313 test.stripFiles, test.stripDirs, test.zipsToNotStrip)
314
315 closeErr := writer.Close()
316 if closeErr != nil {
Colin Cross7592d5a2023-07-18 15:57:09 -0700317 t.Fatal(closeErr)
Colin Cross24860652018-07-14 22:19:14 -0700318 }
319
320 if test.err != "" {
321 if err == nil {
322 t.Fatal("missing err, expected: ", test.err)
323 } else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) {
324 t.Fatal("incorrect err, want:", test.err, "got:", err)
325 }
326 return
Colin Cross7592d5a2023-07-18 15:57:09 -0700327 } else if err != nil {
328 t.Fatal("unexpected err: ", err)
Colin Cross24860652018-07-14 22:19:14 -0700329 }
330
331 if !bytes.Equal(want, out.Bytes()) {
332 t.Error("incorrect zip output")
333 t.Errorf("want:\n%s", dumpZip(want))
334 t.Errorf("got:\n%s", dumpZip(out.Bytes()))
Colin Cross7592d5a2023-07-18 15:57:09 -0700335 os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
336 os.WriteFile("/tmp/want.zip", want, 0755)
Colin Cross24860652018-07-14 22:19:14 -0700337 }
338 })
339 }
340}
341
342func testZipEntriesToBuf(entries []testZipEntry) []byte {
343 b := &bytes.Buffer{}
344 zw := zip.NewWriter(b)
345
346 for _, e := range entries {
347 fh := zip.FileHeader{
348 Name: e.name,
349 }
350 fh.SetMode(e.mode)
Colin Cross7592d5a2023-07-18 15:57:09 -0700351 fh.Method = e.method
Colin Crossfa24df62023-11-01 11:18:45 -0700352 fh.SetModTime(e.timestamp)
Colin Cross7592d5a2023-07-18 15:57:09 -0700353 fh.UncompressedSize64 = uint64(len(e.data))
354 fh.CRC32 = crc32.ChecksumIEEE(e.data)
355 if fh.Method == zip.Store {
356 fh.CompressedSize64 = fh.UncompressedSize64
357 }
Colin Cross24860652018-07-14 22:19:14 -0700358
Colin Cross7592d5a2023-07-18 15:57:09 -0700359 w, err := zw.CreateHeaderAndroid(&fh)
Colin Cross24860652018-07-14 22:19:14 -0700360 if err != nil {
361 panic(err)
362 }
363
364 _, err = w.Write(e.data)
365 if err != nil {
366 panic(err)
367 }
368 }
369
370 err := zw.Close()
371 if err != nil {
372 panic(err)
373 }
374
375 return b.Bytes()
376}
377
378func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader {
379 b := testZipEntriesToBuf(entries)
380 r := bytes.NewReader(b)
381
382 zr, err := zip.NewReader(r, int64(len(b)))
383 if err != nil {
384 panic(err)
385 }
386
387 return zr
388}
389
390func dumpZip(buf []byte) string {
391 r := bytes.NewReader(buf)
392 zr, err := zip.NewReader(r, int64(len(buf)))
393 if err != nil {
394 panic(err)
395 }
396
397 var ret string
398
399 for _, f := range zr.File {
Colin Crossfa24df62023-11-01 11:18:45 -0700400 ret += fmt.Sprintf("%v: %v %v %08x %s\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32, f.ModTime())
Colin Cross24860652018-07-14 22:19:14 -0700401 }
402
403 return ret
404}
Sasha Smundak1459a922019-07-16 18:45:24 -0700405
406type DummyInpuZip struct {
407 isOpen bool
408}
409
410func (diz *DummyInpuZip) Name() string {
411 return "dummy"
412}
413
414func (diz *DummyInpuZip) Open() error {
415 diz.isOpen = true
416 return nil
417}
418
419func (diz *DummyInpuZip) Close() error {
420 diz.isOpen = false
421 return nil
422}
423
424func (DummyInpuZip) Entries() []*zip.File {
425 panic("implement me")
426}
427
428func (diz *DummyInpuZip) IsOpen() bool {
429 return diz.isOpen
430}
431
432func TestInputZipsManager(t *testing.T) {
433 const nInputZips = 20
434 const nMaxOpenZips = 10
435 izm := NewInputZipsManager(20, 10)
436 managedZips := make([]InputZip, nInputZips)
437 for i := 0; i < nInputZips; i++ {
438 managedZips[i] = izm.Manage(&DummyInpuZip{})
439 }
440
441 t.Run("InputZipsManager", func(t *testing.T) {
442 for i, iz := range managedZips {
443 if err := iz.Open(); err != nil {
444 t.Fatalf("Step %d: open failed: %s", i, err)
445 return
446 }
447 if izm.nOpenZips > nMaxOpenZips {
448 t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips)
449 }
450 }
451 if !managedZips[nInputZips-1].IsOpen() {
452 t.Error("The last input should stay open")
453 }
454 for _, iz := range managedZips {
455 iz.Close()
456 }
457 if izm.nOpenZips > 0 {
458 t.Error("Some input zips are still open")
459 }
460 })
461}