cmd/merge_zips/merge_zips_test.go

// Copyright 2018 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"hash/crc32"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"android/soong/jar"
	"android/soong/third_party/zip"
)

type testZipEntry struct {
	name      string
	mode      os.FileMode
	data      []byte
	method    uint16
	timestamp time.Time
}

var (
	A     = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
	a     = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
	a2    = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate, jar.DefaultTime}
	a3    = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate, jar.DefaultTime}
	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
	bbb   = testZipEntry{"b/b/b", 0755, nil, zip.Deflate, jar.DefaultTime}
	ba    = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
	bc    = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate, jar.DefaultTime}
	bd    = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate, jar.DefaultTime}
	be    = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate, jar.DefaultTime}

	withTimestamp    = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime.Add(time.Hour)}
	withoutTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime}

	service1a        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store, jar.DefaultTime}
	service1b        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate, jar.DefaultTime}
	service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store, jar.DefaultTime}
	service2         = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate, jar.DefaultTime}

	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate, jar.DefaultTime}
	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate, jar.DefaultTime}
	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate, jar.DefaultTime}
)

type testInputZip struct {
	name    string
	entries []testZipEntry
	reader  *zip.Reader
}

func (tiz *testInputZip) Name() string {
	return tiz.name
}

func (tiz *testInputZip) Open() error {
	if tiz.reader == nil {
		tiz.reader = testZipEntriesToZipReader(tiz.entries)
	}
	return nil
}

func (tiz *testInputZip) Close() error {
	tiz.reader = nil
	return nil
}

func (tiz *testInputZip) Entries() []*zip.File {
	if tiz.reader == nil {
		panic(fmt.Errorf("%s: should be open to get entries", tiz.Name()))
	}
	return tiz.reader.File
}

func (tiz *testInputZip) IsOpen() bool {
	return tiz.reader != nil
}

func TestMergeZips(t *testing.T) {
	testCases := []struct {
		name             string
		in               [][]testZipEntry
		stripFiles       []string
		stripDirs        []string
		jar              bool
		par              bool
		sort             bool
		ignoreDuplicates bool
		stripDirEntries  bool
		zipsToNotStrip   map[string]bool

		out []testZipEntry
		err string
	}{
		{
			name: "duplicates error",
			in: [][]testZipEntry{
				{a},
				{a2},
				{a3},
			},
			out: []testZipEntry{a},
			err: "duplicate",
		},
		{
			name: "duplicates take first",
			in: [][]testZipEntry{
				{a},
				{a2},
				{a3},
			},
			out: []testZipEntry{a},

			ignoreDuplicates: true,
		},
		{
			name: "duplicates identical",
			in: [][]testZipEntry{
				{a},
				{a},
			},
			out: []testZipEntry{a},
		},
		{
			name: "sort",
			in: [][]testZipEntry{
				{be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile},
			},
			out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be},

			sort: true,
		},
		{
			name: "jar sort",
			in: [][]testZipEntry{
				{be, bc, bDir, A, metainfDir, manifestFile},
			},
			out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},

			jar: true,
		},
		{
			name: "jar merge",
			in: [][]testZipEntry{
				{metainfDir, manifestFile, bDir, be},
				{metainfDir, manifestFile2, bDir, bc},
				{metainfDir, manifestFile2, A},
			},
			out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},

			jar: true,
		},
		{
			name: "merge",
			in: [][]testZipEntry{
				{bDir, be},
				{bDir, bc},
				{A},
			},
			out: []testZipEntry{bDir, be, bc, A},
		},
		{
			name: "strip dir entries",
			in: [][]testZipEntry{
				{a, bDir, bbDir, bbb, bc, bd, be},
			},
			out: []testZipEntry{a, bbb, bc, bd, be},

			stripDirEntries: true,
		},
		{
			name: "strip files",
			in: [][]testZipEntry{
				{a, bDir, bbDir, bbb, bc, bd, be},
			},
			out: []testZipEntry{a, bDir, bbDir, bbb, bc},

			stripFiles: []string{"b/d", "b/e"},
		},
		{
			// merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the
			// root of the zip.
			name: "strip file name",
			in: [][]testZipEntry{
				{a, bDir, ba},
			},
			out: []testZipEntry{bDir, ba},

			stripFiles: []string{"a"},
		},
		{
			name: "strip files glob",
			in: [][]testZipEntry{
				{a, bDir, ba},
			},
			out: []testZipEntry{bDir},

			stripFiles: []string{"**/a"},
		},
		{
			name: "strip dirs",
			in: [][]testZipEntry{
				{a, bDir, bbDir, bbb, bc, bd, be},
			},
			out: []testZipEntry{a},

			stripDirs: []string{"b"},
		},
		{
			name: "strip dirs glob",
			in: [][]testZipEntry{
				{a, bDir, bbDir, bbb, bc, bd, be},
			},
			out: []testZipEntry{a, bDir, bc, bd, be},

			stripDirs: []string{"b/*"},
		},
		{
			name: "zips to not strip",
			in: [][]testZipEntry{
				{a, bDir, bc},
				{bDir, bd},
				{bDir, be},
			},
			out: []testZipEntry{a, bDir, bd},

			stripDirs: []string{"b"},
			zipsToNotStrip: map[string]bool{
				"in1": true,
			},
		},
		{
			name: "services",
			in: [][]testZipEntry{
				{service1a, service2},
				{service1b},
			},
			jar: true,
			out: []testZipEntry{service1combined, service2},
		},
		{
			name: "strip timestamps",
			in: [][]testZipEntry{
				{withTimestamp},
				{a},
			},
			out: []testZipEntry{withoutTimestamp, a},
		},
		{
			name: "emulate par",
			in: [][]testZipEntry{
				{
					testZipEntry{name: "3/main.py"},
					testZipEntry{name: "c/main.py"},
					testZipEntry{name: "a/main.py"},
					testZipEntry{name: "2/main.py"},
					testZipEntry{name: "b/main.py"},
					testZipEntry{name: "1/main.py"},
				},
			},
			out: []testZipEntry{
				testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
				testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime},
				testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime},
				testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime},
				testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime},
				testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime},
				testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime},
			},
			par: true,
		},
	}

	for _, test := range testCases {
		t.Run(test.name, func(t *testing.T) {
			inputZips := make([]InputZip, len(test.in))
			for i, in := range test.in {
				inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in}
			}

			want := testZipEntriesToBuf(test.out)

			out := &bytes.Buffer{}
			writer := zip.NewWriter(out)

			err := mergeZips(inputZips, writer, "", "",
				test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates,
				test.stripFiles, test.stripDirs, test.zipsToNotStrip)

			closeErr := writer.Close()
			if closeErr != nil {
				t.Fatal(closeErr)
			}

			if test.err != "" {
				if err == nil {
					t.Fatal("missing err, expected: ", test.err)
				} else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) {
					t.Fatal("incorrect err, want:", test.err, "got:", err)
				}
				return
			} else if err != nil {
				t.Fatal("unexpected err: ", err)
			}

			if !bytes.Equal(want, out.Bytes()) {
				t.Error("incorrect zip output")
				t.Errorf("want:\n%s", dumpZip(want))
				t.Errorf("got:\n%s", dumpZip(out.Bytes()))
				os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
				os.WriteFile("/tmp/want.zip", want, 0755)
			}
		})
	}
}

func testZipEntriesToBuf(entries []testZipEntry) []byte {
	b := &bytes.Buffer{}
	zw := zip.NewWriter(b)

	for _, e := range entries {
		fh := zip.FileHeader{
			Name: e.name,
		}
		fh.SetMode(e.mode)
		fh.Method = e.method
		fh.SetModTime(e.timestamp)
		fh.UncompressedSize64 = uint64(len(e.data))
		fh.CRC32 = crc32.ChecksumIEEE(e.data)
		if fh.Method == zip.Store {
			fh.CompressedSize64 = fh.UncompressedSize64
		}

		w, err := zw.CreateHeaderAndroid(&fh)
		if err != nil {
			panic(err)
		}

		_, err = w.Write(e.data)
		if err != nil {
			panic(err)
		}
	}

	err := zw.Close()
	if err != nil {
		panic(err)
	}

	return b.Bytes()
}

func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader {
	b := testZipEntriesToBuf(entries)
	r := bytes.NewReader(b)

	zr, err := zip.NewReader(r, int64(len(b)))
	if err != nil {
		panic(err)
	}

	return zr
}

func dumpZip(buf []byte) string {
	r := bytes.NewReader(buf)
	zr, err := zip.NewReader(r, int64(len(buf)))
	if err != nil {
		panic(err)
	}

	var ret string

	for _, f := range zr.File {
		ret += fmt.Sprintf("%v: %v %v %08x %s\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32, f.ModTime())
	}

	return ret
}

type DummyInpuZip struct {
	isOpen bool
}

func (diz *DummyInpuZip) Name() string {
	return "dummy"
}

func (diz *DummyInpuZip) Open() error {
	diz.isOpen = true
	return nil
}

func (diz *DummyInpuZip) Close() error {
	diz.isOpen = false
	return nil
}

func (DummyInpuZip) Entries() []*zip.File {
	panic("implement me")
}

func (diz *DummyInpuZip) IsOpen() bool {
	return diz.isOpen
}

func TestInputZipsManager(t *testing.T) {
	const nInputZips = 20
	const nMaxOpenZips = 10
	izm := NewInputZipsManager(20, 10)
	managedZips := make([]InputZip, nInputZips)
	for i := 0; i < nInputZips; i++ {
		managedZips[i] = izm.Manage(&DummyInpuZip{})
	}

	t.Run("InputZipsManager", func(t *testing.T) {
		for i, iz := range managedZips {
			if err := iz.Open(); err != nil {
				t.Fatalf("Step %d: open failed: %s", i, err)
				return
			}
			if izm.nOpenZips > nMaxOpenZips {
				t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips)
			}
		}
		if !managedZips[nInputZips-1].IsOpen() {
			t.Error("The last input should stay open")
		}
		for _, iz := range managedZips {
			iz.Close()
		}
		if izm.nOpenZips > 0 {
			t.Error("Some input zips are still open")
		}
	})
}