Merge META-INF/services/* files in merge_zips -jar
kotlinx_coroutines_test and kotlinx_coroutine_android each provide a
META-INF/services/kotlinx.coroutines.CoroutineExceptionHandler with
different contents, and the final contents needs to be the combination
of the two files. Implement service merging in merge_zips when the
-jar argument is provided.
Bug: 290933559
Test: TestMergeZips
Change-Id: I69f80d1265c64c671d308ef4cdccfa1564abe056
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go
index e3d1179..a70a9d1 100644
--- a/cmd/merge_zips/merge_zips.go
+++ b/cmd/merge_zips/merge_zips.go
@@ -122,7 +122,7 @@
}
func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error {
- w, err := zw.CreateHeader(be.fh)
+ w, err := zw.CreateHeaderAndroid(be.fh)
if err != nil {
return err
}
@@ -562,6 +562,8 @@
}
}
+ var jarServices jar.Services
+
// Finally, add entries from all the input zips.
for _, inputZip := range inputZips {
_, copyFully := zipsToNotStrip[inputZip.Name()]
@@ -570,6 +572,14 @@
}
for i, entry := range inputZip.Entries() {
+ if emulateJar && jarServices.IsServiceFile(entry) {
+ // If this is a jar, collect service files to combine instead of adding them to the zip.
+ err := jarServices.AddServiceFile(entry)
+ if err != nil {
+ return err
+ }
+ continue
+ }
if copyFully || !out.isEntryExcluded(entry.Name) {
if err := out.copyEntry(inputZip, i); err != nil {
return err
@@ -585,6 +595,16 @@
}
if emulateJar {
+ // Combine all the service files into a single list of combined service files and add them to the zip.
+ for _, serviceFile := range jarServices.ServiceFiles() {
+ _, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{
+ fh: serviceFile.FileHeader,
+ content: serviceFile.Contents,
+ })
+ if err != nil {
+ return err
+ }
+ }
return out.writeEntries(out.jarSorted())
} else if sortEntries {
return out.writeEntries(out.alphanumericSorted())
diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go
index cb58436..767d4e6 100644
--- a/cmd/merge_zips/merge_zips_test.go
+++ b/cmd/merge_zips/merge_zips_test.go
@@ -17,6 +17,7 @@
import (
"bytes"
"fmt"
+ "hash/crc32"
"os"
"strconv"
"strings"
@@ -27,28 +28,34 @@
)
type testZipEntry struct {
- name string
- mode os.FileMode
- data []byte
+ name string
+ mode os.FileMode
+ data []byte
+ method uint16
}
var (
- A = testZipEntry{"A", 0755, []byte("foo")}
- a = testZipEntry{"a", 0755, []byte("foo")}
- a2 = testZipEntry{"a", 0755, []byte("FOO2")}
- a3 = testZipEntry{"a", 0755, []byte("Foo3")}
- bDir = testZipEntry{"b/", os.ModeDir | 0755, nil}
- bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil}
- bbb = testZipEntry{"b/b/b", 0755, nil}
- ba = testZipEntry{"b/a", 0755, []byte("foob")}
- bc = testZipEntry{"b/c", 0755, []byte("bar")}
- bd = testZipEntry{"b/d", 0700, []byte("baz")}
- be = testZipEntry{"b/e", 0700, []byte("")}
+ A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate}
+ a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate}
+ a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate}
+ a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate}
+ bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate}
+ bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate}
+ bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate}
+ ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate}
+ bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate}
+ bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate}
+ be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate}
- metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil}
- manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")}
- manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")}
- moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")}
+ service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store}
+ service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate}
+ service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store}
+ service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate}
+
+ metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate}
+ manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate}
+ manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate}
+ moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate}
)
type testInputZip struct {
@@ -236,6 +243,15 @@
"in1": true,
},
},
+ {
+ name: "services",
+ in: [][]testZipEntry{
+ {service1a, service2},
+ {service1b},
+ },
+ jar: true,
+ out: []testZipEntry{service1combined, service2},
+ },
}
for _, test := range testCases {
@@ -256,7 +272,7 @@
closeErr := writer.Close()
if closeErr != nil {
- t.Fatal(err)
+ t.Fatal(closeErr)
}
if test.err != "" {
@@ -266,12 +282,16 @@
t.Fatal("incorrect err, want:", test.err, "got:", err)
}
return
+ } else if err != nil {
+ t.Fatal("unexpected err: ", err)
}
if !bytes.Equal(want, out.Bytes()) {
t.Error("incorrect zip output")
t.Errorf("want:\n%s", dumpZip(want))
t.Errorf("got:\n%s", dumpZip(out.Bytes()))
+ os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
+ os.WriteFile("/tmp/want.zip", want, 0755)
}
})
}
@@ -286,8 +306,14 @@
Name: e.name,
}
fh.SetMode(e.mode)
+ fh.Method = e.method
+ fh.UncompressedSize64 = uint64(len(e.data))
+ fh.CRC32 = crc32.ChecksumIEEE(e.data)
+ if fh.Method == zip.Store {
+ fh.CompressedSize64 = fh.UncompressedSize64
+ }
- w, err := zw.CreateHeader(&fh)
+ w, err := zw.CreateHeaderAndroid(&fh)
if err != nil {
panic(err)
}
diff --git a/jar/Android.bp b/jar/Android.bp
index 46113d8..c03e491 100644
--- a/jar/Android.bp
+++ b/jar/Android.bp
@@ -21,6 +21,7 @@
pkgPath: "android/soong/jar",
srcs: [
"jar.go",
+ "services.go",
],
testSrcs: [
"jar_test.go",
diff --git a/jar/services.go b/jar/services.go
new file mode 100644
index 0000000..d06a6dc
--- /dev/null
+++ b/jar/services.go
@@ -0,0 +1,128 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jar
+
+import (
+ "android/soong/third_party/zip"
+ "bufio"
+ "hash/crc32"
+ "sort"
+ "strings"
+)
+
+const servicesPrefix = "META-INF/services/"
+
+// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing
+// the unique lines from all the input zip entries with the same name.
+type Services struct {
+ services map[string]*ServiceFile
+}
+
+// ServiceFile contains the combined contents of all input zip entries with a single name.
+type ServiceFile struct {
+ Name string
+ FileHeader *zip.FileHeader
+ Contents []byte
+ Lines []string
+}
+
+// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory.
+func (Services) IsServiceFile(entry *zip.File) bool {
+ return strings.HasPrefix(entry.Name, servicesPrefix)
+}
+
+// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need
+// to be combined.
+func (j *Services) AddServiceFile(entry *zip.File) error {
+ if j.services == nil {
+ j.services = map[string]*ServiceFile{}
+ }
+
+ service := entry.Name
+ serviceFile := j.services[service]
+ fh := entry.FileHeader
+ if serviceFile == nil {
+ serviceFile = &ServiceFile{
+ Name: service,
+ FileHeader: &fh,
+ }
+ j.services[service] = serviceFile
+ }
+
+ f, err := entry.Open()
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if line != "" {
+ serviceFile.Lines = append(serviceFile.Lines, line)
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// ServiceFiles returns the list of combined service files, each containing all the unique lines from the
+// corresponding service files in the input zip entries.
+func (j *Services) ServiceFiles() []ServiceFile {
+ services := make([]ServiceFile, 0, len(j.services))
+
+ for _, serviceFile := range j.services {
+ serviceFile.Lines = dedupServicesLines(serviceFile.Lines)
+ serviceFile.Lines = append(serviceFile.Lines, "")
+ serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n"))
+
+ serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents))
+ serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents)
+ if serviceFile.FileHeader.Method == zip.Store {
+ serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64
+ }
+
+ services = append(services, *serviceFile)
+ }
+
+ sort.Slice(services, func(i, j int) bool {
+ return services[i].Name < services[j].Name
+ })
+
+ return services
+}
+
+func dedupServicesLines(in []string) []string {
+ writeIndex := 0
+outer:
+ for readIndex := 0; readIndex < len(in); readIndex++ {
+ for compareIndex := 0; compareIndex < writeIndex; compareIndex++ {
+ if interface{}(in[readIndex]) == interface{}(in[compareIndex]) {
+ // The value at readIndex already exists somewhere in the output region
+ // of the slice before writeIndex, skip it.
+ continue outer
+ }
+ }
+ if readIndex != writeIndex {
+ in[writeIndex] = in[readIndex]
+ }
+ writeIndex++
+ }
+ return in[0:writeIndex]
+}
diff --git a/third_party/zip/android.go b/third_party/zip/android.go
index f8e45c5..0f41f62 100644
--- a/third_party/zip/android.go
+++ b/third_party/zip/android.go
@@ -170,7 +170,7 @@
func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) {
writeDataDescriptor := fh.Method != Store
if writeDataDescriptor {
- fh.Flags &= DataDescriptorFlag
+ fh.Flags |= DataDescriptorFlag
} else {
fh.Flags &= ^uint16(DataDescriptorFlag)
}