Rearrange manifest file handling in merge_zips and soong_zip
Jar always puts default MANIFEST.MF files in if none was specified.
Copying that behavior in soong_zip causes problems with merge_zips,
because it ends up taking the default manifest from the classes.jar
instead of the user's manifest from res.jar. We don't want the
user's manifest in the classes.jar, otherwise a change to the
manifest will cause all the class files to rebuild. Instead,
move the manifest insertion to the final merge_zips stage.
Test: m -j checkbuild
Change-Id: Id6376961dbaf743c2fb92843f9bdf2e44b963be0
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go
index 9fd5ddd..a94392b 100644
--- a/cmd/merge_zips/merge_zips.go
+++ b/cmd/merge_zips/merge_zips.go
@@ -15,8 +15,10 @@
package main
import (
+ "errors"
"flag"
"fmt"
+ "hash/crc32"
"log"
"os"
"path/filepath"
@@ -52,10 +54,12 @@
}
var (
- sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
- emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
- stripDirs []string
- zipsToNotStrip = make(map[string]bool)
+ sortEntries = flag.Bool("s", false, "sort entries (defaults to the order from the input zip files)")
+ emulateJar = flag.Bool("j", false, "sort zip entries using jar ordering (META-INF first)")
+ stripDirs []string
+ zipsToNotStrip = make(map[string]bool)
+ stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
+ manifest = flag.String("m", "", "manifest file to insert in jar")
)
func init() {
@@ -65,7 +69,7 @@
func main() {
flag.Usage = func() {
- fmt.Fprintln(os.Stderr, "usage: merge_zips [-j] output [inputs...]")
+ fmt.Fprintln(os.Stderr, "usage: merge_zips [-jsD] [-m manifest] output [inputs...]")
flag.PrintDefaults()
}
@@ -107,8 +111,13 @@
readers = append(readers, namedReader)
}
+ if *manifest != "" && !*emulateJar {
+ log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
+ }
+
// do merge
- if err := mergeZips(readers, writer, *sortEntries, *emulateJar); err != nil {
+ err = mergeZips(readers, writer, *manifest, *sortEntries, *emulateJar, *stripDirEntries)
+ if err != nil {
log.Fatal(err)
}
}
@@ -129,23 +138,109 @@
return p.zipName + "/" + p.entryName
}
-// a zipEntry knows the location and content of a file within a zip
+// a zipEntry is a zipSource that pulls its content from another zip
type zipEntry struct {
path zipEntryPath
content *zip.File
}
-// a fileMapping specifies to copy a zip entry from one place to another
-type fileMapping struct {
- source zipEntry
- dest string
+func (ze zipEntry) String() string {
+ return ze.path.String()
}
-func mergeZips(readers []namedZipReader, writer *zip.Writer, sortEntries bool, emulateJar bool) error {
+func (ze zipEntry) IsDir() bool {
+ return ze.content.FileInfo().IsDir()
+}
- mappingsByDest := make(map[string]fileMapping, 0)
+func (ze zipEntry) CRC32() uint32 {
+ return ze.content.FileHeader.CRC32
+}
+
+func (ze zipEntry) WriteToZip(dest string, zw *zip.Writer) error {
+ return zw.CopyFrom(ze.content, dest)
+}
+
+// a bufferEntry is a zipSource that pulls its content from a []byte
+type bufferEntry struct {
+ fh *zip.FileHeader
+ content []byte
+}
+
+func (be bufferEntry) String() string {
+ return "internal buffer"
+}
+
+func (be bufferEntry) IsDir() bool {
+ return be.fh.FileInfo().IsDir()
+}
+
+func (be bufferEntry) CRC32() uint32 {
+ return crc32.ChecksumIEEE(be.content)
+}
+
+func (be bufferEntry) WriteToZip(dest string, zw *zip.Writer) error {
+ w, err := zw.CreateHeader(be.fh)
+ if err != nil {
+ return err
+ }
+
+ if !be.IsDir() {
+ _, err = w.Write(be.content)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+type zipSource interface {
+ String() string
+ IsDir() bool
+ CRC32() uint32
+ WriteToZip(dest string, zw *zip.Writer) error
+}
+
+// a fileMapping specifies to copy a zip entry from one place to another
+type fileMapping struct {
+ dest string
+ source zipSource
+}
+
+func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest string,
+ sortEntries, emulateJar, stripDirEntries bool) error {
+
+ sourceByDest := make(map[string]zipSource, 0)
orderedMappings := []fileMapping{}
+ // if dest already exists returns a non-null zipSource for the existing source
+ addMapping := func(dest string, source zipSource) zipSource {
+ mapKey := filepath.Clean(dest)
+ if existingSource, exists := sourceByDest[mapKey]; exists {
+ return existingSource
+ }
+
+ sourceByDest[mapKey] = source
+ orderedMappings = append(orderedMappings, fileMapping{source: source, dest: dest})
+ return nil
+ }
+
+ if manifest != "" {
+ if !stripDirEntries {
+ dirHeader := jar.MetaDirFileHeader()
+ dirSource := bufferEntry{dirHeader, nil}
+ addMapping(jar.MetaDir, dirSource)
+ }
+
+ fh, buf, err := jar.ManifestFileContents(manifest)
+ if err != nil {
+ return err
+ }
+
+ fileSource := bufferEntry{fh, buf}
+ addMapping(jar.ManifestFile, fileSource)
+ }
+
for _, namedReader := range readers {
_, skipStripThisZip := zipsToNotStrip[namedReader.path]
FileLoop:
@@ -163,46 +258,39 @@
}
}
}
+
+ if stripDirEntries && file.FileInfo().IsDir() {
+ continue
+ }
+
// check for other files or directories destined for the same path
dest := file.Name
- mapKey := dest
- if strings.HasSuffix(mapKey, "/") {
- mapKey = mapKey[:len(mapKey)-1]
- }
- existingMapping, exists := mappingsByDest[mapKey]
// make a new entry to add
source := zipEntry{path: zipEntryPath{zipName: namedReader.path, entryName: file.Name}, content: file}
- newMapping := fileMapping{source: source, dest: dest}
- if exists {
+ if existingSource := addMapping(dest, source); existingSource != nil {
// handle duplicates
- wasDir := existingMapping.source.content.FileHeader.FileInfo().IsDir()
- isDir := newMapping.source.content.FileHeader.FileInfo().IsDir()
- if wasDir != isDir {
+ if existingSource.IsDir() != source.IsDir() {
return fmt.Errorf("Directory/file mismatch at %v from %v and %v\n",
- dest, existingMapping.source.path, newMapping.source.path)
+ dest, existingSource, source)
}
if emulateJar &&
file.Name == jar.ManifestFile || file.Name == jar.ModuleInfoClass {
// Skip manifest and module info files that are not from the first input file
continue
}
- if !isDir {
+ if !source.IsDir() {
if emulateJar {
- if existingMapping.source.content.CRC32 != newMapping.source.content.CRC32 {
+ if existingSource.CRC32() != source.CRC32() {
fmt.Fprintf(os.Stdout, "WARNING: Duplicate path %v found in %v and %v\n",
- dest, existingMapping.source.path, newMapping.source.path)
+ dest, existingSource, source)
}
} else {
return fmt.Errorf("Duplicate path %v found in %v and %v\n",
- dest, existingMapping.source.path, newMapping.source.path)
+ dest, existingSource, source)
}
}
- } else {
- // save entry
- mappingsByDest[mapKey] = newMapping
- orderedMappings = append(orderedMappings, newMapping)
}
}
}
@@ -214,7 +302,7 @@
}
for _, entry := range orderedMappings {
- if err := writer.CopyFrom(entry.source.content, entry.dest); err != nil {
+ if err := entry.source.WriteToZip(entry.dest, writer); err != nil {
return err
}
}
diff --git a/cmd/soong_zip/soong_zip.go b/cmd/soong_zip/soong_zip.go
index 4cf0764..cb9df9a 100644
--- a/cmd/soong_zip/soong_zip.go
+++ b/cmd/soong_zip/soong_zip.go
@@ -58,7 +58,7 @@
}
type byteReaderCloser struct {
- bytes.Reader
+ *bytes.Reader
io.Closer
}
@@ -252,7 +252,7 @@
}
w := &zipWriter{
- time: time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC),
+ time: jar.DefaultTime,
createdDirs: make(map[string]string),
createdFiles: make(map[string]string),
directories: *directories,
@@ -353,14 +353,14 @@
z.memoryRateLimiter.Stop()
}()
- if manifest != "" {
- if !*emulateJar {
- return errors.New("must specify --jar when specifying a manifest via -m")
- }
- pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
+ if manifest != "" && !*emulateJar {
+ return errors.New("must specify --jar when specifying a manifest via -m")
}
if *emulateJar {
+ // manifest may be empty, in which case addManifest will fill in a default
+ pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
+
jarSort(pathMappings)
}
@@ -524,11 +524,6 @@
}
func (z *zipWriter) addManifest(dest string, src string, method uint16) error {
- givenBytes, err := ioutil.ReadFile(src)
- if err != nil {
- return err
- }
-
if prev, exists := z.createdDirs[dest]; exists {
return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
}
@@ -536,27 +531,18 @@
return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
}
- manifestMarker := []byte("Manifest-Version:")
- header := append(manifestMarker, []byte(" 1.0\nCreated-By: soong_zip\n")...)
-
- var finalBytes []byte
- if !bytes.Contains(givenBytes, manifestMarker) {
- finalBytes = append(append(header, givenBytes...), byte('\n'))
- } else {
- finalBytes = givenBytes
+ if err := z.writeDirectory(filepath.Dir(dest), src); err != nil {
+ return err
}
- byteReader := bytes.NewReader(finalBytes)
-
- reader := &byteReaderCloser{*byteReader, ioutil.NopCloser(nil)}
-
- fileHeader := &zip.FileHeader{
- Name: dest,
- Method: zip.Store,
- UncompressedSize64: uint64(byteReader.Len()),
+ fh, buf, err := jar.ManifestFileContents(src)
+ if err != nil {
+ return err
}
- return z.writeFileContents(fileHeader, reader)
+ reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
+
+ return z.writeFileContents(fh, reader)
}
func (z *zipWriter) writeFileContents(header *zip.FileHeader, r readerSeekerCloser) (err error) {
@@ -770,19 +756,6 @@
close(compressChan)
}
-func (z *zipWriter) addExtraField(zipHeader *zip.FileHeader, fieldHeader [2]byte, data []byte) {
- // add the field header in little-endian order
- zipHeader.Extra = append(zipHeader.Extra, fieldHeader[1], fieldHeader[0])
-
- // specify the length of the data (in little-endian order)
- dataLength := len(data)
- lengthBytes := []byte{byte(dataLength % 256), byte(dataLength / 256)}
- zipHeader.Extra = append(zipHeader.Extra, lengthBytes...)
-
- // add the contents of the extra field
- zipHeader.Extra = append(zipHeader.Extra, data...)
-}
-
// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
// the directory entry to the zip file if directories are enabled.
func (z *zipWriter) writeDirectory(dir, src string) error {
@@ -810,17 +783,19 @@
if z.directories {
// make a directory entry for each uncreated directory
for _, cleanDir := range zipDirs {
- dirHeader := &zip.FileHeader{
- Name: cleanDir + "/",
- }
- dirHeader.SetMode(0700 | os.ModeDir)
- dirHeader.SetModTime(z.time)
+ var dirHeader *zip.FileHeader
- if *emulateJar && dir == "META-INF/" {
- // Jar files have a 0-length extra field with header "CAFE"
- z.addExtraField(dirHeader, [2]byte{0xca, 0xfe}, []byte{})
+ if *emulateJar && cleanDir+"/" == jar.MetaDir {
+ dirHeader = jar.MetaDirFileHeader()
+ } else {
+ dirHeader = &zip.FileHeader{
+ Name: cleanDir + "/",
+ }
+ dirHeader.SetMode(0700 | os.ModeDir)
}
+ dirHeader.SetModTime(z.time)
+
ze := make(chan *zipEntry, 1)
ze <- &zipEntry{
fh: dirHeader,