Merge "Support streaming of compressed assets > 1 megabyte" into gingerbread
diff --git a/include/utils/Asset.h b/include/utils/Asset.h
index 5908bcc..2a09095 100644
--- a/include/utils/Asset.h
+++ b/include/utils/Asset.h
@@ -61,15 +61,6 @@
ACCESS_BUFFER,
} AccessMode;
- enum {
- /* data larger than this does not get uncompressed into a buffer */
-#ifdef HAVE_ANDROID_OS
- UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024
-#else
- UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024
-#endif
- };
-
/*
* Read data from the current offset. Returns the actual number of
* bytes read, 0 on EOF, or -1 on error.
@@ -317,6 +308,8 @@
FileMap* mMap; // for memory-mapped input
int mFd; // for file input
+ class StreamingZipInflater* mZipInflater; // for streaming large compressed assets
+
unsigned char* mBuf; // for getBuffer()
};
diff --git a/include/utils/StreamingZipInflater.h b/include/utils/StreamingZipInflater.h
new file mode 100644
index 0000000..16867d8
--- /dev/null
+++ b/include/utils/StreamingZipInflater.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LIBS_STREAMINGZIPINFLATER_H
+#define __LIBS_STREAMINGZIPINFLATER_H
+
+#include <unistd.h>
+#include <inttypes.h>
+#include <zlib.h>
+
+namespace android {
+
+class StreamingZipInflater {
+public:
+ static const size_t INPUT_CHUNK_SIZE = 64 * 1024;
+ static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024;
+
+ // Flavor that pages in the compressed data from a fd
+ StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize);
+
+ // Flavor that gets the compressed data from an in-memory buffer
+ StreamingZipInflater(class FileMap* dataMap, size_t uncompSize);
+
+ ~StreamingZipInflater();
+
+ // read 'count' bytes of uncompressed data from the current position. outBuf may
+ // be NULL, in which case the data is consumed and discarded.
+ ssize_t read(void* outBuf, size_t count);
+
+ // seeking backwards requires uncompressing fom the beginning, so is very
+ // expensive. seeking forwards only requires uncompressing from the current
+ // position to the destination.
+ off_t seekAbsolute(off_t absoluteInputPosition);
+
+private:
+ void initInflateState();
+ int readNextChunk();
+
+ // where to find the uncompressed data
+ int mFd;
+ off_t mInFileStart; // where the compressed data lives in the file
+ class FileMap* mDataMap;
+
+ z_stream mInflateState;
+ bool mStreamNeedsInit;
+
+ // output invariants for this asset
+ uint8_t* mOutBuf; // output buf for decompressed bytes
+ size_t mOutBufSize; // allocated size of mOutBuf
+ size_t mOutTotalSize; // total uncompressed size of the blob
+
+ // current output state bookkeeping
+ off_t mOutCurPosition; // current position in total offset
+ size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf
+ size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf
+
+ // input invariants
+ uint8_t* mInBuf;
+ size_t mInBufSize; // allocated size of mInBuf;
+ size_t mInTotalSize; // total size of compressed data for this blob
+
+ // input state bookkeeping
+ size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies
+ // the z_stream contains state about input block consumption
+};
+
+}
+
+#endif
diff --git a/libs/utils/Android.mk b/libs/utils/Android.mk
index 8bd5823..2e20268 100644
--- a/libs/utils/Android.mk
+++ b/libs/utils/Android.mk
@@ -33,6 +33,7 @@
SharedBuffer.cpp \
Static.cpp \
StopWatch.cpp \
+ StreamingZipInflater.cpp \
String8.cpp \
String16.cpp \
StringArray.cpp \
@@ -131,4 +132,4 @@
# team really wants is to build the stuff defined by this makefile.
ifeq (,$(ONE_SHOT_MAKEFILE))
include $(call first-makefiles-under,$(LOCAL_PATH))
-endif
\ No newline at end of file
+endif
diff --git a/libs/utils/Asset.cpp b/libs/utils/Asset.cpp
index 4295123..cef7db4 100644
--- a/libs/utils/Asset.cpp
+++ b/libs/utils/Asset.cpp
@@ -24,6 +24,7 @@
#include <utils/Asset.h>
#include <utils/Atomic.h>
#include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
#include <utils/ZipUtils.h>
#include <utils/ZipFileRO.h>
#include <utils/Log.h>
@@ -659,7 +660,7 @@
*/
_CompressedAsset::_CompressedAsset(void)
: mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
- mMap(NULL), mFd(-1), mBuf(NULL)
+ mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
{
}
@@ -698,6 +699,10 @@
mFd = fd;
assert(mBuf == NULL);
+ if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+ mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
+ }
+
return NO_ERROR;
}
@@ -724,6 +729,9 @@
mUncompressedLen = uncompressedLen;
assert(mOffset == 0);
+ if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+ mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
+ }
return NO_ERROR;
}
@@ -739,26 +747,29 @@
assert(mOffset >= 0 && mOffset <= mUncompressedLen);
- // TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly
+ /* If we're relying on a streaming inflater, go through that */
+ if (mZipInflater) {
+ actual = mZipInflater->read(buf, count);
+ } else {
+ if (mBuf == NULL) {
+ if (getBuffer(false) == NULL)
+ return -1;
+ }
+ assert(mBuf != NULL);
- if (mBuf == NULL) {
- if (getBuffer(false) == NULL)
- return -1;
+ /* adjust count if we're near EOF */
+ maxLen = mUncompressedLen - mOffset;
+ if (count > maxLen)
+ count = maxLen;
+
+ if (!count)
+ return 0;
+
+ /* copy from buffer */
+ //printf("comp buf read\n");
+ memcpy(buf, (char*)mBuf + mOffset, count);
+ actual = count;
}
- assert(mBuf != NULL);
-
- /* adjust count if we're near EOF */
- maxLen = mUncompressedLen - mOffset;
- if (count > maxLen)
- count = maxLen;
-
- if (!count)
- return 0;
-
- /* copy from buffer */
- //printf("comp buf read\n");
- memcpy(buf, (char*)mBuf + mOffset, count);
- actual = count;
mOffset += actual;
return actual;
@@ -780,6 +791,9 @@
if (newPosn == (off_t) -1)
return newPosn;
+ if (mZipInflater) {
+ mZipInflater->seekAbsolute(newPosn);
+ }
mOffset = newPosn;
return mOffset;
}
@@ -793,10 +807,12 @@
mMap->release();
mMap = NULL;
}
- if (mBuf != NULL) {
- delete[] mBuf;
- mBuf = NULL;
- }
+
+ delete[] mBuf;
+ mBuf = NULL;
+
+ delete mZipInflater;
+ mZipInflater = NULL;
if (mFd > 0) {
::close(mFd);
@@ -817,12 +833,6 @@
if (mBuf != NULL)
return mBuf;
- if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
- LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
- (long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
- goto bail;
- }
-
/*
* Allocate a buffer and read the file into it.
*/
@@ -853,7 +863,13 @@
goto bail;
}
- /* success! */
+ /*
+ * Success - now that we have the full asset in RAM we
+ * no longer need the streaming inflater
+ */
+ delete mZipInflater;
+ mZipInflater = NULL;
+
mBuf = buf;
buf = NULL;
diff --git a/libs/utils/StreamingZipInflater.cpp b/libs/utils/StreamingZipInflater.cpp
new file mode 100644
index 0000000..2ebec93
--- /dev/null
+++ b/libs/utils/StreamingZipInflater.cpp
@@ -0,0 +1,224 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "szipinf"
+#include <utils/Log.h>
+
+#include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
+#include <string.h>
+#include <assert.h>
+
+static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }
+
+using namespace android;
+
+/*
+ * Streaming access to compressed asset data in an open fd
+ */
+StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
+ size_t uncompSize, size_t compSize) {
+ mFd = fd;
+ mDataMap = NULL;
+ mInFileStart = compDataStart;
+ mOutTotalSize = uncompSize;
+ mInTotalSize = compSize;
+
+ mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
+ mInBuf = new uint8_t[mInBufSize];
+
+ mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+ mOutBuf = new uint8_t[mOutBufSize];
+
+ initInflateState();
+}
+
+/*
+ * Streaming access to compressed data held in an mmapped region of memory
+ */
+StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
+ mFd = -1;
+ mDataMap = dataMap;
+ mOutTotalSize = uncompSize;
+ mInTotalSize = dataMap->getDataLength();
+
+ mInBuf = (uint8_t*) dataMap->getDataPtr();
+ mInBufSize = mInTotalSize;
+
+ mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+ mOutBuf = new uint8_t[mOutBufSize];
+
+ initInflateState();
+}
+
+StreamingZipInflater::~StreamingZipInflater() {
+ // tear down the in-flight zip state just in case
+ ::inflateEnd(&mInflateState);
+
+ if (mDataMap == NULL) {
+ delete [] mInBuf;
+ }
+ delete [] mOutBuf;
+}
+
+void StreamingZipInflater::initInflateState() {
+ LOGD("Initializing inflate state");
+
+ memset(&mInflateState, 0, sizeof(mInflateState));
+ mInflateState.zalloc = Z_NULL;
+ mInflateState.zfree = Z_NULL;
+ mInflateState.opaque = Z_NULL;
+ mInflateState.next_in = (Bytef*)mInBuf;
+ mInflateState.next_out = (Bytef*) mOutBuf;
+ mInflateState.avail_out = mOutBufSize;
+ mInflateState.data_type = Z_UNKNOWN;
+
+ mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
+ mInNextChunkOffset = 0;
+ mStreamNeedsInit = true;
+
+ if (mDataMap == NULL) {
+ ::lseek(mFd, mInFileStart, SEEK_SET);
+ mInflateState.avail_in = 0; // set when a chunk is read in
+ } else {
+ mInflateState.avail_in = mInBufSize;
+ }
+}
+
+/*
+ * Basic approach:
+ *
+ * 1. If we have undelivered uncompressed data, send it. At this point
+ * either we've satisfied the request, or we've exhausted the available
+ * output data in mOutBuf.
+ *
+ * 2. While we haven't sent enough data to satisfy the request:
+ * 0. if the request is for more data than exists, bail.
+ * a. if there is no input data to decode, read some into the input buffer
+ * and readjust the z_stream input pointers
+ * b. point the output to the start of the output buffer and decode what we can
+ * c. deliver whatever output data we can
+ */
+ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
+ uint8_t* dest = (uint8_t*) outBuf;
+ size_t bytesRead = 0;
+ size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
+ while (toRead > 0) {
+ // First, write from whatever we already have decoded and ready to go
+ size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
+ if (deliverable > 0) {
+ if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
+ mOutDeliverable += deliverable;
+ mOutCurPosition += deliverable;
+ dest += deliverable;
+ bytesRead += deliverable;
+ toRead -= deliverable;
+ }
+
+ // need more data? time to decode some.
+ if (toRead > 0) {
+ // if we don't have any data to decode, read some in. If we're working
+ // from mmapped data this won't happen, because the clipping to total size
+ // will prevent reading off the end of the mapped input chunk.
+ if (mInflateState.avail_in == 0) {
+ int err = readNextChunk();
+ if (err < 0) {
+ LOGE("Unable to access asset data: %d", err);
+ if (!mStreamNeedsInit) {
+ ::inflateEnd(&mInflateState);
+ initInflateState();
+ }
+ return -1;
+ }
+ }
+ // we know we've drained whatever is in the out buffer now, so just
+ // start from scratch there, reading all the input we have at present.
+ mInflateState.next_out = (Bytef*) mOutBuf;
+ mInflateState.avail_out = mOutBufSize;
+
+ /*
+ LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
+ mInflateState.avail_in, mInflateState.avail_out,
+ mInflateState.next_in, mInflateState.next_out);
+ */
+ int result = Z_OK;
+ if (mStreamNeedsInit) {
+ LOGI("Initializing zlib to inflate");
+ result = inflateInit2(&mInflateState, -MAX_WBITS);
+ mStreamNeedsInit = false;
+ }
+ if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
+ if (result < 0) {
+ // Whoops, inflation failed
+ LOGE("Error inflating asset: %d", result);
+ ::inflateEnd(&mInflateState);
+ initInflateState();
+ return -1;
+ } else {
+ if (result == Z_STREAM_END) {
+ // we know we have to have reached the target size here and will
+ // not try to read any further, so just wind things up.
+ ::inflateEnd(&mInflateState);
+ }
+
+ // Note how much data we got, and off we go
+ mOutDeliverable = 0;
+ mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
+ }
+ }
+ }
+ return bytesRead;
+}
+
+int StreamingZipInflater::readNextChunk() {
+ assert(mDataMap == NULL);
+
+ if (mInNextChunkOffset < mInTotalSize) {
+ size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
+ if (toRead > 0) {
+ ssize_t didRead = ::read(mFd, mInBuf, toRead);
+ //LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
+ if (didRead < 0) {
+ // TODO: error
+ LOGE("Error reading asset data");
+ return didRead;
+ } else {
+ mInNextChunkOffset += didRead;
+ mInflateState.next_in = (Bytef*) mInBuf;
+ mInflateState.avail_in = didRead;
+ }
+ }
+ }
+ return 0;
+}
+
+// seeking backwards requires uncompressing fom the beginning, so is very
+// expensive. seeking forwards only requires uncompressing from the current
+// position to the destination.
+off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
+ if (absoluteInputPosition < mOutCurPosition) {
+ // rewind and reprocess the data from the beginning
+ if (!mStreamNeedsInit) {
+ ::inflateEnd(&mInflateState);
+ }
+ initInflateState();
+ read(NULL, absoluteInputPosition);
+ } else if (absoluteInputPosition > mOutCurPosition) {
+ read(NULL, absoluteInputPosition - mOutCurPosition);
+ }
+ // else if the target position *is* our current position, do nothing
+ return absoluteInputPosition;
+}