Move StringPool to libandroidfw
Test: verified affected tests pass
Bug: 232940948
Change-Id: I22089893d7e5013f759c39ce190bec07fa6435db
diff --git a/libs/androidfw/Android.bp b/libs/androidfw/Android.bp
index 8a379d5..779c4b7 100644
--- a/libs/androidfw/Android.bp
+++ b/libs/androidfw/Android.bp
@@ -61,6 +61,7 @@
"AssetManager2.cpp",
"AssetsProvider.cpp",
"AttributeResolution.cpp",
+ "BigBuffer.cpp",
"ChunkIterator.cpp",
"ConfigDescription.cpp",
"Idmap.cpp",
@@ -73,6 +74,7 @@
"ResourceTypes.cpp",
"ResourceUtils.cpp",
"StreamingZipInflater.cpp",
+ "StringPool.cpp",
"TypeWrappers.cpp",
"Util.cpp",
"ZipFileRO.cpp",
@@ -162,6 +164,7 @@
"tests/AssetManager2_test.cpp",
"tests/AttributeFinder_test.cpp",
"tests/AttributeResolution_test.cpp",
+ "tests/BigBuffer_test.cpp",
"tests/ByteBucketArray_test.cpp",
"tests/Config_test.cpp",
"tests/ConfigDescription_test.cpp",
@@ -174,6 +177,7 @@
"tests/ResTable_test.cpp",
"tests/Split_test.cpp",
"tests/StringPiece_test.cpp",
+ "tests/StringPool_test.cpp",
"tests/Theme_test.cpp",
"tests/TypeWrappers_test.cpp",
"tests/ZipUtils_test.cpp",
diff --git a/libs/androidfw/AssetManager2.cpp b/libs/androidfw/AssetManager2.cpp
index 136fc6c..39c7d19 100644
--- a/libs/androidfw/AssetManager2.cpp
+++ b/libs/androidfw/AssetManager2.cpp
@@ -601,6 +601,10 @@
return base::unexpected(result.error());
}
+ if (type_idx == 0x1c) {
+ LOG(ERROR) << base::StringPrintf("foobar first result %s", result->package_name->c_str());
+ }
+
bool overlaid = false;
if (!stop_at_first_match && !ignore_configuration && !apk_assets_[result->cookie]->IsLoader()) {
for (const auto& id_map : package_group.overlays_) {
diff --git a/libs/androidfw/BigBuffer.cpp b/libs/androidfw/BigBuffer.cpp
new file mode 100644
index 0000000..bedfc49
--- /dev/null
+++ b/libs/androidfw/BigBuffer.cpp
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <androidfw/BigBuffer.h>
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "android-base/logging.h"
+
+namespace android {
+
+void* BigBuffer::NextBlockImpl(size_t size) {
+ if (!blocks_.empty()) {
+ Block& block = blocks_.back();
+ if (block.block_size_ - block.size >= size) {
+ void* out_buffer = block.buffer.get() + block.size;
+ block.size += size;
+ size_ += size;
+ return out_buffer;
+ }
+ }
+
+ const size_t actual_size = std::max(block_size_, size);
+
+ Block block = {};
+
+ // Zero-allocate the block's buffer.
+ block.buffer = std::unique_ptr<uint8_t[]>(new uint8_t[actual_size]());
+ CHECK(block.buffer);
+
+ block.size = size;
+ block.block_size_ = actual_size;
+
+ blocks_.push_back(std::move(block));
+ size_ += size;
+ return blocks_.back().buffer.get();
+}
+
+void* BigBuffer::NextBlock(size_t* out_size) {
+ if (!blocks_.empty()) {
+ Block& block = blocks_.back();
+ if (block.size != block.block_size_) {
+ void* out_buffer = block.buffer.get() + block.size;
+ size_t size = block.block_size_ - block.size;
+ block.size = block.block_size_;
+ size_ += size;
+ *out_size = size;
+ return out_buffer;
+ }
+ }
+
+ // Zero-allocate the block's buffer.
+ Block block = {};
+ block.buffer = std::unique_ptr<uint8_t[]>(new uint8_t[block_size_]());
+ CHECK(block.buffer);
+ block.size = block_size_;
+ block.block_size_ = block_size_;
+ blocks_.push_back(std::move(block));
+ size_ += block_size_;
+ *out_size = block_size_;
+ return blocks_.back().buffer.get();
+}
+
+std::string BigBuffer::to_string() const {
+ std::string result;
+ for (const Block& block : blocks_) {
+ result.append(block.buffer.get(), block.buffer.get() + block.size);
+ }
+ return result;
+}
+
+} // namespace android
diff --git a/libs/androidfw/StringPool.cpp b/libs/androidfw/StringPool.cpp
new file mode 100644
index 0000000..b59e906
--- /dev/null
+++ b/libs/androidfw/StringPool.cpp
@@ -0,0 +1,508 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <androidfw/BigBuffer.h>
+#include <androidfw/StringPool.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "android-base/logging.h"
+#include "androidfw/ResourceTypes.h"
+#include "androidfw/StringPiece.h"
+#include "androidfw/Util.h"
+
+using ::android::StringPiece;
+
+namespace android {
+
+StringPool::Ref::Ref() : entry_(nullptr) {
+}
+
+StringPool::Ref::Ref(const StringPool::Ref& rhs) : entry_(rhs.entry_) {
+ if (entry_ != nullptr) {
+ entry_->ref_++;
+ }
+}
+
+StringPool::Ref::Ref(StringPool::Entry* entry) : entry_(entry) {
+ if (entry_ != nullptr) {
+ entry_->ref_++;
+ }
+}
+
+StringPool::Ref::~Ref() {
+ if (entry_ != nullptr) {
+ entry_->ref_--;
+ }
+}
+
+StringPool::Ref& StringPool::Ref::operator=(const StringPool::Ref& rhs) {
+ if (rhs.entry_ != nullptr) {
+ rhs.entry_->ref_++;
+ }
+
+ if (entry_ != nullptr) {
+ entry_->ref_--;
+ }
+ entry_ = rhs.entry_;
+ return *this;
+}
+
+bool StringPool::Ref::operator==(const Ref& rhs) const {
+ return entry_->value == rhs.entry_->value;
+}
+
+bool StringPool::Ref::operator!=(const Ref& rhs) const {
+ return entry_->value != rhs.entry_->value;
+}
+
+const std::string* StringPool::Ref::operator->() const {
+ return &entry_->value;
+}
+
+const std::string& StringPool::Ref::operator*() const {
+ return entry_->value;
+}
+
+size_t StringPool::Ref::index() const {
+ // Account for the styles, which *always* come first.
+ return entry_->pool_->styles_.size() + entry_->index_;
+}
+
+const StringPool::Context& StringPool::Ref::GetContext() const {
+ return entry_->context;
+}
+
+StringPool::StyleRef::StyleRef() : entry_(nullptr) {
+}
+
+StringPool::StyleRef::StyleRef(const StringPool::StyleRef& rhs) : entry_(rhs.entry_) {
+ if (entry_ != nullptr) {
+ entry_->ref_++;
+ }
+}
+
+StringPool::StyleRef::StyleRef(StringPool::StyleEntry* entry) : entry_(entry) {
+ if (entry_ != nullptr) {
+ entry_->ref_++;
+ }
+}
+
+StringPool::StyleRef::~StyleRef() {
+ if (entry_ != nullptr) {
+ entry_->ref_--;
+ }
+}
+
+StringPool::StyleRef& StringPool::StyleRef::operator=(const StringPool::StyleRef& rhs) {
+ if (rhs.entry_ != nullptr) {
+ rhs.entry_->ref_++;
+ }
+
+ if (entry_ != nullptr) {
+ entry_->ref_--;
+ }
+ entry_ = rhs.entry_;
+ return *this;
+}
+
+bool StringPool::StyleRef::operator==(const StyleRef& rhs) const {
+ if (entry_->value != rhs.entry_->value) {
+ return false;
+ }
+
+ if (entry_->spans.size() != rhs.entry_->spans.size()) {
+ return false;
+ }
+
+ auto rhs_iter = rhs.entry_->spans.begin();
+ for (const Span& span : entry_->spans) {
+ const Span& rhs_span = *rhs_iter;
+ if (span.first_char != rhs_span.first_char || span.last_char != rhs_span.last_char ||
+ span.name != rhs_span.name) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool StringPool::StyleRef::operator!=(const StyleRef& rhs) const {
+ return !operator==(rhs);
+}
+
+const StringPool::StyleEntry* StringPool::StyleRef::operator->() const {
+ return entry_;
+}
+
+const StringPool::StyleEntry& StringPool::StyleRef::operator*() const {
+ return *entry_;
+}
+
+size_t StringPool::StyleRef::index() const {
+ return entry_->index_;
+}
+
+const StringPool::Context& StringPool::StyleRef::GetContext() const {
+ return entry_->context;
+}
+
+StringPool::Ref StringPool::MakeRef(const StringPiece& str) {
+ return MakeRefImpl(str, Context{}, true);
+}
+
+StringPool::Ref StringPool::MakeRef(const StringPiece& str, const Context& context) {
+ return MakeRefImpl(str, context, true);
+}
+
+StringPool::Ref StringPool::MakeRefImpl(const StringPiece& str, const Context& context,
+ bool unique) {
+ if (unique) {
+ auto range = indexed_strings_.equal_range(str);
+ for (auto iter = range.first; iter != range.second; ++iter) {
+ if (context.priority == iter->second->context.priority) {
+ return Ref(iter->second);
+ }
+ }
+ }
+
+ std::unique_ptr<Entry> entry(new Entry());
+ entry->value = str.to_string();
+ entry->context = context;
+ entry->index_ = strings_.size();
+ entry->ref_ = 0;
+ entry->pool_ = this;
+
+ Entry* borrow = entry.get();
+ strings_.emplace_back(std::move(entry));
+ indexed_strings_.insert(std::make_pair(StringPiece(borrow->value), borrow));
+ return Ref(borrow);
+}
+
+StringPool::Ref StringPool::MakeRef(const Ref& ref) {
+ if (ref.entry_->pool_ == this) {
+ return ref;
+ }
+ return MakeRef(ref.entry_->value, ref.entry_->context);
+}
+
+StringPool::StyleRef StringPool::MakeRef(const StyleString& str) {
+ return MakeRef(str, Context{});
+}
+
+StringPool::StyleRef StringPool::MakeRef(const StyleString& str, const Context& context) {
+ std::unique_ptr<StyleEntry> entry(new StyleEntry());
+ entry->value = str.str;
+ entry->context = context;
+ entry->index_ = styles_.size();
+ entry->ref_ = 0;
+ for (const android::Span& span : str.spans) {
+ entry->spans.emplace_back(Span{MakeRef(span.name), span.first_char, span.last_char});
+ }
+
+ StyleEntry* borrow = entry.get();
+ styles_.emplace_back(std::move(entry));
+ return StyleRef(borrow);
+}
+
+StringPool::StyleRef StringPool::MakeRef(const StyleRef& ref) {
+ std::unique_ptr<StyleEntry> entry(new StyleEntry());
+ entry->value = ref.entry_->value;
+ entry->context = ref.entry_->context;
+ entry->index_ = styles_.size();
+ entry->ref_ = 0;
+ for (const Span& span : ref.entry_->spans) {
+ entry->spans.emplace_back(Span{MakeRef(*span.name), span.first_char, span.last_char});
+ }
+
+ StyleEntry* borrow = entry.get();
+ styles_.emplace_back(std::move(entry));
+ return StyleRef(borrow);
+}
+
+void StringPool::ReAssignIndices() {
+ // Assign the style indices.
+ const size_t style_len = styles_.size();
+ for (size_t index = 0; index < style_len; index++) {
+ styles_[index]->index_ = index;
+ }
+
+ // Assign the string indices.
+ const size_t string_len = strings_.size();
+ for (size_t index = 0; index < string_len; index++) {
+ strings_[index]->index_ = index;
+ }
+}
+
+void StringPool::Merge(StringPool&& pool) {
+ // First, change the owning pool for the incoming strings.
+ for (std::unique_ptr<Entry>& entry : pool.strings_) {
+ entry->pool_ = this;
+ }
+
+ // Now move the styles, strings, and indices over.
+ std::move(pool.styles_.begin(), pool.styles_.end(), std::back_inserter(styles_));
+ pool.styles_.clear();
+ std::move(pool.strings_.begin(), pool.strings_.end(), std::back_inserter(strings_));
+ pool.strings_.clear();
+ indexed_strings_.insert(pool.indexed_strings_.begin(), pool.indexed_strings_.end());
+ pool.indexed_strings_.clear();
+
+ ReAssignIndices();
+}
+
+void StringPool::HintWillAdd(size_t string_count, size_t style_count) {
+ strings_.reserve(strings_.size() + string_count);
+ styles_.reserve(styles_.size() + style_count);
+}
+
+void StringPool::Prune() {
+ const auto iter_end = indexed_strings_.end();
+ auto index_iter = indexed_strings_.begin();
+ while (index_iter != iter_end) {
+ if (index_iter->second->ref_ <= 0) {
+ index_iter = indexed_strings_.erase(index_iter);
+ } else {
+ ++index_iter;
+ }
+ }
+
+ auto end_iter2 =
+ std::remove_if(strings_.begin(), strings_.end(),
+ [](const std::unique_ptr<Entry>& entry) -> bool { return entry->ref_ <= 0; });
+ auto end_iter3 = std::remove_if(
+ styles_.begin(), styles_.end(),
+ [](const std::unique_ptr<StyleEntry>& entry) -> bool { return entry->ref_ <= 0; });
+
+ // Remove the entries at the end or else we'll be accessing a deleted string from the StyleEntry.
+ strings_.erase(end_iter2, strings_.end());
+ styles_.erase(end_iter3, styles_.end());
+
+ ReAssignIndices();
+}
+
+template <typename E>
+static void SortEntries(
+ std::vector<std::unique_ptr<E>>& entries,
+ const std::function<int(const StringPool::Context&, const StringPool::Context&)>& cmp) {
+ using UEntry = std::unique_ptr<E>;
+
+ if (cmp != nullptr) {
+ std::sort(entries.begin(), entries.end(), [&cmp](const UEntry& a, const UEntry& b) -> bool {
+ int r = cmp(a->context, b->context);
+ if (r == 0) {
+ r = a->value.compare(b->value);
+ }
+ return r < 0;
+ });
+ } else {
+ std::sort(entries.begin(), entries.end(),
+ [](const UEntry& a, const UEntry& b) -> bool { return a->value < b->value; });
+ }
+}
+
+void StringPool::Sort(const std::function<int(const Context&, const Context&)>& cmp) {
+ SortEntries(styles_, cmp);
+ SortEntries(strings_, cmp);
+ ReAssignIndices();
+}
+
+template <typename T>
+static T* EncodeLength(T* data, size_t length) {
+ static_assert(std::is_integral<T>::value, "wat.");
+
+ constexpr size_t kMask = 1 << ((sizeof(T) * 8) - 1);
+ constexpr size_t kMaxSize = kMask - 1;
+ if (length > kMaxSize) {
+ *data++ = kMask | (kMaxSize & (length >> (sizeof(T) * 8)));
+ }
+ *data++ = length;
+ return data;
+}
+
+/**
+ * Returns the maximum possible string length that can be successfully encoded
+ * using 2 units of the specified T.
+ * EncodeLengthMax<char> -> maximum unit length of 0x7FFF
+ * EncodeLengthMax<char16_t> -> maximum unit length of 0x7FFFFFFF
+ **/
+template <typename T>
+static size_t EncodeLengthMax() {
+ static_assert(std::is_integral<T>::value, "wat.");
+
+ constexpr size_t kMask = 1 << ((sizeof(T) * 8 * 2) - 1);
+ constexpr size_t max = kMask - 1;
+ return max;
+}
+
+/**
+ * Returns the number of units (1 or 2) needed to encode the string length
+ * before writing the string.
+ */
+template <typename T>
+static size_t EncodedLengthUnits(size_t length) {
+ static_assert(std::is_integral<T>::value, "wat.");
+
+ constexpr size_t kMask = 1 << ((sizeof(T) * 8) - 1);
+ constexpr size_t kMaxSize = kMask - 1;
+ return length > kMaxSize ? 2 : 1;
+}
+
+const std::string kStringTooLarge = "STRING_TOO_LARGE";
+
+static bool EncodeString(const std::string& str, const bool utf8, BigBuffer* out,
+ IDiagnostics* diag) {
+ if (utf8) {
+ const std::string& encoded = util::Utf8ToModifiedUtf8(str);
+ const ssize_t utf16_length =
+ utf8_to_utf16_length(reinterpret_cast<const uint8_t*>(encoded.data()), encoded.size());
+ CHECK(utf16_length >= 0);
+
+ // Make sure the lengths to be encoded do not exceed the maximum length that
+ // can be encoded using chars
+ if ((((size_t)encoded.size()) > EncodeLengthMax<char>()) ||
+ (((size_t)utf16_length) > EncodeLengthMax<char>())) {
+ diag->Error(DiagMessage() << "string too large to encode using UTF-8 "
+ << "written instead as '" << kStringTooLarge << "'");
+
+ EncodeString(kStringTooLarge, utf8, out, diag);
+ return false;
+ }
+
+ const size_t total_size = EncodedLengthUnits<char>(utf16_length) +
+ EncodedLengthUnits<char>(encoded.size()) + encoded.size() + 1;
+
+ char* data = out->NextBlock<char>(total_size);
+
+ // First encode the UTF16 string length.
+ data = EncodeLength(data, utf16_length);
+
+ // Now encode the size of the real UTF8 string.
+ data = EncodeLength(data, encoded.size());
+ strncpy(data, encoded.data(), encoded.size());
+
+ } else {
+ const std::u16string encoded = util::Utf8ToUtf16(str);
+ const ssize_t utf16_length = encoded.size();
+
+ // Make sure the length to be encoded does not exceed the maximum possible
+ // length that can be encoded
+ if (((size_t)utf16_length) > EncodeLengthMax<char16_t>()) {
+ diag->Error(DiagMessage() << "string too large to encode using UTF-16 "
+ << "written instead as '" << kStringTooLarge << "'");
+
+ EncodeString(kStringTooLarge, utf8, out, diag);
+ return false;
+ }
+
+ // Total number of 16-bit words to write.
+ const size_t total_size = EncodedLengthUnits<char16_t>(utf16_length) + encoded.size() + 1;
+
+ char16_t* data = out->NextBlock<char16_t>(total_size);
+
+ // Encode the actual UTF16 string length.
+ data = EncodeLength(data, utf16_length);
+ const size_t byte_length = encoded.size() * sizeof(char16_t);
+
+ // NOTE: For some reason, strncpy16(data, entry->value.data(),
+ // entry->value.size()) truncates the string.
+ memcpy(data, encoded.data(), byte_length);
+
+ // The null-terminating character is already here due to the block of data
+ // being set to 0s on allocation.
+ }
+
+ return true;
+}
+
+bool StringPool::Flatten(BigBuffer* out, const StringPool& pool, bool utf8, IDiagnostics* diag) {
+ bool no_error = true;
+ const size_t start_index = out->size();
+ android::ResStringPool_header* header = out->NextBlock<android::ResStringPool_header>();
+ header->header.type = util::HostToDevice16(android::RES_STRING_POOL_TYPE);
+ header->header.headerSize = util::HostToDevice16(sizeof(*header));
+ header->stringCount = util::HostToDevice32(pool.size());
+ header->styleCount = util::HostToDevice32(pool.styles_.size());
+ if (utf8) {
+ header->flags |= android::ResStringPool_header::UTF8_FLAG;
+ }
+
+ uint32_t* indices = pool.size() != 0 ? out->NextBlock<uint32_t>(pool.size()) : nullptr;
+ uint32_t* style_indices =
+ pool.styles_.size() != 0 ? out->NextBlock<uint32_t>(pool.styles_.size()) : nullptr;
+
+ const size_t before_strings_index = out->size();
+ header->stringsStart = before_strings_index - start_index;
+
+ // Styles always come first.
+ for (const std::unique_ptr<StyleEntry>& entry : pool.styles_) {
+ *indices++ = out->size() - before_strings_index;
+ no_error = EncodeString(entry->value, utf8, out, diag) && no_error;
+ }
+
+ for (const std::unique_ptr<Entry>& entry : pool.strings_) {
+ *indices++ = out->size() - before_strings_index;
+ no_error = EncodeString(entry->value, utf8, out, diag) && no_error;
+ }
+
+ out->Align4();
+
+ if (style_indices != nullptr) {
+ const size_t before_styles_index = out->size();
+ header->stylesStart = util::HostToDevice32(before_styles_index - start_index);
+
+ for (const std::unique_ptr<StyleEntry>& entry : pool.styles_) {
+ *style_indices++ = out->size() - before_styles_index;
+
+ if (!entry->spans.empty()) {
+ android::ResStringPool_span* span =
+ out->NextBlock<android::ResStringPool_span>(entry->spans.size());
+ for (const Span& s : entry->spans) {
+ span->name.index = util::HostToDevice32(s.name.index());
+ span->firstChar = util::HostToDevice32(s.first_char);
+ span->lastChar = util::HostToDevice32(s.last_char);
+ span++;
+ }
+ }
+
+ uint32_t* spanEnd = out->NextBlock<uint32_t>();
+ *spanEnd = android::ResStringPool_span::END;
+ }
+
+ // The error checking code in the platform looks for an entire
+ // ResStringPool_span structure worth of 0xFFFFFFFF at the end
+ // of the style block, so fill in the remaining 2 32bit words
+ // with 0xFFFFFFFF.
+ const size_t padding_length =
+ sizeof(android::ResStringPool_span) - sizeof(android::ResStringPool_span::name);
+ uint8_t* padding = out->NextBlock<uint8_t>(padding_length);
+ memset(padding, 0xff, padding_length);
+ out->Align4();
+ }
+ header->header.size = util::HostToDevice32(out->size() - start_index);
+ return no_error;
+}
+
+bool StringPool::FlattenUtf8(BigBuffer* out, const StringPool& pool, IDiagnostics* diag) {
+ return Flatten(out, pool, true, diag);
+}
+
+bool StringPool::FlattenUtf16(BigBuffer* out, const StringPool& pool, IDiagnostics* diag) {
+ return Flatten(out, pool, false, diag);
+}
+
+} // namespace android
diff --git a/libs/androidfw/Util.cpp b/libs/androidfw/Util.cpp
index 59c9d64..be9edc4 100644
--- a/libs/androidfw/Util.cpp
+++ b/libs/androidfw/Util.cpp
@@ -68,6 +68,107 @@
return utf8;
}
+std::string Utf8ToModifiedUtf8(const std::string& utf8) {
+ // Java uses Modified UTF-8 which only supports the 1, 2, and 3 byte formats of UTF-8. To encode
+ // 4 byte UTF-8 codepoints, Modified UTF-8 allows the use of surrogate pairs in the same format
+ // of CESU-8 surrogate pairs. Calculate the size of the utf8 string with all 4 byte UTF-8
+ // codepoints replaced with 2 3 byte surrogate pairs
+ size_t modified_size = 0;
+ const size_t size = utf8.size();
+ for (size_t i = 0; i < size; i++) {
+ if (((uint8_t)utf8[i] >> 4) == 0xF) {
+ modified_size += 6;
+ i += 3;
+ } else {
+ modified_size++;
+ }
+ }
+
+ // Early out if no 4 byte codepoints are found
+ if (size == modified_size) {
+ return utf8;
+ }
+
+ std::string output;
+ output.reserve(modified_size);
+ for (size_t i = 0; i < size; i++) {
+ if (((uint8_t)utf8[i] >> 4) == 0xF) {
+ int32_t codepoint = utf32_from_utf8_at(utf8.data(), size, i, nullptr);
+
+ // Calculate the high and low surrogates as UTF-16 would
+ int32_t high = ((codepoint - 0x10000) / 0x400) + 0xD800;
+ int32_t low = ((codepoint - 0x10000) % 0x400) + 0xDC00;
+
+ // Encode each surrogate in UTF-8
+ output.push_back((char)(0xE4 | ((high >> 12) & 0xF)));
+ output.push_back((char)(0x80 | ((high >> 6) & 0x3F)));
+ output.push_back((char)(0x80 | (high & 0x3F)));
+ output.push_back((char)(0xE4 | ((low >> 12) & 0xF)));
+ output.push_back((char)(0x80 | ((low >> 6) & 0x3F)));
+ output.push_back((char)(0x80 | (low & 0x3F)));
+ i += 3;
+ } else {
+ output.push_back(utf8[i]);
+ }
+ }
+
+ return output;
+}
+
+std::string ModifiedUtf8ToUtf8(const std::string& modified_utf8) {
+ // The UTF-8 representation will have a byte length less than or equal to the Modified UTF-8
+ // representation.
+ std::string output;
+ output.reserve(modified_utf8.size());
+
+ size_t index = 0;
+ const size_t modified_size = modified_utf8.size();
+ while (index < modified_size) {
+ size_t next_index;
+ int32_t high_surrogate =
+ utf32_from_utf8_at(modified_utf8.data(), modified_size, index, &next_index);
+ if (high_surrogate < 0) {
+ return {};
+ }
+
+ // Check that the first codepoint is within the high surrogate range
+ if (high_surrogate >= 0xD800 && high_surrogate <= 0xDB7F) {
+ int32_t low_surrogate =
+ utf32_from_utf8_at(modified_utf8.data(), modified_size, next_index, &next_index);
+ if (low_surrogate < 0) {
+ return {};
+ }
+
+ // Check that the second codepoint is within the low surrogate range
+ if (low_surrogate >= 0xDC00 && low_surrogate <= 0xDFFF) {
+ const char32_t codepoint =
+ (char32_t)(((high_surrogate - 0xD800) * 0x400) + (low_surrogate - 0xDC00) + 0x10000);
+
+ // The decoded codepoint should represent a 4 byte, UTF-8 character
+ const size_t utf8_length = (size_t)utf32_to_utf8_length(&codepoint, 1);
+ if (utf8_length != 4) {
+ return {};
+ }
+
+ // Encode the UTF-8 representation of the codepoint into the string
+ char* start = &output[output.size()];
+ output.resize(output.size() + utf8_length);
+ utf32_to_utf8((char32_t*)&codepoint, 1, start, utf8_length + 1);
+
+ index = next_index;
+ continue;
+ }
+ }
+
+ // Append non-surrogate pairs to the output string
+ for (size_t i = index; i < next_index; i++) {
+ output.push_back(modified_utf8[i]);
+ }
+ index = next_index;
+ }
+ return output;
+}
+
static std::vector<std::string> SplitAndTransform(
const StringPiece& str, char sep, const std::function<char(char)>& f) {
std::vector<std::string> parts;
@@ -90,6 +191,29 @@
return SplitAndTransform(str, sep, ::tolower);
}
+std::unique_ptr<uint8_t[]> Copy(const BigBuffer& buffer) {
+ std::unique_ptr<uint8_t[]> data = std::unique_ptr<uint8_t[]>(new uint8_t[buffer.size()]);
+ uint8_t* p = data.get();
+ for (const auto& block : buffer) {
+ memcpy(p, block.buffer.get(), block.size);
+ p += block.size;
+ }
+ return data;
+}
+
+StringPiece16 GetString16(const android::ResStringPool& pool, size_t idx) {
+ if (auto str = pool.stringAt(idx); str.ok()) {
+ return *str;
+ }
+ return StringPiece16();
+}
+
+std::string GetString(const android::ResStringPool& pool, size_t idx) {
+ if (auto str = pool.string8At(idx); str.ok()) {
+ return ModifiedUtf8ToUtf8(str->to_string());
+ }
+ return Utf16ToUtf8(GetString16(pool, idx));
+}
} // namespace util
} // namespace android
diff --git a/libs/androidfw/include/androidfw/BigBuffer.h b/libs/androidfw/include/androidfw/BigBuffer.h
new file mode 100644
index 0000000..b99a4ed
--- /dev/null
+++ b/libs/androidfw/include/androidfw/BigBuffer.h
@@ -0,0 +1,192 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _ANDROID_BIG_BUFFER_H
+#define _ANDROID_BIG_BUFFER_H
+
+#include <cstring>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "android-base/logging.h"
+#include "android-base/macros.h"
+
+namespace android {
+
+/**
+ * Inspired by protobuf's ZeroCopyOutputStream, offers blocks of memory
+ * in which to write without knowing the full size of the entire payload.
+ * This is essentially a list of memory blocks. As one fills up, another
+ * block is allocated and appended to the end of the list.
+ */
+class BigBuffer {
+ public:
+ /**
+ * A contiguous block of allocated memory.
+ */
+ struct Block {
+ /**
+ * Pointer to the memory.
+ */
+ std::unique_ptr<uint8_t[]> buffer;
+
+ /**
+ * Size of memory that is currently occupied. The actual
+ * allocation may be larger.
+ */
+ size_t size;
+
+ private:
+ friend class BigBuffer;
+
+ /**
+ * The size of the memory block allocation.
+ */
+ size_t block_size_;
+ };
+
+ typedef std::vector<Block>::const_iterator const_iterator;
+
+ /**
+ * Create a BigBuffer with block allocation sizes
+ * of block_size.
+ */
+ explicit BigBuffer(size_t block_size);
+
+ BigBuffer(BigBuffer&& rhs) noexcept;
+
+ /**
+ * Number of occupied bytes in all the allocated blocks.
+ */
+ size_t size() const;
+
+ /**
+ * Returns a pointer to an array of T, where T is
+ * a POD type. The elements are zero-initialized.
+ */
+ template <typename T>
+ T* NextBlock(size_t count = 1);
+
+ /**
+ * Returns the next block available and puts the size in out_count.
+ * This is useful for grabbing blocks where the size doesn't matter.
+ * Use BackUp() to give back any bytes that were not used.
+ */
+ void* NextBlock(size_t* out_count);
+
+ /**
+ * Backs up count bytes. This must only be called after NextBlock()
+ * and can not be larger than sizeof(T) * count of the last NextBlock()
+ * call.
+ */
+ void BackUp(size_t count);
+
+ /**
+ * Moves the specified BigBuffer into this one. When this method
+ * returns, buffer is empty.
+ */
+ void AppendBuffer(BigBuffer&& buffer);
+
+ /**
+ * Pads the block with 'bytes' bytes of zero values.
+ */
+ void Pad(size_t bytes);
+
+ /**
+ * Pads the block so that it aligns on a 4 byte boundary.
+ */
+ void Align4();
+
+ size_t block_size() const;
+
+ const_iterator begin() const;
+ const_iterator end() const;
+
+ std::string to_string() const;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(BigBuffer);
+
+ /**
+ * Returns a pointer to a buffer of the requested size.
+ * The buffer is zero-initialized.
+ */
+ void* NextBlockImpl(size_t size);
+
+ size_t block_size_;
+ size_t size_;
+ std::vector<Block> blocks_;
+};
+
+inline BigBuffer::BigBuffer(size_t block_size) : block_size_(block_size), size_(0) {
+}
+
+inline BigBuffer::BigBuffer(BigBuffer&& rhs) noexcept
+ : block_size_(rhs.block_size_), size_(rhs.size_), blocks_(std::move(rhs.blocks_)) {
+}
+
+inline size_t BigBuffer::size() const {
+ return size_;
+}
+
+inline size_t BigBuffer::block_size() const {
+ return block_size_;
+}
+
+template <typename T>
+inline T* BigBuffer::NextBlock(size_t count) {
+ static_assert(std::is_standard_layout<T>::value, "T must be standard_layout type");
+ CHECK(count != 0);
+ return reinterpret_cast<T*>(NextBlockImpl(sizeof(T) * count));
+}
+
+inline void BigBuffer::BackUp(size_t count) {
+ Block& block = blocks_.back();
+ block.size -= count;
+ size_ -= count;
+}
+
+inline void BigBuffer::AppendBuffer(BigBuffer&& buffer) {
+ std::move(buffer.blocks_.begin(), buffer.blocks_.end(), std::back_inserter(blocks_));
+ size_ += buffer.size_;
+ buffer.blocks_.clear();
+ buffer.size_ = 0;
+}
+
+inline void BigBuffer::Pad(size_t bytes) {
+ NextBlock<char>(bytes);
+}
+
+inline void BigBuffer::Align4() {
+ const size_t unaligned = size_ % 4;
+ if (unaligned != 0) {
+ Pad(4 - unaligned);
+ }
+}
+
+inline BigBuffer::const_iterator BigBuffer::begin() const {
+ return blocks_.begin();
+}
+
+inline BigBuffer::const_iterator BigBuffer::end() const {
+ return blocks_.end();
+}
+
+} // namespace android
+
+#endif // _ANDROID_BIG_BUFFER_H
diff --git a/libs/androidfw/include/androidfw/IDiagnostics.h b/libs/androidfw/include/androidfw/IDiagnostics.h
new file mode 100644
index 0000000..273b05a
--- /dev/null
+++ b/libs/androidfw/include/androidfw/IDiagnostics.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _ANDROID_DIAGNOSTICS_H
+#define _ANDROID_DIAGNOSTICS_H
+
+#include <sstream>
+#include <string>
+
+#include "Source.h"
+#include "android-base/macros.h"
+#include "androidfw/StringPiece.h"
+
+namespace android {
+
+struct DiagMessageActual {
+ Source source;
+ std::string message;
+};
+
+struct DiagMessage {
+ public:
+ DiagMessage() = default;
+
+ explicit DiagMessage(const android::StringPiece& src) : source_(src) {
+ }
+
+ explicit DiagMessage(const Source& src) : source_(src) {
+ }
+
+ explicit DiagMessage(size_t line) : source_(Source().WithLine(line)) {
+ }
+
+ template <typename T>
+ DiagMessage& operator<<(const T& value) {
+ message_ << value;
+ return *this;
+ }
+
+ DiagMessageActual Build() const {
+ return DiagMessageActual{source_, message_.str()};
+ }
+
+ private:
+ Source source_;
+ std::stringstream message_;
+};
+
+template <>
+inline DiagMessage& DiagMessage::operator<<(const ::std::u16string& value) {
+ message_ << android::StringPiece16(value);
+ return *this;
+}
+
+struct IDiagnostics {
+ virtual ~IDiagnostics() = default;
+
+ enum class Level { Note, Warn, Error };
+
+ virtual void Log(Level level, DiagMessageActual& actualMsg) = 0;
+
+ virtual void Error(const DiagMessage& message) {
+ DiagMessageActual actual = message.Build();
+ Log(Level::Error, actual);
+ }
+
+ virtual void Warn(const DiagMessage& message) {
+ DiagMessageActual actual = message.Build();
+ Log(Level::Warn, actual);
+ }
+
+ virtual void Note(const DiagMessage& message) {
+ DiagMessageActual actual = message.Build();
+ Log(Level::Note, actual);
+ }
+};
+
+class SourcePathDiagnostics : public IDiagnostics {
+ public:
+ SourcePathDiagnostics(const Source& src, IDiagnostics* diag) : source_(src), diag_(diag) {
+ }
+
+ void Log(Level level, DiagMessageActual& actual_msg) override {
+ actual_msg.source.path = source_.path;
+ diag_->Log(level, actual_msg);
+ if (level == Level::Error) {
+ error = true;
+ }
+ }
+
+ bool HadError() {
+ return error;
+ }
+
+ private:
+ Source source_;
+ IDiagnostics* diag_;
+ bool error = false;
+
+ DISALLOW_COPY_AND_ASSIGN(SourcePathDiagnostics);
+};
+
+class NoOpDiagnostics : public IDiagnostics {
+ public:
+ NoOpDiagnostics() = default;
+
+ void Log(Level level, DiagMessageActual& actual_msg) override {
+ (void)level;
+ (void)actual_msg;
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(NoOpDiagnostics);
+};
+
+} // namespace android
+
+#endif /* _ANDROID_DIAGNOSTICS_H */
diff --git a/libs/androidfw/include/androidfw/Source.h b/libs/androidfw/include/androidfw/Source.h
new file mode 100644
index 0000000..0421a91
--- /dev/null
+++ b/libs/androidfw/include/androidfw/Source.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _ANDROID_SOURCE_H
+#define _ANDROID_SOURCE_H
+
+#include <optional>
+#include <ostream>
+#include <string>
+
+#include "android-base/stringprintf.h"
+#include "androidfw/StringPiece.h"
+
+namespace android {
+
+// Represents a file on disk. Used for logging and showing errors.
+struct Source {
+ std::string path;
+ std::optional<size_t> line;
+ std::optional<std::string> archive;
+
+ Source() = default;
+
+ inline Source(const android::StringPiece& path) : path(path.to_string()) { // NOLINT(implicit)
+ }
+
+ inline Source(const android::StringPiece& path, const android::StringPiece& archive)
+ : path(path.to_string()), archive(archive.to_string()) {
+ }
+
+ inline Source(const android::StringPiece& path, size_t line)
+ : path(path.to_string()), line(line) {
+ }
+
+ inline Source WithLine(size_t line) const {
+ return Source(path, line);
+ }
+
+ std::string to_string() const {
+ std::string s = path;
+ if (archive) {
+ s = ::android::base::StringPrintf("%s@%s", archive.value().c_str(), s.c_str());
+ }
+ if (line) {
+ s = ::android::base::StringPrintf("%s:%zd", s.c_str(), line.value());
+ }
+ return s;
+ }
+};
+
+//
+// Implementations
+//
+
+inline ::std::ostream& operator<<(::std::ostream& out, const Source& source) {
+ return out << source.to_string();
+}
+
+inline bool operator==(const Source& lhs, const Source& rhs) {
+ return lhs.path == rhs.path && lhs.line == rhs.line;
+}
+
+inline bool operator<(const Source& lhs, const Source& rhs) {
+ int cmp = lhs.path.compare(rhs.path);
+ if (cmp < 0) return true;
+ if (cmp > 0) return false;
+ if (lhs.line) {
+ if (rhs.line) {
+ return lhs.line.value() < rhs.line.value();
+ }
+ return false;
+ }
+ return bool(rhs.line);
+}
+
+} // namespace android
+
+#endif // _ANDROID_SOURCE_H
diff --git a/libs/androidfw/include/androidfw/StringPool.h b/libs/androidfw/include/androidfw/StringPool.h
new file mode 100644
index 0000000..25174d8
--- /dev/null
+++ b/libs/androidfw/include/androidfw/StringPool.h
@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _ANDROID_STRING_POOL_H
+#define _ANDROID_STRING_POOL_H
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "BigBuffer.h"
+#include "IDiagnostics.h"
+#include "android-base/macros.h"
+#include "androidfw/ConfigDescription.h"
+#include "androidfw/StringPiece.h"
+
+namespace android {
+
+struct Span {
+ std::string name;
+ uint32_t first_char;
+ uint32_t last_char;
+
+ bool operator==(const Span& right) const {
+ return name == right.name && first_char == right.first_char && last_char == right.last_char;
+ }
+};
+
+struct StyleString {
+ std::string str;
+ std::vector<Span> spans;
+};
+
+// A StringPool for storing the value of String and StyledString resources.
+// Styles and Strings are stored separately, since the runtime variant of this
+// class -- ResStringPool -- requires that styled strings *always* appear first, since their
+// style data is stored as an array indexed by the same indices as the main string pool array.
+// Otherwise, the style data array would have to be sparse and take up more space.
+class StringPool {
+ public:
+ using size_type = size_t;
+
+ class Context {
+ public:
+ enum : uint32_t {
+ kHighPriority = 1u,
+ kNormalPriority = 0x7fffffffu,
+ kLowPriority = 0xffffffffu,
+ };
+ uint32_t priority = kNormalPriority;
+ android::ConfigDescription config;
+
+ Context() = default;
+ Context(uint32_t p, const android::ConfigDescription& c) : priority(p), config(c) {
+ }
+ explicit Context(uint32_t p) : priority(p) {
+ }
+ explicit Context(const android::ConfigDescription& c) : priority(kNormalPriority), config(c) {
+ }
+ };
+
+ class Entry;
+
+ class Ref {
+ public:
+ Ref();
+ Ref(const Ref&);
+ ~Ref();
+
+ Ref& operator=(const Ref& rhs);
+ bool operator==(const Ref& rhs) const;
+ bool operator!=(const Ref& rhs) const;
+ const std::string* operator->() const;
+ const std::string& operator*() const;
+
+ size_t index() const;
+ const Context& GetContext() const;
+
+ private:
+ friend class StringPool;
+
+ explicit Ref(Entry* entry);
+
+ Entry* entry_;
+ };
+
+ class StyleEntry;
+
+ class StyleRef {
+ public:
+ StyleRef();
+ StyleRef(const StyleRef&);
+ ~StyleRef();
+
+ StyleRef& operator=(const StyleRef& rhs);
+ bool operator==(const StyleRef& rhs) const;
+ bool operator!=(const StyleRef& rhs) const;
+ const StyleEntry* operator->() const;
+ const StyleEntry& operator*() const;
+
+ size_t index() const;
+ const Context& GetContext() const;
+
+ private:
+ friend class StringPool;
+
+ explicit StyleRef(StyleEntry* entry);
+
+ StyleEntry* entry_;
+ };
+
+ class Entry {
+ public:
+ std::string value;
+ Context context;
+
+ private:
+ friend class StringPool;
+ friend class Ref;
+
+ size_t index_;
+ int ref_;
+ const StringPool* pool_;
+ };
+
+ struct Span {
+ Ref name;
+ uint32_t first_char;
+ uint32_t last_char;
+ };
+
+ class StyleEntry {
+ public:
+ std::string value;
+ Context context;
+ std::vector<Span> spans;
+
+ private:
+ friend class StringPool;
+ friend class StyleRef;
+
+ size_t index_;
+ int ref_;
+ };
+
+ static bool FlattenUtf8(BigBuffer* out, const StringPool& pool, IDiagnostics* diag);
+ static bool FlattenUtf16(BigBuffer* out, const StringPool& pool, IDiagnostics* diag);
+
+ StringPool() = default;
+ StringPool(StringPool&&) = default;
+ StringPool& operator=(StringPool&&) = default;
+
+ // Adds a string to the pool, unless it already exists. Returns a reference to the string in the
+ // pool.
+ Ref MakeRef(const android::StringPiece& str);
+
+ // Adds a string to the pool, unless it already exists, with a context object that can be used
+ // when sorting the string pool. Returns a reference to the string in the pool.
+ Ref MakeRef(const android::StringPiece& str, const Context& context);
+
+ // Adds a string from another string pool. Returns a reference to the string in the string pool.
+ Ref MakeRef(const Ref& ref);
+
+ // Adds a style to the string pool and returns a reference to it.
+ StyleRef MakeRef(const StyleString& str);
+
+ // Adds a style to the string pool with a context object that can be used when sorting the string
+ // pool. Returns a reference to the style in the string pool.
+ StyleRef MakeRef(const StyleString& str, const Context& context);
+
+ // Adds a style from another string pool. Returns a reference to the style in the string pool.
+ StyleRef MakeRef(const StyleRef& ref);
+
+ // Moves pool into this one without coalescing strings. When this function returns, pool will be
+ // empty.
+ void Merge(StringPool&& pool);
+
+ inline const std::vector<std::unique_ptr<Entry>>& strings() const {
+ return strings_;
+ }
+
+ // Returns the number of strings in the table.
+ inline size_t size() const {
+ return styles_.size() + strings_.size();
+ }
+
+ // Reserves space for strings and styles as an optimization.
+ void HintWillAdd(size_t string_count, size_t style_count);
+
+ // Sorts the strings according to their Context using some comparison function.
+ // Equal Contexts are further sorted by string value, lexicographically.
+ // If no comparison function is provided, values are only sorted lexicographically.
+ void Sort(const std::function<int(const Context&, const Context&)>& cmp = nullptr);
+
+ // Removes any strings that have no references.
+ void Prune();
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(StringPool);
+
+ static bool Flatten(BigBuffer* out, const StringPool& pool, bool utf8, IDiagnostics* diag);
+
+ Ref MakeRefImpl(const android::StringPiece& str, const Context& context, bool unique);
+ void ReAssignIndices();
+
+ std::vector<std::unique_ptr<Entry>> strings_;
+ std::vector<std::unique_ptr<StyleEntry>> styles_;
+ std::unordered_multimap<android::StringPiece, Entry*> indexed_strings_;
+};
+
+} // namespace android
+
+#endif // _ANDROID_STRING_POOL_H
diff --git a/libs/androidfw/include/androidfw/Util.h b/libs/androidfw/include/androidfw/Util.h
index c59b5b6..1bbc7f5 100644
--- a/libs/androidfw/include/androidfw/Util.h
+++ b/libs/androidfw/include/androidfw/Util.h
@@ -17,15 +17,18 @@
#ifndef UTIL_H_
#define UTIL_H_
+#include <android-base/macros.h>
+#include <util/map_ptr.h>
+
#include <cstdlib>
#include <memory>
#include <sstream>
#include <vector>
-#include <android-base/macros.h>
-#include <util/map_ptr.h>
-
+#include "androidfw/BigBuffer.h"
+#include "androidfw/ResourceTypes.h"
#include "androidfw/StringPiece.h"
+#include "utils/ByteOrder.h"
#ifdef __ANDROID__
#define ANDROID_LOG(x) LOG(x)
@@ -125,6 +128,28 @@
// Converts a UTF-16 string to a UTF-8 string.
std::string Utf16ToUtf8(const StringPiece16& utf16);
+// Converts a UTF8 string into Modified UTF8
+std::string Utf8ToModifiedUtf8(const std::string& utf8);
+
+// Converts a Modified UTF8 string into a UTF8 string
+std::string ModifiedUtf8ToUtf8(const std::string& modified_utf8);
+
+inline uint16_t HostToDevice16(uint16_t value) {
+ return htods(value);
+}
+
+inline uint32_t HostToDevice32(uint32_t value) {
+ return htodl(value);
+}
+
+inline uint16_t DeviceToHost16(uint16_t value) {
+ return dtohs(value);
+}
+
+inline uint32_t DeviceToHost32(uint32_t value) {
+ return dtohl(value);
+}
+
std::vector<std::string> SplitAndLowercase(const android::StringPiece& str, char sep);
template <typename T>
@@ -136,6 +161,18 @@
return ((size_t)data & 0x3U) == 0;
}
+// Helper method to extract a UTF-16 string from a StringPool. If the string is stored as UTF-8,
+// the conversion to UTF-16 happens within ResStringPool.
+android::StringPiece16 GetString16(const android::ResStringPool& pool, size_t idx);
+
+// Helper method to extract a UTF-8 string from a StringPool. If the string is stored as UTF-16,
+// the conversion from UTF-16 to UTF-8 does not happen in ResStringPool and is done by this method,
+// which maintains no state or cache. This means we must return an std::string copy.
+std::string GetString(const android::ResStringPool& pool, size_t idx);
+
+// Copies the entire BigBuffer into a single buffer.
+std::unique_ptr<uint8_t[]> Copy(const android::BigBuffer& buffer);
+
} // namespace util
} // namespace android
diff --git a/libs/androidfw/tests/BigBuffer_test.cpp b/libs/androidfw/tests/BigBuffer_test.cpp
new file mode 100644
index 0000000..382d21e
--- /dev/null
+++ b/libs/androidfw/tests/BigBuffer_test.cpp
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "androidfw/BigBuffer.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::NotNull;
+
+namespace android {
+
+TEST(BigBufferTest, AllocateSingleBlock) {
+ BigBuffer buffer(4);
+
+ EXPECT_THAT(buffer.NextBlock<char>(2), NotNull());
+ EXPECT_EQ(2u, buffer.size());
+}
+
+TEST(BigBufferTest, ReturnSameBlockIfNextAllocationFits) {
+ BigBuffer buffer(16);
+
+ char* b1 = buffer.NextBlock<char>(8);
+ EXPECT_THAT(b1, NotNull());
+
+ char* b2 = buffer.NextBlock<char>(4);
+ EXPECT_THAT(b2, NotNull());
+
+ EXPECT_EQ(b1 + 8, b2);
+}
+
+TEST(BigBufferTest, AllocateExactSizeBlockIfLargerThanBlockSize) {
+ BigBuffer buffer(16);
+
+ EXPECT_THAT(buffer.NextBlock<char>(32), NotNull());
+ EXPECT_EQ(32u, buffer.size());
+}
+
+TEST(BigBufferTest, AppendAndMoveBlock) {
+ BigBuffer buffer(16);
+
+ uint32_t* b1 = buffer.NextBlock<uint32_t>();
+ ASSERT_THAT(b1, NotNull());
+ *b1 = 33;
+
+ {
+ BigBuffer buffer2(16);
+ b1 = buffer2.NextBlock<uint32_t>();
+ ASSERT_THAT(b1, NotNull());
+ *b1 = 44;
+
+ buffer.AppendBuffer(std::move(buffer2));
+ EXPECT_EQ(0u, buffer2.size()); // NOLINT
+ EXPECT_EQ(buffer2.begin(), buffer2.end());
+ }
+
+ EXPECT_EQ(2 * sizeof(uint32_t), buffer.size());
+
+ auto b = buffer.begin();
+ ASSERT_NE(b, buffer.end());
+ ASSERT_EQ(sizeof(uint32_t), b->size);
+ ASSERT_EQ(33u, *reinterpret_cast<uint32_t*>(b->buffer.get()));
+ ++b;
+
+ ASSERT_NE(b, buffer.end());
+ ASSERT_EQ(sizeof(uint32_t), b->size);
+ ASSERT_EQ(44u, *reinterpret_cast<uint32_t*>(b->buffer.get()));
+ ++b;
+
+ ASSERT_EQ(b, buffer.end());
+}
+
+TEST(BigBufferTest, PadAndAlignProperly) {
+ BigBuffer buffer(16);
+
+ ASSERT_THAT(buffer.NextBlock<char>(2), NotNull());
+ ASSERT_EQ(2u, buffer.size());
+ buffer.Pad(2);
+ ASSERT_EQ(4u, buffer.size());
+ buffer.Align4();
+ ASSERT_EQ(4u, buffer.size());
+ buffer.Pad(2);
+ ASSERT_EQ(6u, buffer.size());
+ buffer.Align4();
+ ASSERT_EQ(8u, buffer.size());
+}
+
+} // namespace android
diff --git a/libs/androidfw/tests/StringPool_test.cpp b/libs/androidfw/tests/StringPool_test.cpp
new file mode 100644
index 0000000..047d457
--- /dev/null
+++ b/libs/androidfw/tests/StringPool_test.cpp
@@ -0,0 +1,388 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "androidfw/StringPool.h"
+
+#include <string>
+
+#include "androidfw/IDiagnostics.h"
+#include "androidfw/StringPiece.h"
+#include "androidfw/Util.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::android::StringPiece;
+using ::android::StringPiece16;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::NotNull;
+using ::testing::Pointee;
+
+namespace android {
+
+TEST(StringPoolTest, InsertOneString) {
+ StringPool pool;
+
+ StringPool::Ref ref = pool.MakeRef("wut");
+ EXPECT_THAT(*ref, Eq("wut"));
+}
+
+TEST(StringPoolTest, InsertTwoUniqueStrings) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("wut");
+ StringPool::Ref ref_b = pool.MakeRef("hey");
+
+ EXPECT_THAT(*ref_a, Eq("wut"));
+ EXPECT_THAT(*ref_b, Eq("hey"));
+}
+
+TEST(StringPoolTest, DoNotInsertNewDuplicateString) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("wut");
+ StringPool::Ref ref_b = pool.MakeRef("wut");
+
+ EXPECT_THAT(*ref_a, Eq("wut"));
+ EXPECT_THAT(*ref_b, Eq("wut"));
+ EXPECT_THAT(pool.size(), Eq(1u));
+}
+
+TEST(StringPoolTest, DoNotDedupeSameStringDifferentPriority) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("wut", StringPool::Context(0x81010001));
+ StringPool::Ref ref_b = pool.MakeRef("wut", StringPool::Context(0x81010002));
+
+ EXPECT_THAT(*ref_a, Eq("wut"));
+ EXPECT_THAT(*ref_b, Eq("wut"));
+ EXPECT_THAT(pool.size(), Eq(2u));
+}
+
+TEST(StringPoolTest, MaintainInsertionOrderIndex) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("z");
+ StringPool::Ref ref_b = pool.MakeRef("a");
+ StringPool::Ref ref_c = pool.MakeRef("m");
+
+ EXPECT_THAT(ref_a.index(), Eq(0u));
+ EXPECT_THAT(ref_b.index(), Eq(1u));
+ EXPECT_THAT(ref_c.index(), Eq(2u));
+}
+
+TEST(StringPoolTest, PruneStringsWithNoReferences) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("foo");
+
+ {
+ StringPool::Ref ref_b = pool.MakeRef("wut");
+ EXPECT_THAT(*ref_b, Eq("wut"));
+ EXPECT_THAT(pool.size(), Eq(2u));
+ pool.Prune();
+ EXPECT_THAT(pool.size(), Eq(2u));
+ }
+ EXPECT_THAT(pool.size(), Eq(2u));
+
+ {
+ StringPool::Ref ref_c = pool.MakeRef("bar");
+ EXPECT_THAT(pool.size(), Eq(3u));
+
+ pool.Prune();
+ EXPECT_THAT(pool.size(), Eq(2u));
+ }
+ EXPECT_THAT(pool.size(), Eq(2u));
+
+ pool.Prune();
+ EXPECT_THAT(pool.size(), Eq(1u));
+}
+
+TEST(StringPoolTest, SortAndMaintainIndexesInStringReferences) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("z");
+ StringPool::Ref ref_b = pool.MakeRef("a");
+ StringPool::Ref ref_c = pool.MakeRef("m");
+
+ EXPECT_THAT(*ref_a, Eq("z"));
+ EXPECT_THAT(ref_a.index(), Eq(0u));
+
+ EXPECT_THAT(*ref_b, Eq("a"));
+ EXPECT_THAT(ref_b.index(), Eq(1u));
+
+ EXPECT_THAT(*ref_c, Eq("m"));
+ EXPECT_THAT(ref_c.index(), Eq(2u));
+
+ pool.Sort();
+
+ EXPECT_THAT(*ref_a, Eq("z"));
+ EXPECT_THAT(ref_a.index(), Eq(2u));
+
+ EXPECT_THAT(*ref_b, Eq("a"));
+ EXPECT_THAT(ref_b.index(), Eq(0u));
+
+ EXPECT_THAT(*ref_c, Eq("m"));
+ EXPECT_THAT(ref_c.index(), Eq(1u));
+}
+
+TEST(StringPoolTest, SortAndStillDedupe) {
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("z");
+ StringPool::Ref ref_b = pool.MakeRef("a");
+ StringPool::Ref ref_c = pool.MakeRef("m");
+
+ pool.Sort();
+
+ StringPool::Ref ref_d = pool.MakeRef("z");
+ StringPool::Ref ref_e = pool.MakeRef("a");
+ StringPool::Ref ref_f = pool.MakeRef("m");
+
+ EXPECT_THAT(ref_d.index(), Eq(ref_a.index()));
+ EXPECT_THAT(ref_e.index(), Eq(ref_b.index()));
+ EXPECT_THAT(ref_f.index(), Eq(ref_c.index()));
+}
+
+TEST(StringPoolTest, AddStyles) {
+ StringPool pool;
+
+ StringPool::StyleRef ref = pool.MakeRef(StyleString{{"android"}, {Span{{"b"}, 2, 6}}});
+ EXPECT_THAT(ref.index(), Eq(0u));
+ EXPECT_THAT(ref->value, Eq("android"));
+ ASSERT_THAT(ref->spans.size(), Eq(1u));
+
+ const StringPool::Span& span = ref->spans.front();
+ EXPECT_THAT(*span.name, Eq("b"));
+ EXPECT_THAT(span.first_char, Eq(2u));
+ EXPECT_THAT(span.last_char, Eq(6u));
+}
+
+TEST(StringPoolTest, DoNotDedupeStyleWithSameStringAsNonStyle) {
+ StringPool pool;
+
+ StringPool::Ref ref = pool.MakeRef("android");
+
+ StyleString str{{"android"}, {}};
+ StringPool::StyleRef style_ref = pool.MakeRef(StyleString{{"android"}, {}});
+
+ EXPECT_THAT(ref.index(), Ne(style_ref.index()));
+}
+
+TEST(StringPoolTest, StylesAndStringsAreSeparateAfterSorting) {
+ StringPool pool;
+
+ StringPool::StyleRef ref_a = pool.MakeRef(StyleString{{"beta"}, {}});
+ StringPool::Ref ref_b = pool.MakeRef("alpha");
+ StringPool::StyleRef ref_c = pool.MakeRef(StyleString{{"alpha"}, {}});
+
+ EXPECT_THAT(ref_b.index(), Ne(ref_c.index()));
+
+ pool.Sort();
+
+ EXPECT_THAT(ref_c.index(), Eq(0u));
+ EXPECT_THAT(ref_a.index(), Eq(1u));
+ EXPECT_THAT(ref_b.index(), Eq(2u));
+}
+
+TEST(StringPoolTest, FlattenEmptyStringPoolUtf8) {
+ using namespace android; // For NO_ERROR on Windows.
+ NoOpDiagnostics diag;
+
+ StringPool pool;
+ BigBuffer buffer(1024);
+ StringPool::FlattenUtf8(&buffer, pool, &diag);
+
+ std::unique_ptr<uint8_t[]> data = android::util::Copy(buffer);
+ ResStringPool test;
+ ASSERT_THAT(test.setTo(data.get(), buffer.size()), Eq(NO_ERROR));
+}
+
+TEST(StringPoolTest, FlattenOddCharactersUtf16) {
+ using namespace android; // For NO_ERROR on Windows.
+ NoOpDiagnostics diag;
+
+ StringPool pool;
+ pool.MakeRef("\u093f");
+ BigBuffer buffer(1024);
+ StringPool::FlattenUtf16(&buffer, pool, &diag);
+
+ std::unique_ptr<uint8_t[]> data = android::util::Copy(buffer);
+ ResStringPool test;
+ ASSERT_EQ(test.setTo(data.get(), buffer.size()), NO_ERROR);
+ auto str = test.stringAt(0);
+ ASSERT_TRUE(str.has_value());
+ EXPECT_THAT(str->size(), Eq(1u));
+ EXPECT_THAT(str->data(), Pointee(Eq(u'\u093f')));
+ EXPECT_THAT(str->data()[1], Eq(0u));
+}
+
+constexpr const char* sLongString =
+ "バッテリーを長持ちさせるため、バッテリーセーバーは端末のパフォーマンスを抑"
+ "え、バイブレーション、位置情報サービス、大半のバックグラウンドデータを制限"
+ "します。メール、SMSや、同期を使 "
+ "用するその他のアプリは、起動しても更新されないことがあります。バッテリーセ"
+ "ーバーは端末の充電中は自動的にOFFになります。";
+
+TEST(StringPoolTest, Flatten) {
+ using namespace android; // For NO_ERROR on Windows.
+ NoOpDiagnostics diag;
+
+ StringPool pool;
+
+ StringPool::Ref ref_a = pool.MakeRef("hello");
+ StringPool::Ref ref_b = pool.MakeRef("goodbye");
+ StringPool::Ref ref_c = pool.MakeRef(sLongString);
+ StringPool::Ref ref_d = pool.MakeRef("");
+ StringPool::StyleRef ref_e =
+ pool.MakeRef(StyleString{{"style"}, {Span{{"b"}, 0, 1}, Span{{"i"}, 2, 3}}});
+
+ // Styles are always first.
+ EXPECT_THAT(ref_e.index(), Eq(0u));
+
+ EXPECT_THAT(ref_a.index(), Eq(1u));
+ EXPECT_THAT(ref_b.index(), Eq(2u));
+ EXPECT_THAT(ref_c.index(), Eq(3u));
+ EXPECT_THAT(ref_d.index(), Eq(4u));
+
+ BigBuffer buffers[2] = {BigBuffer(1024), BigBuffer(1024)};
+ StringPool::FlattenUtf8(&buffers[0], pool, &diag);
+ StringPool::FlattenUtf16(&buffers[1], pool, &diag);
+
+ // Test both UTF-8 and UTF-16 buffers.
+ for (const BigBuffer& buffer : buffers) {
+ std::unique_ptr<uint8_t[]> data = android::util::Copy(buffer);
+
+ ResStringPool test;
+ ASSERT_EQ(test.setTo(data.get(), buffer.size()), NO_ERROR);
+
+ EXPECT_THAT(android::util::GetString(test, 1), Eq("hello"));
+ EXPECT_THAT(android::util::GetString16(test, 1), Eq(u"hello"));
+
+ EXPECT_THAT(android::util::GetString(test, 2), Eq("goodbye"));
+ EXPECT_THAT(android::util::GetString16(test, 2), Eq(u"goodbye"));
+
+ EXPECT_THAT(android::util::GetString(test, 3), Eq(sLongString));
+ EXPECT_THAT(android::util::GetString16(test, 3), Eq(util::Utf8ToUtf16(sLongString)));
+
+ EXPECT_TRUE(test.stringAt(4).has_value() || test.string8At(4).has_value());
+
+ EXPECT_THAT(android::util::GetString(test, 0), Eq("style"));
+ EXPECT_THAT(android::util::GetString16(test, 0), Eq(u"style"));
+
+ auto span_result = test.styleAt(0);
+ ASSERT_TRUE(span_result.has_value());
+
+ const ResStringPool_span* span = span_result->unsafe_ptr();
+ EXPECT_THAT(android::util::GetString(test, span->name.index), Eq("b"));
+ EXPECT_THAT(android::util::GetString16(test, span->name.index), Eq(u"b"));
+ EXPECT_THAT(span->firstChar, Eq(0u));
+ EXPECT_THAT(span->lastChar, Eq(1u));
+ span++;
+
+ ASSERT_THAT(span->name.index, Ne(ResStringPool_span::END));
+ EXPECT_THAT(android::util::GetString(test, span->name.index), Eq("i"));
+ EXPECT_THAT(android::util::GetString16(test, span->name.index), Eq(u"i"));
+ EXPECT_THAT(span->firstChar, Eq(2u));
+ EXPECT_THAT(span->lastChar, Eq(3u));
+ span++;
+
+ EXPECT_THAT(span->name.index, Eq(ResStringPool_span::END));
+ }
+}
+
+TEST(StringPoolTest, ModifiedUTF8) {
+ using namespace android; // For NO_ERROR on Windows.
+ NoOpDiagnostics diag;
+ StringPool pool;
+ StringPool::Ref ref_a = pool.MakeRef("\xF0\x90\x90\x80"); // 𐐀 (U+10400)
+ StringPool::Ref ref_b = pool.MakeRef("foo \xF0\x90\x90\xB7 bar"); // 𐐷 (U+10437)
+ StringPool::Ref ref_c = pool.MakeRef("\xF0\x90\x90\x80\xF0\x90\x90\xB7");
+
+ BigBuffer buffer(1024);
+ StringPool::FlattenUtf8(&buffer, pool, &diag);
+ std::unique_ptr<uint8_t[]> data = android::util::Copy(buffer);
+
+ // Check that the codepoints are encoded using two three-byte surrogate pairs
+ ResStringPool test;
+ ASSERT_EQ(test.setTo(data.get(), buffer.size()), NO_ERROR);
+ auto str = test.string8At(0);
+ ASSERT_TRUE(str.has_value());
+ EXPECT_THAT(str->to_string(), Eq("\xED\xA0\x81\xED\xB0\x80"));
+
+ str = test.string8At(1);
+ ASSERT_TRUE(str.has_value());
+ EXPECT_THAT(str->to_string(), Eq("foo \xED\xA0\x81\xED\xB0\xB7 bar"));
+
+ str = test.string8At(2);
+ ASSERT_TRUE(str.has_value());
+ EXPECT_THAT(str->to_string(), Eq("\xED\xA0\x81\xED\xB0\x80\xED\xA0\x81\xED\xB0\xB7"));
+
+ // Check that retrieving the strings returns the original UTF-8 character bytes
+ EXPECT_THAT(android::util::GetString(test, 0), Eq("\xF0\x90\x90\x80"));
+ EXPECT_THAT(android::util::GetString(test, 1), Eq("foo \xF0\x90\x90\xB7 bar"));
+ EXPECT_THAT(android::util::GetString(test, 2), Eq("\xF0\x90\x90\x80\xF0\x90\x90\xB7"));
+}
+
+TEST(StringPoolTest, MaxEncodingLength) {
+ NoOpDiagnostics diag;
+ using namespace android; // For NO_ERROR on Windows.
+ ResStringPool test;
+
+ StringPool pool;
+ pool.MakeRef("aaaaaaaaaa");
+ BigBuffer buffers[2] = {BigBuffer(1024), BigBuffer(1024)};
+
+ // Make sure a UTF-8 string under the maximum length does not produce an error
+ EXPECT_THAT(StringPool::FlattenUtf8(&buffers[0], pool, &diag), Eq(true));
+ std::unique_ptr<uint8_t[]> data = android::util::Copy(buffers[0]);
+ test.setTo(data.get(), buffers[0].size());
+ EXPECT_THAT(android::util::GetString(test, 0), Eq("aaaaaaaaaa"));
+
+ // Make sure a UTF-16 string under the maximum length does not produce an error
+ EXPECT_THAT(StringPool::FlattenUtf16(&buffers[1], pool, &diag), Eq(true));
+ data = android::util::Copy(buffers[1]);
+ test.setTo(data.get(), buffers[1].size());
+ EXPECT_THAT(android::util::GetString16(test, 0), Eq(u"aaaaaaaaaa"));
+
+ StringPool pool2;
+ std::string longStr(50000, 'a');
+ pool2.MakeRef("this fits1");
+ pool2.MakeRef(longStr);
+ pool2.MakeRef("this fits2");
+ BigBuffer buffers2[2] = {BigBuffer(1024), BigBuffer(1024)};
+
+ // Make sure a string that exceeds the maximum length of UTF-8 produces an
+ // error and writes a shorter error string instead
+ EXPECT_THAT(StringPool::FlattenUtf8(&buffers2[0], pool2, &diag), Eq(false));
+ data = android::util::Copy(buffers2[0]);
+ test.setTo(data.get(), buffers2[0].size());
+ EXPECT_THAT(android::util::GetString(test, 0), "this fits1");
+ EXPECT_THAT(android::util::GetString(test, 1), "STRING_TOO_LARGE");
+ EXPECT_THAT(android::util::GetString(test, 2), "this fits2");
+
+ // Make sure a string that a string that exceeds the maximum length of UTF-8
+ // but not UTF-16 does not error for UTF-16
+ StringPool pool3;
+ std::u16string longStr16(50000, 'a');
+ pool3.MakeRef(longStr);
+ EXPECT_THAT(StringPool::FlattenUtf16(&buffers2[1], pool3, &diag), Eq(true));
+ data = android::util::Copy(buffers2[1]);
+ test.setTo(data.get(), buffers2[1].size());
+ EXPECT_THAT(android::util::GetString16(test, 0), Eq(longStr16));
+}
+
+} // namespace android