Merge remote-tracking branch 'goog/upstream-master'.
The following commits were reverted:
840703a Fix update over cellular network on guest account
eaad5d0 Do not merge to AOSP: Fixes the link to brillo-clang-format in CrOS
740efad Reboot even if a system update is not available.
Fixed a few sign compare warnings.
Had to ifdef out 2 SquashfsFilesystemTest because it depends on unsquashfs -m.
Test: update_engine_unittests
Change-Id: I6f4ca5003e78c76064ec60d0797505d8c18d00bf
Merged-In: I6f4ca5003e78c76064ec60d0797505d8c18d00bf
diff --git a/Android.mk b/Android.mk
index 627d2e3..ddf633d 100644
--- a/Android.mk
+++ b/Android.mk
@@ -106,6 +106,7 @@
libbz \
libbspatch \
libbrotli \
+ libpuffpatch \
$(ue_update_metadata_protos_exported_static_libraries)
ue_libpayload_consumer_exported_shared_libraries := \
libcrypto \
@@ -130,8 +131,10 @@
common/terminator.cc \
common/utils.cc \
payload_consumer/bzip_extent_writer.cc \
+ payload_consumer/cached_file_descriptor.cc \
payload_consumer/delta_performer.cc \
payload_consumer/download_action.cc \
+ payload_consumer/extent_reader.cc \
payload_consumer/extent_writer.cc \
payload_consumer/file_descriptor.cc \
payload_consumer/file_descriptor_utils.cc \
@@ -610,8 +613,9 @@
libdivsufsort \
libdivsufsort64 \
libbrotli \
- libpayload_consumer \
liblzma \
+ libpayload_consumer \
+ libpuffdiff \
update_metadata-protos \
$(ue_libpayload_consumer_exported_static_libraries) \
$(ue_update_metadata_protos_exported_static_libraries)
@@ -627,6 +631,7 @@
payload_generator/block_mapping.cc \
payload_generator/bzip.cc \
payload_generator/cycle_breaker.cc \
+ payload_generator/deflate_utils.cc \
payload_generator/delta_diff_generator.cc \
payload_generator/delta_diff_utils.cc \
payload_generator/ext2_filesystem.cc \
@@ -641,6 +646,7 @@
payload_generator/payload_generation_config.cc \
payload_generator/payload_signer.cc \
payload_generator/raw_filesystem.cc \
+ payload_generator/squashfs_filesystem.cc \
payload_generator/tarjan.cc \
payload_generator/topological_sort.cc \
payload_generator/xz_android.cc
@@ -659,8 +665,9 @@
libbsdiff \
libdivsufsort \
libdivsufsort64 \
- libpayload_consumer \
liblzma \
+ libpayload_consumer \
+ libpuffdiff \
update_metadata-protos \
$(ue_common_static_libraries) \
$(ue_libpayload_consumer_exported_static_libraries) \
@@ -911,8 +918,10 @@
common/test_utils.cc \
common/utils_unittest.cc \
payload_consumer/bzip_extent_writer_unittest.cc \
+ payload_consumer/cached_file_descriptor_unittest.cc \
payload_consumer/delta_performer_integration_test.cc \
payload_consumer/delta_performer_unittest.cc \
+ payload_consumer/extent_reader_unittest.cc \
payload_consumer/extent_writer_unittest.cc \
payload_consumer/fake_file_descriptor.cc \
payload_consumer/file_descriptor_utils_unittest.cc \
@@ -924,6 +933,7 @@
payload_generator/blob_file_writer_unittest.cc \
payload_generator/block_mapping_unittest.cc \
payload_generator/cycle_breaker_unittest.cc \
+ payload_generator/deflate_utils_unittest.cc \
payload_generator/delta_diff_utils_unittest.cc \
payload_generator/ext2_filesystem_unittest.cc \
payload_generator/extent_ranges_unittest.cc \
@@ -936,6 +946,7 @@
payload_generator/payload_file_unittest.cc \
payload_generator/payload_generation_config_unittest.cc \
payload_generator/payload_signer_unittest.cc \
+ payload_generator/squashfs_filesystem_unittest.cc \
payload_generator/tarjan_unittest.cc \
payload_generator/topological_sort_unittest.cc \
payload_generator/zip_unittest.cc \
@@ -990,10 +1001,6 @@
LOCAL_SRC_FILES += \
update_attempter_android_unittest.cc
endif # local_use_omaha == 1
-ifeq ($(local_use_chrome_network_proxy),1)
-LOCAL_SRC_FILES += \
- chrome_browser_proxy_resolver_unittest.cc
-endif # local_use_chrome_network_proxy == 1
include $(BUILD_NATIVE_TEST)
# Update payload signing public key.
diff --git a/COMMIT-QUEUE.ini b/COMMIT-QUEUE.ini
new file mode 100644
index 0000000..ed99b9f
--- /dev/null
+++ b/COMMIT-QUEUE.ini
@@ -0,0 +1,11 @@
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Per-project Commit Queue settings.
+# Documentation: http://goo.gl/5J7oND
+
+[GENERAL]
+
+# Moblab testing is needed because of the udpate_payloads ebuild.
+pre-cq-configs: default guado_moblab-no-vmtest-pre-cq
diff --git a/OWNERS b/OWNERS
index f64bd32..0bf7587 100644
--- a/OWNERS
+++ b/OWNERS
@@ -6,3 +6,4 @@
# Chromium OS maintainers:
benchan@google.com
+ahassani@google.com
diff --git a/PRESUBMIT.cfg b/PRESUBMIT.cfg
index 087dfa3..3b8b271 100644
--- a/PRESUBMIT.cfg
+++ b/PRESUBMIT.cfg
@@ -1,3 +1,7 @@
+[Hook Scripts]
+hook0=../../../../chromite/bin/cros lint ${PRESUBMIT_FILES}
+hook1=../../../platform2/common-mk/gyplint.py ${PRESUBMIT_FILES}
+
[Hook Overrides]
cros_license_check: false
aosp_license_check: true
diff --git a/boot_control_android.cc b/boot_control_android.cc
index e3ea66d..8c1603b 100644
--- a/boot_control_android.cc
+++ b/boot_control_android.cc
@@ -20,7 +20,6 @@
#include <base/files/file_util.h>
#include <base/logging.h>
#include <base/strings/string_util.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/message_loop.h>
#include "update_engine/common/utils.h"
diff --git a/boot_control_chromeos.cc b/boot_control_chromeos.cc
index e9ad698..aa94d3c 100644
--- a/boot_control_chromeos.cc
+++ b/boot_control_chromeos.cc
@@ -22,7 +22,6 @@
#include <base/files/file_path.h>
#include <base/files/file_util.h>
#include <base/strings/string_util.h>
-#include <brillo/make_unique_ptr.h>
#include <rootdev/rootdev.h>
extern "C" {
diff --git a/boot_control_recovery.cc b/boot_control_recovery.cc
index 39b5ff1..b74f4aa 100644
--- a/boot_control_recovery.cc
+++ b/boot_control_recovery.cc
@@ -20,7 +20,6 @@
#include <base/files/file_util.h>
#include <base/logging.h>
#include <base/strings/string_util.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/message_loop.h>
#include "update_engine/common/utils.h"
diff --git a/certificate_checker_unittest.cc b/certificate_checker_unittest.cc
index 20efce9..66b92d6 100644
--- a/certificate_checker_unittest.cc
+++ b/certificate_checker_unittest.cc
@@ -29,7 +29,7 @@
using ::testing::DoAll;
using ::testing::Return;
-using ::testing::SetArgumentPointee;
+using ::testing::SetArgPointee;
using ::testing::SetArrayArgument;
using ::testing::_;
using std::string;
@@ -78,8 +78,8 @@
TEST_F(CertificateCheckerTest, NewCertificate) {
EXPECT_CALL(openssl_wrapper_, GetCertificateDigest(nullptr, _, _, _))
.WillOnce(DoAll(
- SetArgumentPointee<1>(depth_),
- SetArgumentPointee<2>(length_),
+ SetArgPointee<1>(depth_),
+ SetArgPointee<2>(length_),
SetArrayArgument<3>(digest_, digest_ + 4),
Return(true)));
EXPECT_CALL(prefs_, GetString(cert_key_, _)).WillOnce(Return(false));
@@ -95,12 +95,12 @@
TEST_F(CertificateCheckerTest, SameCertificate) {
EXPECT_CALL(openssl_wrapper_, GetCertificateDigest(nullptr, _, _, _))
.WillOnce(DoAll(
- SetArgumentPointee<1>(depth_),
- SetArgumentPointee<2>(length_),
+ SetArgPointee<1>(depth_),
+ SetArgPointee<2>(length_),
SetArrayArgument<3>(digest_, digest_ + 4),
Return(true)));
EXPECT_CALL(prefs_, GetString(cert_key_, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(digest_hex_), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(digest_hex_), Return(true)));
EXPECT_CALL(prefs_, SetString(_, _)).Times(0);
EXPECT_CALL(observer_,
CertificateChecked(server_to_check_,
@@ -113,12 +113,12 @@
TEST_F(CertificateCheckerTest, ChangedCertificate) {
EXPECT_CALL(openssl_wrapper_, GetCertificateDigest(nullptr, _, _, _))
.WillOnce(DoAll(
- SetArgumentPointee<1>(depth_),
- SetArgumentPointee<2>(length_),
+ SetArgPointee<1>(depth_),
+ SetArgPointee<2>(length_),
SetArrayArgument<3>(digest_, digest_ + 4),
Return(true)));
EXPECT_CALL(prefs_, GetString(cert_key_, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(diff_digest_hex_), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(diff_digest_hex_), Return(true)));
EXPECT_CALL(observer_,
CertificateChecked(server_to_check_,
CertificateCheckResult::kValidChanged));
diff --git a/chrome_browser_proxy_resolver.cc b/chrome_browser_proxy_resolver.cc
index 12a8328..5beecc1 100644
--- a/chrome_browser_proxy_resolver.cc
+++ b/chrome_browser_proxy_resolver.cc
@@ -20,85 +20,26 @@
#include <base/bind.h>
#include <base/memory/ptr_util.h>
-#include <base/strings/string_tokenizer.h>
#include <base/strings/string_util.h>
+#include <brillo/http/http_proxy.h>
-#include "network_proxy/dbus-proxies.h"
+#include "update_engine/dbus_connection.h"
namespace chromeos_update_engine {
-using base::StringTokenizer;
-using std::deque;
-using std::string;
-
-namespace {
-
-// Timeout for D-Bus calls in milliseconds.
-constexpr int kTimeoutMs = 5000;
-
-} // namespace
-
-ChromeBrowserProxyResolver::ChromeBrowserProxyResolver(
- org::chromium::NetworkProxyServiceInterfaceProxyInterface* dbus_proxy)
- : dbus_proxy_(dbus_proxy),
- next_request_id_(kProxyRequestIdNull + 1),
+ChromeBrowserProxyResolver::ChromeBrowserProxyResolver()
+ : next_request_id_(kProxyRequestIdNull + 1),
weak_ptr_factory_(this) {}
ChromeBrowserProxyResolver::~ChromeBrowserProxyResolver() = default;
-// static
-deque<string> ChromeBrowserProxyResolver::ParseProxyString(
- const string& input) {
- deque<string> ret;
- // Some of this code taken from
- // http://src.chromium.org/svn/trunk/src/net/proxy/proxy_server.cc and
- // http://src.chromium.org/svn/trunk/src/net/proxy/proxy_list.cc
- StringTokenizer entry_tok(input, ";");
- while (entry_tok.GetNext()) {
- string token = entry_tok.token();
- base::TrimWhitespaceASCII(token, base::TRIM_ALL, &token);
-
- // Start by finding the first space (if any).
- string::iterator space;
- for (space = token.begin(); space != token.end(); ++space) {
- if (base::IsAsciiWhitespace(*space)) {
- break;
- }
- }
-
- string scheme = base::ToLowerASCII(string(token.begin(), space));
- // Chrome uses "socks" to mean socks4 and "proxy" to mean http.
- if (scheme == "socks")
- scheme += "4";
- else if (scheme == "proxy")
- scheme = "http";
- else if (scheme != "https" &&
- scheme != "socks4" &&
- scheme != "socks5" &&
- scheme != "direct")
- continue; // Invalid proxy scheme
-
- string host_and_port = string(space, token.end());
- base::TrimWhitespaceASCII(host_and_port, base::TRIM_ALL, &host_and_port);
- if (scheme != "direct" && host_and_port.empty())
- continue; // Must supply host/port when non-direct proxy used.
- ret.push_back(scheme + "://" + host_and_port);
- }
- if (ret.empty() || *ret.rbegin() != kNoProxy)
- ret.push_back(kNoProxy);
- return ret;
-}
-
ProxyRequestId ChromeBrowserProxyResolver::GetProxiesForUrl(
- const string& url, const ProxiesResolvedFn& callback) {
+ const std::string& url, const ProxiesResolvedFn& callback) {
const ProxyRequestId id = next_request_id_++;
- dbus_proxy_->ResolveProxyAsync(
- url,
- base::Bind(&ChromeBrowserProxyResolver::OnResolveProxyResponse,
- weak_ptr_factory_.GetWeakPtr(), id),
- base::Bind(&ChromeBrowserProxyResolver::OnResolveProxyError,
- weak_ptr_factory_.GetWeakPtr(), id),
- kTimeoutMs);
+ brillo::http::GetChromeProxyServersAsync(
+ DBusConnection::Get()->GetDBus(), url,
+ base::Bind(&ChromeBrowserProxyResolver::OnGetChromeProxyServers,
+ weak_ptr_factory_.GetWeakPtr(), id));
pending_callbacks_[id] = callback;
return id;
}
@@ -107,32 +48,18 @@
return pending_callbacks_.erase(request) != 0;
}
-void ChromeBrowserProxyResolver::OnResolveProxyResponse(
- ProxyRequestId request_id,
- const std::string& proxy_info,
- const std::string& error_message) {
- if (!error_message.empty())
- LOG(WARNING) << "Got error resolving proxy: " << error_message;
- RunCallback(request_id, ParseProxyString(proxy_info));
-}
-
-void ChromeBrowserProxyResolver::OnResolveProxyError(ProxyRequestId request_id,
- brillo::Error* error) {
- LOG(WARNING) << "Failed to resolve proxy: "
- << (error ? error->GetMessage() : "[null]");
- RunCallback(request_id, deque<string>{kNoProxy});
-}
-
-void ChromeBrowserProxyResolver::RunCallback(
- ProxyRequestId request_id,
- const std::deque<std::string>& proxies) {
+void ChromeBrowserProxyResolver::OnGetChromeProxyServers(
+ ProxyRequestId request_id, bool success,
+ const std::vector<std::string>& proxies) {
+ // If |success| is false, |proxies| will still hold the direct proxy option
+ // which is what we do in our error case.
auto it = pending_callbacks_.find(request_id);
if (it == pending_callbacks_.end())
return;
ProxiesResolvedFn callback = it->second;
pending_callbacks_.erase(it);
- callback.Run(proxies);
+ callback.Run(std::deque<std::string>(proxies.begin(), proxies.end()));
}
-} // namespace chromeos_update_engine
+} // namespace chromeos_update_engine
diff --git a/chrome_browser_proxy_resolver.h b/chrome_browser_proxy_resolver.h
index 03dbdad..fcf85b6 100644
--- a/chrome_browser_proxy_resolver.h
+++ b/chrome_browser_proxy_resolver.h
@@ -20,46 +20,29 @@
#include <deque>
#include <map>
#include <string>
+#include <vector>
#include <base/memory/weak_ptr.h>
#include "update_engine/proxy_resolver.h"
-namespace brillo {
-class Error;
-} // namespace brillo
-
-namespace org {
-namespace chromium {
-class NetworkProxyServiceInterfaceProxyInterface;
-} // namespace chromium
-} // namespace org
-
namespace chromeos_update_engine {
class ChromeBrowserProxyResolver : public ProxyResolver {
public:
- explicit ChromeBrowserProxyResolver(
- org::chromium::NetworkProxyServiceInterfaceProxyInterface* dbus_proxy);
+ ChromeBrowserProxyResolver();
~ChromeBrowserProxyResolver() override;
- // Parses a string-encoded list of proxies and returns a deque
- // of individual proxies. The last one will always be kNoProxy.
- static std::deque<std::string> ParseProxyString(const std::string& input);
-
// ProxyResolver:
ProxyRequestId GetProxiesForUrl(const std::string& url,
const ProxiesResolvedFn& callback) override;
bool CancelProxyRequest(ProxyRequestId request) override;
-private:
- // Callback for successful D-Bus calls made by GetProxiesForUrl().
- void OnResolveProxyResponse(ProxyRequestId request_id,
- const std::string& proxy_info,
- const std::string& error_message);
-
- // Callback for failed D-Bus calls made by GetProxiesForUrl().
- void OnResolveProxyError(ProxyRequestId request_id, brillo::Error* error);
+ private:
+ // Callback for calls made by GetProxiesForUrl().
+ void OnGetChromeProxyServers(ProxyRequestId request_id,
+ bool success,
+ const std::vector<std::string>& proxies);
// Finds the callback identified by |request_id| in |pending_callbacks_|,
// passes |proxies| to it, and deletes it. Does nothing if the request has
@@ -67,9 +50,6 @@
void RunCallback(ProxyRequestId request_id,
const std::deque<std::string>& proxies);
- // D-Bus proxy for resolving network proxies.
- org::chromium::NetworkProxyServiceInterfaceProxyInterface* dbus_proxy_;
-
// Next ID to return from GetProxiesForUrl().
ProxyRequestId next_request_id_;
diff --git a/chrome_browser_proxy_resolver_unittest.cc b/chrome_browser_proxy_resolver_unittest.cc
deleted file mode 100644
index dc71d2b..0000000
--- a/chrome_browser_proxy_resolver_unittest.cc
+++ /dev/null
@@ -1,180 +0,0 @@
-//
-// Copyright (C) 2011 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-#include "update_engine/chrome_browser_proxy_resolver.h"
-
-#include <deque>
-#include <string>
-#include <vector>
-
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-
-#include <base/macros.h>
-#include <brillo/errors/error.h>
-
-#include "network_proxy/dbus-proxies.h"
-#include "network_proxy/dbus-proxy-mocks.h"
-#include "update_engine/dbus_test_utils.h"
-
-using ::testing::DoAll;
-using ::testing::SaveArg;
-using ::testing::StrEq;
-using ::testing::_;
-using org::chromium::NetworkProxyServiceInterfaceProxyMock;
-using std::deque;
-using std::string;
-using std::vector;
-
-namespace chromeos_update_engine {
-
-namespace {
-
-// Callback for ProxyResolver::GetProxiesForUrl() that copies |src| to |dest|.
-void CopyProxies(deque<string>* dest, const deque<string>& src) {
- *dest = src;
-}
-
-} // namespace
-
-class ChromeBrowserProxyResolverTest : public ::testing::Test {
- public:
- ChromeBrowserProxyResolverTest() = default;
- ~ChromeBrowserProxyResolverTest() override = default;
-
- protected:
- // Adds a GoogleMock expectation for a call to |dbus_proxy_|'s
- // ResolveProxyAsync method to resolve |url|.
- void AddResolveProxyExpectation(const std::string& url) {
- EXPECT_CALL(dbus_proxy_, ResolveProxyAsync(StrEq(url), _, _, _))
- .WillOnce(DoAll(SaveArg<1>(&success_callback_),
- SaveArg<2>(&error_callback_)));
- }
-
- NetworkProxyServiceInterfaceProxyMock dbus_proxy_;
- ChromeBrowserProxyResolver resolver_{&dbus_proxy_};
-
- // Callbacks that were passed to |dbus_proxy_|'s ResolveProxyAsync method.
- base::Callback<void(const std::string&, const std::string&)>
- success_callback_;
- base::Callback<void(brillo::Error*)> error_callback_;
-
- private:
- DISALLOW_COPY_AND_ASSIGN(ChromeBrowserProxyResolverTest);
-};
-
-TEST_F(ChromeBrowserProxyResolverTest, Parse) {
- // Test ideas from
- // http://src.chromium.org/svn/trunk/src/net/proxy/proxy_list_unittest.cc
- vector<string> inputs = {
- "PROXY foopy:10",
- " DIRECT", // leading space.
- "PROXY foopy1 ; proxy foopy2;\t DIRECT",
- "proxy foopy1 ; SOCKS foopy2",
- "DIRECT ; proxy foopy1 ; DIRECT ; SOCKS5 foopy2;DIRECT ",
- "DIRECT ; proxy foopy1:80; DIRECT ; DIRECT",
- "PROXY-foopy:10",
- "PROXY",
- "PROXY foopy1 ; JUNK ; JUNK ; SOCKS5 foopy2 ; ;",
- "HTTP foopy1; SOCKS5 foopy2",
- };
- vector<deque<string>> outputs = {
- {"http://foopy:10", kNoProxy},
- {kNoProxy},
- {"http://foopy1", "http://foopy2", kNoProxy},
- {"http://foopy1", "socks4://foopy2", kNoProxy},
- {kNoProxy, "http://foopy1", kNoProxy, "socks5://foopy2", kNoProxy},
- {kNoProxy, "http://foopy1:80", kNoProxy, kNoProxy},
- {kNoProxy},
- {kNoProxy},
- {"http://foopy1", "socks5://foopy2", kNoProxy},
- {"socks5://foopy2", kNoProxy},
- };
- ASSERT_EQ(inputs.size(), outputs.size());
-
- for (size_t i = 0; i < inputs.size(); i++) {
- deque<string> results =
- ChromeBrowserProxyResolver::ParseProxyString(inputs[i]);
- deque<string>& expected = outputs[i];
- EXPECT_EQ(results.size(), expected.size()) << "i = " << i;
- if (expected.size() != results.size())
- continue;
- for (size_t j = 0; j < expected.size(); j++) {
- EXPECT_EQ(expected[j], results[j]) << "i = " << i;
- }
- }
-}
-
-TEST_F(ChromeBrowserProxyResolverTest, Success) {
- const char kUrl[] = "http://example.com/blah";
- const char kProxyConfig[] = "SOCKS5 192.168.52.83:5555;DIRECT";
- AddResolveProxyExpectation(kUrl);
- deque<string> proxies;
- resolver_.GetProxiesForUrl(kUrl, base::Bind(&CopyProxies, &proxies));
-
- // Run the D-Bus success callback and verify that the proxies are passed to
- // the supplied function.
- ASSERT_FALSE(success_callback_.is_null());
- success_callback_.Run(kProxyConfig, string());
- ASSERT_EQ(2u, proxies.size());
- EXPECT_EQ("socks5://192.168.52.83:5555", proxies[0]);
- EXPECT_EQ(kNoProxy, proxies[1]);
-}
-
-TEST_F(ChromeBrowserProxyResolverTest, Failure) {
- const char kUrl[] = "http://example.com/blah";
- AddResolveProxyExpectation(kUrl);
- deque<string> proxies;
- resolver_.GetProxiesForUrl(kUrl, base::Bind(&CopyProxies, &proxies));
-
- // Run the D-Bus error callback and verify that the supplied function is
- // instructed to use a direct connection.
- ASSERT_FALSE(error_callback_.is_null());
- brillo::ErrorPtr error = brillo::Error::Create(FROM_HERE, "", "", "");
- error_callback_.Run(error.get());
- ASSERT_EQ(1u, proxies.size());
- EXPECT_EQ(kNoProxy, proxies[0]);
-}
-
-TEST_F(ChromeBrowserProxyResolverTest, CancelCallback) {
- const char kUrl[] = "http://example.com/blah";
- AddResolveProxyExpectation(kUrl);
- int called = 0;
- auto callback = base::Bind(
- [](int* called, const deque<string>& proxies) { (*called)++; }, &called);
- ProxyRequestId request = resolver_.GetProxiesForUrl(kUrl, callback);
-
- // Cancel the request and then run the D-Bus success callback. The original
- // callback shouldn't be run.
- EXPECT_TRUE(resolver_.CancelProxyRequest(request));
- ASSERT_FALSE(success_callback_.is_null());
- success_callback_.Run("DIRECT", string());
- EXPECT_EQ(0, called);
-}
-
-TEST_F(ChromeBrowserProxyResolverTest, CancelCallbackTwice) {
- const char kUrl[] = "http://example.com/blah";
- AddResolveProxyExpectation(kUrl);
- deque<string> proxies;
- ProxyRequestId request =
- resolver_.GetProxiesForUrl(kUrl, base::Bind(&CopyProxies, &proxies));
-
- // Cancel the same request twice. The second call should fail.
- EXPECT_TRUE(resolver_.CancelProxyRequest(request));
- EXPECT_FALSE(resolver_.CancelProxyRequest(request));
-}
-
-} // namespace chromeos_update_engine
diff --git a/common/fake_hardware.h b/common/fake_hardware.h
index a384597..f2b2c9d 100644
--- a/common/fake_hardware.h
+++ b/common/fake_hardware.h
@@ -83,6 +83,14 @@
int64_t GetBuildTimestamp() const override { return build_timestamp_; }
+ bool GetFirstActiveOmahaPingSent() const override {
+ return first_active_omaha_ping_sent_;
+ }
+
+ void SetFirstActiveOmahaPingSent() override {
+ first_active_omaha_ping_sent_ = true;
+ }
+
// Setters
void SetIsOfficialBuild(bool is_official_build) {
is_official_build_ = is_official_build;
@@ -144,6 +152,7 @@
int powerwash_count_{kPowerwashCountNotSet};
bool powerwash_scheduled_{false};
int64_t build_timestamp_{0};
+ bool first_active_omaha_ping_sent_{false};
DISALLOW_COPY_AND_ASSIGN(FakeHardware);
};
diff --git a/common/hardware_interface.h b/common/hardware_interface.h
index c9e2f85..94442d1 100644
--- a/common/hardware_interface.h
+++ b/common/hardware_interface.h
@@ -94,6 +94,15 @@
// Returns the timestamp of the current OS build.
virtual int64_t GetBuildTimestamp() const = 0;
+
+ // Returns whether the first active ping was sent to Omaha at some point, and
+ // that the value is persisted across recovery (and powerwash) once set with
+ // |SetFirstActiveOmahaPingSent()|.
+ virtual bool GetFirstActiveOmahaPingSent() const = 0;
+
+ // Persist the fact that first active ping was sent to omaha. It bails out if
+ // it fails.
+ virtual void SetFirstActiveOmahaPingSent() = 0;
};
} // namespace chromeos_update_engine
diff --git a/common/http_fetcher_unittest.cc b/common/http_fetcher_unittest.cc
index eb85f68..867216e 100644
--- a/common/http_fetcher_unittest.cc
+++ b/common/http_fetcher_unittest.cc
@@ -633,7 +633,7 @@
fetcher->Unpause();
fetcher->Pause();
// Proxy resolver comes back after we paused the fetcher.
- ASSERT_TRUE(proxy_callback);
+ ASSERT_FALSE(proxy_callback.is_null());
proxy_callback.Run({1, kNoProxy});
}
diff --git a/common/mock_hardware.h b/common/mock_hardware.h
index 1c4253a..42fa7ba 100644
--- a/common/mock_hardware.h
+++ b/common/mock_hardware.h
@@ -63,6 +63,12 @@
ON_CALL(*this, GetPowerwashSafeDirectory(testing::_))
.WillByDefault(testing::Invoke(&fake_,
&FakeHardware::GetPowerwashSafeDirectory));
+ ON_CALL(*this, GetFirstActiveOmahaPingSent())
+ .WillByDefault(testing::Invoke(&fake_,
+ &FakeHardware::GetFirstActiveOmahaPingSent()));
+ ON_CALL(*this, SetFirstActiveOmahaPingSent())
+ .WillByDefault(testing::Invoke(&fake_,
+ &FakeHardware::SetFirstActiveOmahaPingSent()));
}
~MockHardware() override = default;
@@ -78,6 +84,7 @@
MOCK_CONST_METHOD0(GetPowerwashCount, int());
MOCK_CONST_METHOD1(GetNonVolatileDirectory, bool(base::FilePath*));
MOCK_CONST_METHOD1(GetPowerwashSafeDirectory, bool(base::FilePath*));
+ MOCK_CONST_METHOD0(GetFirstActiveOmahaPingSent, bool());
// Returns a reference to the underlying FakeHardware.
FakeHardware& fake() {
diff --git a/common/utils.h b/common/utils.h
index 2117836..e4ffcf8 100644
--- a/common/utils.h
+++ b/common/utils.h
@@ -88,8 +88,8 @@
bool PReadAll(int fd, void* buf, size_t count, off_t offset,
ssize_t* out_bytes_read);
-bool PReadAll(const FileDescriptorPtr& fd, void* buf, size_t count, off_t offset,
- ssize_t* out_bytes_read);
+bool PReadAll(const FileDescriptorPtr& fd, void* buf, size_t count,
+ off_t offset, ssize_t* out_bytes_read);
// Opens |path| for reading and appends its entire content to the container
// pointed to by |out_p|. Returns true upon successfully reading all of the
@@ -256,6 +256,16 @@
}
}
+// Return the total number of blocks in the passed |extents| collection.
+template <class T>
+uint64_t BlocksInExtents(const T& extents) {
+ uint64_t sum = 0;
+ for (const auto& ext : extents) {
+ sum += ext.num_blocks();
+ }
+ return sum;
+}
+
// Converts seconds into human readable notation including days, hours, minutes
// and seconds. For example, 185 will yield 3m5s, 4300 will yield 1h11m40s, and
// 360000 will yield 4d4h0m0s. Zero padding not applied. Seconds are always
diff --git a/common_service_unittest.cc b/common_service_unittest.cc
index c124466..d9ef567 100644
--- a/common_service_unittest.cc
+++ b/common_service_unittest.cc
@@ -30,7 +30,7 @@
using std::string;
using testing::_;
using testing::Return;
-using testing::SetArgumentPointee;
+using testing::SetArgPointee;
using update_engine::UpdateAttemptFlags;
namespace chromeos_update_engine {
@@ -102,7 +102,7 @@
policy::MockDevicePolicy mock_device_policy;
fake_system_state_.set_device_policy(&mock_device_policy);
EXPECT_CALL(mock_device_policy, GetReleaseChannelDelegated(_))
- .WillOnce(DoAll(SetArgumentPointee<0>(true), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<0>(true), Return(true)));
EXPECT_CALL(*fake_system_state_.mock_request_params(),
SetTargetChannel("beta-channel", true, _))
.WillOnce(Return(true));
diff --git a/connection_manager_unittest.cc b/connection_manager_unittest.cc
index 0bb5547..e26a686 100644
--- a/connection_manager_unittest.cc
+++ b/connection_manager_unittest.cc
@@ -21,7 +21,6 @@
#include <base/logging.h>
#include <brillo/any.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/fake_message_loop.h>
#include <brillo/variant_dictionary.h>
#include <gmock/gmock.h>
diff --git a/dbus_bindings/org.chromium.NetworkProxyService.dbus-xml b/dbus_bindings/org.chromium.NetworkProxyService.dbus-xml
deleted file mode 100644
index 90686ca..0000000
--- a/dbus_bindings/org.chromium.NetworkProxyService.dbus-xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<node name="/org/chromium/NetworkProxyService"
- xmlns:tp="http://telepathy.freedesktop.org/wiki/DbusSpec#extensions-v0">
- <interface name="org.chromium.NetworkProxyServiceInterface">
- <method name="ResolveProxy">
- <arg name="source_url" type="s" direction="in" />
- <arg name="proxy_info" type="s" direction="out" />
- <arg name="error_message" type="s" direction="out" />
- <annotation name="org.chromium.DBus.Method.Kind" value="async" />
- </method>
- </interface>
-</node>
diff --git a/fake_system_state.cc b/fake_system_state.cc
index 5abf71b..1bfcafa 100644
--- a/fake_system_state.cc
+++ b/fake_system_state.cc
@@ -21,7 +21,7 @@
// Mock the SystemStateInterface so that we could lie that
// OOBE is completed even when there's no such marker file, etc.
FakeSystemState::FakeSystemState()
- : mock_update_attempter_(this, nullptr, nullptr),
+ : mock_update_attempter_(this, nullptr),
mock_request_params_(this),
fake_update_manager_(&fake_clock_),
clock_(&fake_clock_),
diff --git a/hardware_android.cc b/hardware_android.cc
index 86432a9..947b13a 100644
--- a/hardware_android.cc
+++ b/hardware_android.cc
@@ -28,7 +28,6 @@
#include <android-base/properties.h>
#include <base/files/file_util.h>
#include <base/strings/stringprintf.h>
-#include <brillo/make_unique_ptr.h>
#include "update_engine/common/hardware.h"
#include "update_engine/common/platform_constants.h"
@@ -99,7 +98,7 @@
// Factory defined in hardware.h.
std::unique_ptr<HardwareInterface> CreateHardware() {
- return brillo::make_unique_ptr(new HardwareAndroid());
+ return std::make_unique<HardwareAndroid>();
}
} // namespace hardware
@@ -201,4 +200,14 @@
return GetIntProperty<int64_t>(kPropBuildDateUTC, 0);
}
+bool HardwareAndroid::GetFirstActiveOmahaPingSent() const {
+ LOG(WARNING) << "STUB: Assuming first active omaha was never set.";
+ return false;
+}
+
+void HardwareAndroid::SetFirstActiveOmahaPingSent() {
+ LOG(WARNING) << "STUB: Assuming first active omaha is never set.";
+ return;
+}
+
} // namespace chromeos_update_engine
diff --git a/hardware_android.h b/hardware_android.h
index 80fd9df..ca90b62 100644
--- a/hardware_android.h
+++ b/hardware_android.h
@@ -48,6 +48,8 @@
bool GetNonVolatileDirectory(base::FilePath* path) const override;
bool GetPowerwashSafeDirectory(base::FilePath* path) const override;
int64_t GetBuildTimestamp() const override;
+ bool GetFirstActiveOmahaPingSent() const override;
+ void SetFirstActiveOmahaPingSent() override;
private:
DISALLOW_COPY_AND_ASSIGN(HardwareAndroid);
diff --git a/hardware_chromeos.cc b/hardware_chromeos.cc
index f0f3ea9..f2bb28a 100644
--- a/hardware_chromeos.cc
+++ b/hardware_chromeos.cc
@@ -22,7 +22,6 @@
#include <base/strings/string_number_conversions.h>
#include <base/strings/string_util.h>
#include <brillo/key_value_store.h>
-#include <brillo/make_unique_ptr.h>
#include <debugd/dbus-constants.h>
#include <vboot/crossystem.h>
@@ -69,6 +68,8 @@
// UpdateManager config options:
const char* kConfigOptsIsOOBEEnabled = "is_oobe_enabled";
+const char* kActivePingKey = "first_active_omaha_ping_sent";
+
} // namespace
namespace chromeos_update_engine {
@@ -253,4 +254,49 @@
is_oobe_enabled_ = true; // Default value.
}
+bool HardwareChromeOS::GetFirstActiveOmahaPingSent() const {
+ int exit_code = 0;
+ string active_ping_str;
+ vector<string> cmd = { "vpd_get_value", kActivePingKey };
+ if (!Subprocess::SynchronousExec(cmd, &exit_code, &active_ping_str) ||
+ exit_code) {
+ LOG(ERROR) << "Failed to get vpd key for " << kActivePingKey
+ << " with exit code: " << exit_code;
+ return false;
+ }
+
+ base::TrimWhitespaceASCII(active_ping_str,
+ base::TRIM_ALL,
+ &active_ping_str);
+ int active_ping;
+ if (active_ping_str.empty() ||
+ !base::StringToInt(active_ping_str, &active_ping)) {
+ LOG(INFO) << "Failed to parse active_ping value: " << active_ping_str;
+ return false;
+ }
+ return static_cast<bool>(active_ping);
+}
+
+void HardwareChromeOS::SetFirstActiveOmahaPingSent() {
+ int exit_code = 0;
+ string output;
+ vector<string> vpd_set_cmd = {
+ "vpd", "-i", "RW_VPD", "-s", string(kActivePingKey) + "=1" };
+ if (!Subprocess::SynchronousExec(vpd_set_cmd, &exit_code, &output) ||
+ exit_code) {
+ LOG(ERROR) << "Failed to set vpd key for " << kActivePingKey
+ << " with exit code: " << exit_code
+ << " with error: " << output;
+ return;
+ }
+
+ vector<string> vpd_dump_cmd = { "dump_vpd_log", "--force" };
+ if (!Subprocess::SynchronousExec(vpd_dump_cmd, &exit_code, &output) ||
+ exit_code) {
+ LOG(ERROR) << "Failed to cache " << kActivePingKey<< " using dump_vpd_log"
+ << " with exit code: " << exit_code
+ << " with error: " << output;
+ }
+}
+
} // namespace chromeos_update_engine
diff --git a/hardware_chromeos.h b/hardware_chromeos.h
index a756a9b..0cf1214 100644
--- a/hardware_chromeos.h
+++ b/hardware_chromeos.h
@@ -52,6 +52,8 @@
bool GetNonVolatileDirectory(base::FilePath* path) const override;
bool GetPowerwashSafeDirectory(base::FilePath* path) const override;
int64_t GetBuildTimestamp() const override;
+ bool GetFirstActiveOmahaPingSent() const override;
+ void SetFirstActiveOmahaPingSent() override;
private:
friend class HardwareChromeOSTest;
diff --git a/image_properties.h b/image_properties.h
index 2d1a408..f8fe095 100644
--- a/image_properties.h
+++ b/image_properties.h
@@ -79,8 +79,8 @@
// value may be returned instead.
ImageProperties LoadImageProperties(SystemState* system_state);
-// Loads the mutable image properties from the stateful partition if found or the
-// system image otherwise.
+// Loads the mutable image properties from the stateful partition if found or
+// the system image otherwise.
MutableImageProperties LoadMutableImageProperties(SystemState* system_state);
// Stores the mutable image properties in the stateful partition. Returns
@@ -88,6 +88,9 @@
bool StoreMutableImageProperties(SystemState* system_state,
const MutableImageProperties& properties);
+// Logs the image properties.
+void LogImageProperties();
+
// Sets the root_prefix used to load files from during unittests to
// |test_root_prefix|. Passing a nullptr value resets it to the default.
namespace test {
diff --git a/image_properties_android.cc b/image_properties_android.cc
index 608bca7..1ec425d 100644
--- a/image_properties_android.cc
+++ b/image_properties_android.cc
@@ -149,4 +149,8 @@
properties.is_powerwash_allowed));
}
+void LogImageProperties() {
+ // TODO(*): Implement this.
+}
+
} // namespace chromeos_update_engine
diff --git a/image_properties_chromeos.cc b/image_properties_chromeos.cc
index 6bab63f..39ddeb3 100644
--- a/image_properties_chromeos.cc
+++ b/image_properties_chromeos.cc
@@ -26,6 +26,7 @@
#include "update_engine/common/constants.h"
#include "update_engine/common/hardware_interface.h"
#include "update_engine/common/platform_constants.h"
+#include "update_engine/common/utils.h"
#include "update_engine/system_state.h"
namespace {
@@ -149,4 +150,17 @@
return lsb_release.Save(path);
}
+void LogImageProperties() {
+ std::string lsb_release;
+ if (utils::ReadFile(kLsbRelease, &lsb_release)) {
+ LOG(INFO) << "lsb-release inside the old rootfs:\n" << lsb_release;
+ }
+
+ std::string stateful_lsb_release;
+ if (utils::ReadFile(std::string(kStatefulPartition) + kLsbRelease,
+ &stateful_lsb_release)) {
+ LOG(INFO) << "stateful lsb-release:\n" << stateful_lsb_release;
+ }
+}
+
} // namespace chromeos_update_engine
diff --git a/network_selector_android.cc b/network_selector_android.cc
index 6879b69..55ba799 100644
--- a/network_selector_android.cc
+++ b/network_selector_android.cc
@@ -16,9 +16,10 @@
#include "update_engine/network_selector_android.h"
+#include <memory>
+
#include <android/multinetwork.h>
#include <base/logging.h>
-#include <brillo/make_unique_ptr.h>
namespace chromeos_update_engine {
@@ -26,7 +27,7 @@
// Factory defined in network_selector.h.
std::unique_ptr<NetworkSelectorInterface> CreateNetworkSelector() {
- return brillo::make_unique_ptr(new NetworkSelectorAndroid());
+ return std::make_unique<NetworkSelectorAndroid>();
}
} // namespace network
diff --git a/network_selector_stub.cc b/network_selector_stub.cc
index 218d454..67925f4 100644
--- a/network_selector_stub.cc
+++ b/network_selector_stub.cc
@@ -16,8 +16,9 @@
#include "update_engine/network_selector_stub.h"
+#include <memory>
+
#include <base/logging.h>
-#include <brillo/make_unique_ptr.h>
namespace chromeos_update_engine {
@@ -25,7 +26,7 @@
// Factory defined in network_selector.h.
std::unique_ptr<NetworkSelectorInterface> CreateNetworkSelector() {
- return brillo::make_unique_ptr(new NetworkSelectorStub());
+ return std::make_unique<NetworkSelectorStub>();
}
} // namespace network
diff --git a/omaha_request_action.cc b/omaha_request_action.cc
index 4332702..c4db0c7 100644
--- a/omaha_request_action.cc
+++ b/omaha_request_action.cc
@@ -668,6 +668,11 @@
<< "powerwash_count is " << powerwash_count;
return false;
}
+ if (system_state_->hardware()->GetFirstActiveOmahaPingSent()) {
+ LOG(INFO) << "Not sending ping with a=-1 r=-1 to omaha because "
+ << "the first_active_omaha_ping_sent is true";
+ return false;
+ }
return true;
}
return ping_active_days_ > 0 || ping_roll_call_days_ > 0;
@@ -1145,6 +1150,16 @@
LOG_IF(ERROR, !UpdateLastPingDays(&parser_data, system_state_->prefs()))
<< "Failed to update the last ping day preferences!";
+ // Sets first_active_omaha_ping_sent to true (vpd in CrOS). We only do this if
+ // we have got a response from omaha and if its value has never been set to
+ // true before. Failure of this function should be ignored. There should be no
+ // need to check if a=-1 has been sent because older devices have already sent
+ // their a=-1 in the past and we have to set first_active_omaha_ping_sent for
+ // future checks.
+ if (!system_state_->hardware()->GetFirstActiveOmahaPingSent()) {
+ system_state_->hardware()->SetFirstActiveOmahaPingSent();
+ }
+
if (!HasOutputPipe()) {
// Just set success to whether or not the http transfer succeeded,
// which must be true at this point in the code.
diff --git a/omaha_request_action_unittest.cc b/omaha_request_action_unittest.cc
index a189b52..d57abe5 100644
--- a/omaha_request_action_unittest.cc
+++ b/omaha_request_action_unittest.cc
@@ -18,18 +18,19 @@
#include <stdint.h>
+#include <memory>
#include <string>
#include <vector>
#include <base/bind.h>
#include <base/files/file_util.h>
#include <base/files/scoped_temp_dir.h>
+#include <base/memory/ptr_util.h>
#include <base/strings/string_number_conversions.h>
#include <base/strings/string_util.h>
#include <base/strings/stringprintf.h>
#include <base/time/time.h>
#include <brillo/bind_lambda.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/fake_message_loop.h>
#include <brillo/message_loops/message_loop.h>
#include <brillo/message_loops/message_loop_utils.h>
@@ -62,7 +63,7 @@
using testing::Return;
using testing::ReturnPointee;
using testing::SaveArg;
-using testing::SetArgumentPointee;
+using testing::SetArgPointee;
using testing::_;
namespace {
@@ -378,7 +379,7 @@
fake_system_state_.set_request_params(request_params);
OmahaRequestAction action(&fake_system_state_,
nullptr,
- brillo::make_unique_ptr(fetcher),
+ base::WrapUnique(fetcher),
ping_only);
OmahaRequestActionTestProcessorDelegate delegate;
delegate.expected_code_ = expected_code;
@@ -430,7 +431,7 @@
fake_system_state.set_request_params(¶ms);
OmahaRequestAction action(&fake_system_state,
event,
- brillo::make_unique_ptr(fetcher),
+ base::WrapUnique(fetcher),
false);
OmahaRequestActionTestProcessorDelegate delegate;
ActionProcessor processor;
@@ -734,8 +735,8 @@
MockHttpFetcher* fetcher =
new MockHttpFetcher(http_response.data(), http_response.size(), nullptr);
- OmahaRequestAction action(
- &fake_system_state_, nullptr, brillo::make_unique_ptr(fetcher), false);
+ OmahaRequestAction action(&fake_system_state_, nullptr,
+ base::WrapUnique(fetcher), false);
ActionProcessor processor;
processor.EnqueueAction(&action);
@@ -761,8 +762,8 @@
EXPECT_CALL(mock_cm, GetConnectionProperties(_, _))
.WillRepeatedly(
- DoAll(SetArgumentPointee<0>(ConnectionType::kEthernet),
- SetArgumentPointee<1>(ConnectionTethering::kUnknown),
+ DoAll(SetArgPointee<0>(ConnectionType::kEthernet),
+ SetArgPointee<1>(ConnectionTethering::kUnknown),
Return(true)));
EXPECT_CALL(mock_cm, IsUpdateAllowedOver(ConnectionType::kEthernet, _))
.WillRepeatedly(Return(false));
@@ -1216,12 +1217,13 @@
OmahaRequestParams params = request_params_;
fake_system_state_.set_request_params(¶ms);
- OmahaRequestAction action(&fake_system_state_, nullptr,
- brillo::make_unique_ptr(
- new MockHttpFetcher(http_response.data(),
- http_response.size(),
- nullptr)),
- false);
+ OmahaRequestAction action(
+ &fake_system_state_,
+ nullptr,
+ std::make_unique<MockHttpFetcher>(http_response.data(),
+ http_response.size(),
+ nullptr),
+ false);
OmahaRequestActionTestProcessorDelegate delegate;
ActionProcessor processor;
processor.set_delegate(&delegate);
@@ -1392,12 +1394,13 @@
loop.SetAsCurrent();
string http_response("doesn't matter");
- OmahaRequestAction action(&fake_system_state_, nullptr,
- brillo::make_unique_ptr(
- new MockHttpFetcher(http_response.data(),
- http_response.size(),
- nullptr)),
- false);
+ OmahaRequestAction action(
+ &fake_system_state_,
+ nullptr,
+ std::make_unique<MockHttpFetcher>(http_response.data(),
+ http_response.size(),
+ nullptr),
+ false);
TerminateEarlyTestProcessorDelegate delegate;
ActionProcessor processor;
processor.set_delegate(&delegate);
@@ -1532,7 +1535,7 @@
fake_system_state_.set_prefs(&prefs);
EXPECT_CALL(prefs, GetString(kPrefsPreviousVersion, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(string("")), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(string("")), Return(true)));
// An existing but empty previous version means that we didn't reboot to a new
// update, therefore, no need to update the previous version.
EXPECT_CALL(prefs, SetString(kPrefsPreviousVersion, _)).Times(0);
@@ -1608,10 +1611,9 @@
OmahaRequestAction update_check_action(
&fake_system_state_,
nullptr,
- brillo::make_unique_ptr(
- new MockHttpFetcher(http_response.data(),
- http_response.size(),
- nullptr)),
+ std::make_unique<MockHttpFetcher>(http_response.data(),
+ http_response.size(),
+ nullptr),
false);
EXPECT_FALSE(update_check_action.IsEvent());
@@ -1620,10 +1622,9 @@
OmahaRequestAction event_action(
&fake_system_state_,
new OmahaEvent(OmahaEvent::kTypeUpdateComplete),
- brillo::make_unique_ptr(
- new MockHttpFetcher(http_response.data(),
- http_response.size(),
- nullptr)),
+ std::make_unique<MockHttpFetcher>(http_response.data(),
+ http_response.size(),
+ nullptr),
false);
EXPECT_TRUE(event_action.IsEvent());
}
@@ -1740,11 +1741,11 @@
int64_t six_days_ago =
(Time::Now() - TimeDelta::FromHours(6 * 24 + 11)).ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsInstallDateDays, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(0), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(0), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(six_days_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(six_days_ago), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(five_days_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(five_days_ago), Return(true)));
brillo::Blob post_data;
ASSERT_TRUE(TestUpdateCheck(nullptr, // request_params
fake_update_response_.GetNoUpdateResponse(),
@@ -1786,11 +1787,11 @@
(Time::Now() - TimeDelta::FromHours(3 * 24 + 12)).ToInternalValue();
int64_t now = Time::Now().ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsInstallDateDays, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(0), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(0), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(three_days_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(three_days_ago), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(now), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(now), Return(true)));
brillo::Blob post_data;
ASSERT_TRUE(
TestUpdateCheck(nullptr, // request_params
@@ -1818,11 +1819,11 @@
(Time::Now() - TimeDelta::FromHours(4 * 24)).ToInternalValue();
int64_t now = Time::Now().ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsInstallDateDays, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(0), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(0), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(now), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(now), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(four_days_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(four_days_ago), Return(true)));
brillo::Blob post_data;
ASSERT_TRUE(
TestUpdateCheck(nullptr, // request_params
@@ -1849,11 +1850,11 @@
int64_t one_hour_ago =
(Time::Now() - TimeDelta::FromHours(1)).ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsInstallDateDays, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(0), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(0), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(one_hour_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(one_hour_ago), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(one_hour_ago), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(one_hour_ago), Return(true)));
// LastActivePingDay and PrefsLastRollCallPingDay are set even if we didn't
// send a ping.
EXPECT_CALL(prefs, SetInt64(kPrefsLastActivePingDay, _))
@@ -1882,9 +1883,9 @@
fake_system_state_.set_prefs(&prefs);
int64_t now = Time::Now().ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(now), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(now), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(now), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(now), Return(true)));
EXPECT_CALL(prefs, SetInt64(kPrefsLastActivePingDay, _)).Times(0);
EXPECT_CALL(prefs, SetInt64(kPrefsLastRollCallPingDay, _)).Times(0);
brillo::Blob post_data;
@@ -1911,11 +1912,11 @@
int64_t future =
(Time::Now() + TimeDelta::FromHours(3 * 24 + 4)).ToInternalValue();
EXPECT_CALL(prefs, GetInt64(kPrefsInstallDateDays, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(0), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(0), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastActivePingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(future), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(future), Return(true)));
EXPECT_CALL(prefs, GetInt64(kPrefsLastRollCallPingDay, _))
- .WillOnce(DoAll(SetArgumentPointee<1>(future), Return(true)));
+ .WillOnce(DoAll(SetArgPointee<1>(future), Return(true)));
EXPECT_CALL(prefs, SetInt64(kPrefsLastActivePingDay, _))
.WillOnce(Return(true));
EXPECT_CALL(prefs, SetInt64(kPrefsLastRollCallPingDay, _))
@@ -2265,6 +2266,35 @@
EXPECT_EQ(string::npos, post_str.find("<ping"));
}
+// Checks that the initial ping with a=-1 r=-1 is not send when the device
+// first_active_omaha_ping_sent is set.
+TEST_F(OmahaRequestActionTest, PingWhenFirstActiveOmahaPingIsSent) {
+ fake_prefs_.SetString(kPrefsPreviousVersion, "");
+
+ // Flag that the device was not powerwashed in the past.
+ fake_system_state_.fake_hardware()->SetPowerwashCount(0);
+
+ // Flag that the device has sent first active ping in the past.
+ fake_system_state_.fake_hardware()->SetFirstActiveOmahaPingSent();
+
+ brillo::Blob post_data;
+ ASSERT_TRUE(
+ TestUpdateCheck(nullptr, // request_params
+ fake_update_response_.GetNoUpdateResponse(),
+ -1,
+ false, // ping_only
+ ErrorCode::kSuccess,
+ metrics::CheckResult::kNoUpdateAvailable,
+ metrics::CheckReaction::kUnset,
+ metrics::DownloadErrorCode::kUnset,
+ nullptr,
+ &post_data));
+ // We shouldn't send a ping in this case since
+ // first_active_omaha_ping_sent=true
+ string post_str(post_data.begin(), post_data.end());
+ EXPECT_EQ(string::npos, post_str.find("<ping"));
+}
+
// Checks that the event 54 is sent on a reboot to a new update.
TEST_F(OmahaRequestActionTest, RebootAfterUpdateEvent) {
// Flag that the device was updated in a previous boot.
diff --git a/payload_consumer/bzip_extent_writer.cc b/payload_consumer/bzip_extent_writer.cc
index 0fcc8ba..39d9d67 100644
--- a/payload_consumer/bzip_extent_writer.cc
+++ b/payload_consumer/bzip_extent_writer.cc
@@ -16,7 +16,7 @@
#include "update_engine/payload_consumer/bzip_extent_writer.h"
-using std::vector;
+using google::protobuf::RepeatedPtrField;
namespace chromeos_update_engine {
@@ -25,7 +25,7 @@
}
bool BzipExtentWriter::Init(FileDescriptorPtr fd,
- const vector<Extent>& extents,
+ const RepeatedPtrField<Extent>& extents,
uint32_t block_size) {
// Init bzip2 stream
int rc = BZ2_bzDecompressInit(&stream_,
diff --git a/payload_consumer/bzip_extent_writer.h b/payload_consumer/bzip_extent_writer.h
index 0ad542e..86b346a 100644
--- a/payload_consumer/bzip_extent_writer.h
+++ b/payload_consumer/bzip_extent_writer.h
@@ -19,7 +19,7 @@
#include <bzlib.h>
#include <memory>
-#include <vector>
+#include <utility>
#include <brillo/secure_blob.h>
@@ -41,7 +41,7 @@
~BzipExtentWriter() override = default;
bool Init(FileDescriptorPtr fd,
- const std::vector<Extent>& extents,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
uint32_t block_size) override;
bool Write(const void* bytes, size_t count) override;
bool EndImpl() override;
diff --git a/payload_consumer/bzip_extent_writer_unittest.cc b/payload_consumer/bzip_extent_writer_unittest.cc
index 8ac3e59..bf050ef 100644
--- a/payload_consumer/bzip_extent_writer_unittest.cc
+++ b/payload_consumer/bzip_extent_writer_unittest.cc
@@ -19,15 +19,17 @@
#include <fcntl.h>
#include <algorithm>
+#include <memory>
#include <string>
#include <vector>
-#include <brillo/make_unique_ptr.h>
#include <gtest/gtest.h>
#include "update_engine/common/test_utils.h"
#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+using google::protobuf::RepeatedPtrField;
using std::min;
using std::string;
using std::vector;
@@ -55,11 +57,7 @@
};
TEST_F(BzipExtentWriterTest, SimpleTest) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(0);
- extent.set_num_blocks(1);
- extents.push_back(extent);
+ vector<Extent> extents = {ExtentForRange(0, 1)};
// 'echo test | bzip2 | hexdump' yields:
static const char test_uncompressed[] = "test\n";
@@ -70,9 +68,9 @@
0x22, 0x9c, 0x28, 0x48, 0x66, 0x61, 0xb8, 0xea, 0x00,
};
- BzipExtentWriter bzip_writer(
- brillo::make_unique_ptr(new DirectExtentWriter()));
- EXPECT_TRUE(bzip_writer.Init(fd_, extents, kBlockSize));
+ BzipExtentWriter bzip_writer(std::make_unique<DirectExtentWriter>());
+ EXPECT_TRUE(
+ bzip_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
EXPECT_TRUE(bzip_writer.Write(test, sizeof(test)));
EXPECT_TRUE(bzip_writer.End());
@@ -102,15 +100,12 @@
for (size_t i = 0; i < decompressed_data.size(); ++i)
decompressed_data[i] = static_cast<uint8_t>("ABC\n"[i % 4]);
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(0);
- extent.set_num_blocks((kDecompressedLength + kBlockSize - 1) / kBlockSize);
- extents.push_back(extent);
+ vector<Extent> extents = {
+ ExtentForRange(0, (kDecompressedLength + kBlockSize - 1) / kBlockSize)};
- BzipExtentWriter bzip_writer(
- brillo::make_unique_ptr(new DirectExtentWriter()));
- EXPECT_TRUE(bzip_writer.Init(fd_, extents, kBlockSize));
+ BzipExtentWriter bzip_writer(std::make_unique<DirectExtentWriter>());
+ EXPECT_TRUE(
+ bzip_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
brillo::Blob original_compressed_data = compressed_data;
for (brillo::Blob::size_type i = 0; i < compressed_data.size();
diff --git a/payload_consumer/cached_file_descriptor.cc b/payload_consumer/cached_file_descriptor.cc
new file mode 100644
index 0000000..7f2515e
--- /dev/null
+++ b/payload_consumer/cached_file_descriptor.cc
@@ -0,0 +1,98 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/cached_file_descriptor.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+
+#include <base/logging.h>
+
+#include "update_engine/common/utils.h"
+
+namespace chromeos_update_engine {
+
+off64_t CachedFileDescriptor::Seek(off64_t offset, int whence) {
+ // Only support SEEK_SET and SEEK_CUR. I think these two would be enough. If
+ // we want to support SEEK_END then we have to figure out the size of the
+ // underlying file descriptor each time and it may not be a very good idea.
+ CHECK(whence == SEEK_SET || whence == SEEK_CUR);
+ off64_t next_offset = whence == SEEK_SET ? offset : offset_ + offset;
+
+ if (next_offset != offset_) {
+ // We sought somewhere other than what we are now. So we have to flush and
+ // move to the new offset.
+ if (!FlushCache()) {
+ return -1;
+ }
+ // Then we have to seek there.
+ if (fd_->Seek(next_offset, SEEK_SET) < 0) {
+ return -1;
+ }
+ offset_ = next_offset;
+ }
+ return offset_;
+}
+
+ssize_t CachedFileDescriptor::Write(const void* buf, size_t count) {
+ auto bytes = static_cast<const uint8_t*>(buf);
+ size_t total_bytes_wrote = 0;
+ while (total_bytes_wrote < count) {
+ auto bytes_to_cache =
+ std::min(count - total_bytes_wrote, cache_.size() - bytes_cached_);
+ if (bytes_to_cache > 0) { // Which means |cache_| is still have some space.
+ memcpy(cache_.data() + bytes_cached_,
+ bytes + total_bytes_wrote,
+ bytes_to_cache);
+ total_bytes_wrote += bytes_to_cache;
+ bytes_cached_ += bytes_to_cache;
+ }
+ if (bytes_cached_ == cache_.size()) {
+ // Cache is full; write it to the |fd_| as long as you can.
+ if (!FlushCache()) {
+ return -1;
+ }
+ }
+ }
+ offset_ += total_bytes_wrote;
+ return total_bytes_wrote;
+}
+
+bool CachedFileDescriptor::Flush() {
+ return FlushCache() && fd_->Flush();
+}
+
+bool CachedFileDescriptor::Close() {
+ offset_ = 0;
+ return FlushCache() && fd_->Close();
+}
+
+bool CachedFileDescriptor::FlushCache() {
+ size_t begin = 0;
+ while (begin < bytes_cached_) {
+ auto bytes_wrote = fd_->Write(cache_.data() + begin, bytes_cached_ - begin);
+ if (bytes_wrote < 0) {
+ PLOG(ERROR) << "Failed to flush cached data!";
+ return false;
+ }
+ begin += bytes_wrote;
+ }
+ bytes_cached_ = 0;
+ return true;
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/cached_file_descriptor.h b/payload_consumer/cached_file_descriptor.h
new file mode 100644
index 0000000..28c48f7
--- /dev/null
+++ b/payload_consumer/cached_file_descriptor.h
@@ -0,0 +1,76 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_CACHED_FILE_DESCRIPTOR_H_
+#define UPDATE_ENGINE_PAYLOAD_CONSUMER_CACHED_FILE_DESCRIPTOR_H_
+
+#include <errno.h>
+#include <sys/types.h>
+
+#include <memory>
+#include <vector>
+
+#include <brillo/secure_blob.h>
+
+#include "update_engine/payload_consumer/file_descriptor.h"
+
+namespace chromeos_update_engine {
+
+class CachedFileDescriptor : public FileDescriptor {
+ public:
+ CachedFileDescriptor(FileDescriptorPtr fd, size_t cache_size) : fd_(fd) {
+ cache_.resize(cache_size);
+ }
+ ~CachedFileDescriptor() override = default;
+
+ bool Open(const char* path, int flags, mode_t mode) override {
+ return fd_->Open(path, flags, mode);
+ }
+ bool Open(const char* path, int flags) override {
+ return fd_->Open(path, flags);
+ }
+ ssize_t Read(void* buf, size_t count) override {
+ return fd_->Read(buf, count);
+ }
+ ssize_t Write(const void* buf, size_t count) override;
+ off64_t Seek(off64_t offset, int whence) override;
+ uint64_t BlockDevSize() override { return fd_->BlockDevSize(); }
+ bool BlkIoctl(int request,
+ uint64_t start,
+ uint64_t length,
+ int* result) override {
+ return fd_->BlkIoctl(request, start, length, result);
+ }
+ bool Flush() override;
+ bool Close() override;
+ bool IsSettingErrno() override { return fd_->IsSettingErrno(); }
+ bool IsOpen() override { return fd_->IsOpen(); }
+
+ private:
+ // Internal flush without the need to call |fd_->Flush()|.
+ bool FlushCache();
+
+ FileDescriptorPtr fd_;
+ brillo::Blob cache_;
+ size_t bytes_cached_{0};
+ off64_t offset_{0};
+
+ DISALLOW_COPY_AND_ASSIGN(CachedFileDescriptor);
+};
+
+} // namespace chromeos_update_engine
+
+#endif // UPDATE_ENGINE_PAYLOAD_CONSUMER_CACHED_FILE_DESCRIPTOR_H_
diff --git a/payload_consumer/cached_file_descriptor_unittest.cc b/payload_consumer/cached_file_descriptor_unittest.cc
new file mode 100644
index 0000000..6a6302a
--- /dev/null
+++ b/payload_consumer/cached_file_descriptor_unittest.cc
@@ -0,0 +1,204 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/cached_file_descriptor.h"
+
+#include <fcntl.h>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+
+using chromeos_update_engine::test_utils::ExpectVectorsEq;
+using std::min;
+using std::string;
+using std::vector;
+
+namespace chromeos_update_engine {
+
+namespace {
+const size_t kCacheSize = 100;
+const size_t kFileSize = 1024;
+const size_t kRandomIterations = 1000;
+} // namespace
+
+class CachedFileDescriptorTest : public ::testing::Test {
+ public:
+ void Open() {
+ cfd_.reset(new CachedFileDescriptor(fd_, kCacheSize));
+ EXPECT_TRUE(cfd_->Open(temp_file_.path().c_str(), O_RDWR, 0600));
+ }
+
+ void Write(uint8_t* buffer, size_t count) {
+ size_t total_bytes_wrote = 0;
+ while (total_bytes_wrote < count) {
+ auto bytes_wrote =
+ cfd_->Write(buffer + total_bytes_wrote, count - total_bytes_wrote);
+ ASSERT_NE(bytes_wrote, -1);
+ total_bytes_wrote += bytes_wrote;
+ }
+ }
+
+ void Close() { EXPECT_TRUE(cfd_->Close()); }
+
+ void SetUp() override {
+ brillo::Blob zero_blob(kFileSize, 0);
+ EXPECT_TRUE(utils::WriteFile(
+ temp_file_.path().c_str(), zero_blob.data(), zero_blob.size()));
+ Open();
+ }
+
+ void TearDown() override {
+ Close();
+ EXPECT_FALSE(cfd_->IsOpen());
+ }
+
+ protected:
+ FileDescriptorPtr fd_{new EintrSafeFileDescriptor};
+ test_utils::ScopedTempFile temp_file_{"CachedFileDescriptor-file.XXXXXX"};
+ int value_{1};
+ FileDescriptorPtr cfd_;
+};
+
+TEST_F(CachedFileDescriptorTest, IsOpenTest) {
+ EXPECT_TRUE(cfd_->IsOpen());
+}
+
+TEST_F(CachedFileDescriptorTest, SimpleWriteTest) {
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+ brillo::Blob blob_in(kFileSize, value_);
+ Write(blob_in.data(), blob_in.size());
+ EXPECT_TRUE(cfd_->Flush());
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, OneBytePerWriteTest) {
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+ brillo::Blob blob_in(kFileSize, value_);
+ for (size_t idx = 0; idx < blob_in.size(); idx++) {
+ Write(&blob_in[idx], 1);
+ }
+ EXPECT_TRUE(cfd_->Flush());
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, RandomWriteTest) {
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+
+ brillo::Blob blob_in(kFileSize, 0);
+ srand(time(nullptr));
+ uint32_t rand_seed;
+ for (size_t idx = 0; idx < kRandomIterations; idx++) {
+ // zero to full size available.
+ size_t start = rand_r(&rand_seed) % blob_in.size();
+ size_t size = rand_r(&rand_seed) % (blob_in.size() - start);
+ std::fill_n(&blob_in[start], size, idx % 256);
+ EXPECT_EQ(cfd_->Seek(start, SEEK_SET), static_cast<off64_t>(start));
+ Write(&blob_in[start], size);
+ }
+ EXPECT_TRUE(cfd_->Flush());
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, SeekTest) {
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+ EXPECT_EQ(cfd_->Seek(1, SEEK_SET), 1);
+ EXPECT_EQ(cfd_->Seek(kFileSize - 1, SEEK_SET),
+ static_cast<off64_t>(kFileSize - 1));
+ EXPECT_EQ(cfd_->Seek(kFileSize, SEEK_SET), static_cast<off64_t>(kFileSize));
+ EXPECT_EQ(cfd_->Seek(kFileSize + 1, SEEK_SET),
+ static_cast<off64_t>(kFileSize + 1));
+
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+ EXPECT_EQ(cfd_->Seek(1, SEEK_CUR), 1);
+ EXPECT_EQ(cfd_->Seek(1, SEEK_CUR), 2);
+ EXPECT_EQ(cfd_->Seek(kFileSize - 1, SEEK_SET),
+ static_cast<off64_t>(kFileSize - 1));
+ EXPECT_EQ(cfd_->Seek(1, SEEK_CUR), static_cast<off64_t>(kFileSize));
+ EXPECT_EQ(cfd_->Seek(1, SEEK_CUR), static_cast<off64_t>(kFileSize + 1));
+}
+
+TEST_F(CachedFileDescriptorTest, NoFlushTest) {
+ EXPECT_EQ(cfd_->Seek(0, SEEK_SET), 0);
+ brillo::Blob blob_in(kFileSize, value_);
+ Write(blob_in.data(), blob_in.size());
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_NE(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, CacheSizeWriteTest) {
+ off64_t seek = 10;
+ brillo::Blob blob_in(kFileSize, 0);
+ std::fill_n(&blob_in[seek], kCacheSize, value_);
+ // We are writing exactly one cache size; Then it should be commited.
+ EXPECT_EQ(cfd_->Seek(seek, SEEK_SET), seek);
+ Write(&blob_in[seek], kCacheSize);
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, UnderCacheSizeWriteTest) {
+ off64_t seek = 100;
+ size_t less_than_cache_size = kCacheSize - 1;
+ EXPECT_EQ(cfd_->Seek(seek, SEEK_SET), seek);
+ brillo::Blob blob_in(kFileSize, 0);
+ std::fill_n(&blob_in[seek], less_than_cache_size, value_);
+ // We are writing less than one cache size; then it should not be commited.
+ Write(&blob_in[seek], less_than_cache_size);
+
+ // Revert the changes in |blob_in|.
+ std::fill_n(&blob_in[seek], less_than_cache_size, 0);
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+TEST_F(CachedFileDescriptorTest, SeekAfterWriteTest) {
+ off64_t seek = 100;
+ size_t less_than_cache_size = kCacheSize - 3;
+ EXPECT_EQ(cfd_->Seek(seek, SEEK_SET), seek);
+ brillo::Blob blob_in(kFileSize, 0);
+ std::fill_n(&blob_in[seek], less_than_cache_size, value_);
+ // We are writing less than one cache size; then it should not be commited.
+ Write(&blob_in[seek], less_than_cache_size);
+
+ // Then we seek, it should've written the cache after seek.
+ EXPECT_EQ(cfd_->Seek(200, SEEK_SET), 200);
+
+ brillo::Blob blob_out;
+ EXPECT_TRUE(utils::ReadFile(temp_file_.path(), &blob_out));
+ EXPECT_EQ(blob_in, blob_out);
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/delta_performer.cc b/payload_consumer/delta_performer.cc
index f74506f..001c84a 100644
--- a/payload_consumer/delta_performer.cc
+++ b/payload_consumer/delta_performer.cc
@@ -29,13 +29,15 @@
#include <base/files/file_util.h>
#include <base/format_macros.h>
+#include <base/metrics/histogram_macros.h>
#include <base/strings/string_number_conversions.h>
#include <base/strings/string_util.h>
#include <base/strings/stringprintf.h>
+#include <base/time/time.h>
#include <brillo/data_encoding.h>
-#include <brillo/make_unique_ptr.h>
#include <bsdiff/bspatch.h>
#include <google/protobuf/repeated_field.h>
+#include <puffin/puffpatch.h>
#include "update_engine/common/constants.h"
#include "update_engine/common/hardware_interface.h"
@@ -43,7 +45,9 @@
#include "update_engine/common/subprocess.h"
#include "update_engine/common/terminator.h"
#include "update_engine/payload_consumer/bzip_extent_writer.h"
+#include "update_engine/payload_consumer/cached_file_descriptor.h"
#include "update_engine/payload_consumer/download_action.h"
+#include "update_engine/payload_consumer/extent_reader.h"
#include "update_engine/payload_consumer/extent_writer.h"
#include "update_engine/payload_consumer/file_descriptor_utils.h"
#if USE_MTD
@@ -68,7 +72,7 @@
const uint64_t DeltaPerformer::kDeltaMetadataSignatureSizeSize = 4;
const uint64_t DeltaPerformer::kMaxPayloadHeaderSize = 24;
const uint64_t DeltaPerformer::kSupportedMajorPayloadVersion = 2;
-const uint32_t DeltaPerformer::kSupportedMinorPayloadVersion = 3;
+const uint32_t DeltaPerformer::kSupportedMinorPayloadVersion = 4;
const unsigned DeltaPerformer::kProgressLogMaxChunks = 10;
const unsigned DeltaPerformer::kProgressLogTimeoutSeconds = 30;
@@ -82,6 +86,8 @@
const int kUbiVolumeAttachTimeout = 5 * 60;
#endif
+const uint64_t kCacheSize = 1024 * 1024; // 1MB
+
FileDescriptorPtr CreateFileDescriptor(const char* path) {
FileDescriptorPtr ret;
#if USE_MTD
@@ -112,12 +118,20 @@
// Opens path for read/write. On success returns an open FileDescriptor
// and sets *err to 0. On failure, sets *err to errno and returns nullptr.
-FileDescriptorPtr OpenFile(const char* path, int mode, int* err) {
+FileDescriptorPtr OpenFile(const char* path,
+ int mode,
+ bool cache_writes,
+ int* err) {
// Try to mark the block device read-only based on the mode. Ignore any
// failure since this won't work when passing regular files.
- utils::SetBlockDeviceReadOnly(path, (mode & O_ACCMODE) == O_RDONLY);
+ bool read_only = (mode & O_ACCMODE) == O_RDONLY;
+ utils::SetBlockDeviceReadOnly(path, read_only);
FileDescriptorPtr fd = CreateFileDescriptor(path);
+ if (cache_writes && !read_only) {
+ fd = FileDescriptorPtr(new CachedFileDescriptor(fd, kCacheSize));
+ LOG(INFO) << "Caching writes.";
+ }
#if USE_MTD
// On NAND devices, we can either read, or write, but not both. So here we
// use O_WRONLY.
@@ -347,7 +361,7 @@
GetMinorVersion() != kInPlaceMinorPayloadVersion) {
source_path_ = install_part.source_path;
int err;
- source_fd_ = OpenFile(source_path_.c_str(), O_RDONLY, &err);
+ source_fd_ = OpenFile(source_path_.c_str(), O_RDONLY, false, &err);
if (!source_fd_) {
LOG(ERROR) << "Unable to open source partition "
<< partition.partition_name() << " on slot "
@@ -359,7 +373,15 @@
target_path_ = install_part.target_path;
int err;
- target_fd_ = OpenFile(target_path_.c_str(), O_RDWR, &err);
+
+ int flags = O_RDWR;
+ if (!is_interactive_)
+ flags |= O_DSYNC;
+
+ LOG(INFO) << "Opening " << target_path_ << " partition with"
+ << (is_interactive_ ? "out" : "") << " O_DSYNC";
+
+ target_fd_ = OpenFile(target_path_.c_str(), flags, true, &err);
if (!target_fd_) {
LOG(ERROR) << "Unable to open target partition "
<< partition.partition_name() << " on slot "
@@ -585,6 +607,15 @@
return kMetadataParseSuccess;
}
+#define OP_DURATION_HISTOGRAM(_op_name, _start_time) \
+ LOCAL_HISTOGRAM_CUSTOM_TIMES( \
+ "UpdateEngine.DownloadAction.InstallOperation::" \
+ _op_name ".Duration", \
+ base::TimeTicks::Now() - _start_time, \
+ base::TimeDelta::FromMilliseconds(10), \
+ base::TimeDelta::FromMinutes(5), \
+ 20);
+
// Wrapper around write. Returns true if all requested bytes
// were written, or false on any error, regardless of progress
// and stores an action exit code in |error|.
@@ -719,32 +750,41 @@
ScopedTerminatorExitUnblocker exit_unblocker =
ScopedTerminatorExitUnblocker(); // Avoids a compiler unused var bug.
+ base::TimeTicks op_start_time = base::TimeTicks::Now();
+
bool op_result;
switch (op.type()) {
case InstallOperation::REPLACE:
case InstallOperation::REPLACE_BZ:
case InstallOperation::REPLACE_XZ:
op_result = PerformReplaceOperation(op);
+ OP_DURATION_HISTOGRAM("REPLACE", op_start_time);
break;
case InstallOperation::ZERO:
case InstallOperation::DISCARD:
op_result = PerformZeroOrDiscardOperation(op);
+ OP_DURATION_HISTOGRAM("ZERO_OR_DISCARD", op_start_time);
break;
case InstallOperation::MOVE:
op_result = PerformMoveOperation(op);
+ OP_DURATION_HISTOGRAM("MOVE", op_start_time);
break;
case InstallOperation::BSDIFF:
op_result = PerformBsdiffOperation(op);
+ OP_DURATION_HISTOGRAM("BSDIFF", op_start_time);
break;
case InstallOperation::SOURCE_COPY:
op_result = PerformSourceCopyOperation(op, error);
+ OP_DURATION_HISTOGRAM("SOURCE_COPY", op_start_time);
break;
case InstallOperation::SOURCE_BSDIFF:
+ case InstallOperation::BROTLI_BSDIFF:
op_result = PerformSourceBsdiffOperation(op, error);
+ OP_DURATION_HISTOGRAM("SOURCE_BSDIFF", op_start_time);
break;
case InstallOperation::PUFFDIFF:
- // TODO(ahassani): Later add PerformPuffdiffOperation(op, error);
- op_result = false;
+ op_result = PerformPuffDiffOperation(op, error);
+ OP_DURATION_HISTOGRAM("PUFFDIFF", op_start_time);
break;
default:
op_result = false;
@@ -752,6 +792,10 @@
if (!HandleOpResult(op_result, InstallOperationTypeName(op.type()), error))
return false;
+ if (!target_fd_->Flush()) {
+ return false;
+ }
+
next_operation_num_++;
UpdateOverallProgress(false, "Completed ");
CheckpointUpdateProgress();
@@ -919,9 +963,8 @@
}
// Setup the ExtentWriter stack based on the operation type.
- std::unique_ptr<ExtentWriter> writer =
- brillo::make_unique_ptr(new ZeroPadExtentWriter(
- brillo::make_unique_ptr(new DirectExtentWriter())));
+ std::unique_ptr<ExtentWriter> writer = std::make_unique<ZeroPadExtentWriter>(
+ std::make_unique<DirectExtentWriter>());
if (operation.type() == InstallOperation::REPLACE_BZ) {
writer.reset(new BzipExtentWriter(std::move(writer)));
@@ -929,13 +972,8 @@
writer.reset(new XzExtentWriter(std::move(writer)));
}
- // Create a vector of extents to pass to the ExtentWriter.
- vector<Extent> extents;
- for (int i = 0; i < operation.dst_extents_size(); i++) {
- extents.push_back(operation.dst_extents(i));
- }
-
- TEST_AND_RETURN_FALSE(writer->Init(target_fd_, extents, block_size_));
+ TEST_AND_RETURN_FALSE(
+ writer->Init(target_fd_, operation.dst_extents(), block_size_));
TEST_AND_RETURN_FALSE(writer->Write(buffer_.data(), operation.data_length()));
TEST_AND_RETURN_FALSE(writer->End());
@@ -1160,6 +1198,96 @@
return true;
}
+bool DeltaPerformer::CalculateAndValidateSourceHash(
+ const InstallOperation& operation, ErrorCode* error) {
+ const uint64_t kMaxBlocksToRead = 256; // 1MB if block size is 4KB
+ auto total_blocks = utils::BlocksInExtents(operation.src_extents());
+ brillo::Blob buf(std::min(kMaxBlocksToRead, total_blocks) * block_size_);
+ DirectExtentReader reader;
+ TEST_AND_RETURN_FALSE(
+ reader.Init(source_fd_, operation.src_extents(), block_size_));
+ HashCalculator source_hasher;
+ while (total_blocks > 0) {
+ auto read_blocks = std::min(total_blocks, kMaxBlocksToRead);
+ TEST_AND_RETURN_FALSE(reader.Read(buf.data(), read_blocks * block_size_));
+ TEST_AND_RETURN_FALSE(
+ source_hasher.Update(buf.data(), read_blocks * block_size_));
+ total_blocks -= read_blocks;
+ }
+ TEST_AND_RETURN_FALSE(source_hasher.Finalize());
+ TEST_AND_RETURN_FALSE(
+ ValidateSourceHash(source_hasher.raw_hash(), operation, error));
+ return true;
+}
+
+namespace {
+
+class BsdiffExtentFile : public bsdiff::FileInterface {
+ public:
+ BsdiffExtentFile(std::unique_ptr<ExtentReader> reader, size_t size)
+ : BsdiffExtentFile(std::move(reader), nullptr, size) {}
+ BsdiffExtentFile(std::unique_ptr<ExtentWriter> writer, size_t size)
+ : BsdiffExtentFile(nullptr, std::move(writer), size) {}
+
+ ~BsdiffExtentFile() override = default;
+
+ bool Read(void* buf, size_t count, size_t* bytes_read) override {
+ TEST_AND_RETURN_FALSE(reader_->Read(buf, count));
+ *bytes_read = count;
+ offset_ += count;
+ return true;
+ }
+
+ bool Write(const void* buf, size_t count, size_t* bytes_written) override {
+ TEST_AND_RETURN_FALSE(writer_->Write(buf, count));
+ *bytes_written = count;
+ offset_ += count;
+ return true;
+ }
+
+ bool Seek(off_t pos) override {
+ if (reader_ != nullptr) {
+ TEST_AND_RETURN_FALSE(reader_->Seek(pos));
+ offset_ = pos;
+ } else {
+ // For writes technically there should be no change of position, or it
+ // should be equivalent of current offset.
+ TEST_AND_RETURN_FALSE(offset_ == static_cast<uint64_t>(pos));
+ }
+ return true;
+ }
+
+ bool Close() override {
+ if (writer_ != nullptr) {
+ TEST_AND_RETURN_FALSE(writer_->End());
+ }
+ return true;
+ }
+
+ bool GetSize(uint64_t* size) override {
+ *size = size_;
+ return true;
+ }
+
+ private:
+ BsdiffExtentFile(std::unique_ptr<ExtentReader> reader,
+ std::unique_ptr<ExtentWriter> writer,
+ size_t size)
+ : reader_(std::move(reader)),
+ writer_(std::move(writer)),
+ size_(size),
+ offset_(0) {}
+
+ std::unique_ptr<ExtentReader> reader_;
+ std::unique_ptr<ExtentWriter> writer_;
+ uint64_t size_;
+ uint64_t offset_;
+
+ DISALLOW_COPY_AND_ASSIGN(BsdiffExtentFile);
+};
+
+} // namespace
+
bool DeltaPerformer::PerformSourceBsdiffOperation(
const InstallOperation& operation, ErrorCode* error) {
// Since we delete data off the beginning of the buffer as we use it,
@@ -1172,45 +1300,142 @@
TEST_AND_RETURN_FALSE(operation.dst_length() % block_size_ == 0);
if (operation.has_src_sha256_hash()) {
- HashCalculator source_hasher;
- const uint64_t kMaxBlocksToRead = 512; // 2MB if block size is 4KB
- brillo::Blob buf(kMaxBlocksToRead * block_size_);
- for (const Extent& extent : operation.src_extents()) {
- for (uint64_t i = 0; i < extent.num_blocks(); i += kMaxBlocksToRead) {
- uint64_t blocks_to_read = min(
- kMaxBlocksToRead, static_cast<uint64_t>(extent.num_blocks()) - i);
- ssize_t bytes_to_read = blocks_to_read * block_size_;
- ssize_t bytes_read_this_iteration = 0;
- TEST_AND_RETURN_FALSE(
- utils::PReadAll(source_fd_, buf.data(), bytes_to_read,
- (extent.start_block() + i) * block_size_,
- &bytes_read_this_iteration));
- TEST_AND_RETURN_FALSE(bytes_read_this_iteration == bytes_to_read);
- TEST_AND_RETURN_FALSE(source_hasher.Update(buf.data(), bytes_to_read));
- }
- }
- TEST_AND_RETURN_FALSE(source_hasher.Finalize());
- TEST_AND_RETURN_FALSE(
- ValidateSourceHash(source_hasher.raw_hash(), operation, error));
+ TEST_AND_RETURN_FALSE(CalculateAndValidateSourceHash(operation, error));
}
- string input_positions;
- TEST_AND_RETURN_FALSE(ExtentsToBsdiffPositionsString(operation.src_extents(),
- block_size_,
- operation.src_length(),
- &input_positions));
- string output_positions;
- TEST_AND_RETURN_FALSE(ExtentsToBsdiffPositionsString(operation.dst_extents(),
- block_size_,
- operation.dst_length(),
- &output_positions));
+ auto reader = std::make_unique<DirectExtentReader>();
+ TEST_AND_RETURN_FALSE(
+ reader->Init(source_fd_, operation.src_extents(), block_size_));
+ auto src_file = std::make_unique<BsdiffExtentFile>(
+ std::move(reader),
+ utils::BlocksInExtents(operation.src_extents()) * block_size_);
- TEST_AND_RETURN_FALSE(bsdiff::bspatch(source_path_.c_str(),
- target_path_.c_str(),
+ auto writer = std::make_unique<DirectExtentWriter>();
+ TEST_AND_RETURN_FALSE(
+ writer->Init(target_fd_, operation.dst_extents(), block_size_));
+ auto dst_file = std::make_unique<BsdiffExtentFile>(
+ std::move(writer),
+ utils::BlocksInExtents(operation.dst_extents()) * block_size_);
+
+ TEST_AND_RETURN_FALSE(bsdiff::bspatch(std::move(src_file),
+ std::move(dst_file),
buffer_.data(),
- buffer_.size(),
- input_positions.c_str(),
- output_positions.c_str()) == 0);
+ buffer_.size()) == 0);
+ DiscardBuffer(true, buffer_.size());
+ return true;
+}
+
+namespace {
+
+// A class to be passed to |puffpatch| for reading from |source_fd_| and writing
+// into |target_fd_|.
+class PuffinExtentStream : public puffin::StreamInterface {
+ public:
+ // Constructor for creating a stream for reading from an |ExtentReader|.
+ PuffinExtentStream(std::unique_ptr<ExtentReader> reader, size_t size)
+ : PuffinExtentStream(std::move(reader), nullptr, size) {}
+
+ // Constructor for creating a stream for writing to an |ExtentWriter|.
+ PuffinExtentStream(std::unique_ptr<ExtentWriter> writer, size_t size)
+ : PuffinExtentStream(nullptr, std::move(writer), size) {}
+
+ ~PuffinExtentStream() override = default;
+
+ bool GetSize(size_t* size) const override {
+ *size = size_;
+ return true;
+ }
+
+ bool GetOffset(size_t* offset) const override {
+ *offset = offset_;
+ return true;
+ }
+
+ bool Seek(size_t offset) override {
+ if (is_read_) {
+ TEST_AND_RETURN_FALSE(reader_->Seek(offset));
+ offset_ = offset;
+ } else {
+ // For writes technically there should be no change of position, or it
+ // should equivalent of current offset.
+ TEST_AND_RETURN_FALSE(offset_ == offset);
+ }
+ return true;
+ }
+
+ bool Read(void* buffer, size_t count) override {
+ TEST_AND_RETURN_FALSE(is_read_);
+ TEST_AND_RETURN_FALSE(reader_->Read(buffer, count));
+ offset_ += count;
+ return true;
+ }
+
+ bool Write(const void* buffer, size_t count) override {
+ TEST_AND_RETURN_FALSE(!is_read_);
+ TEST_AND_RETURN_FALSE(writer_->Write(buffer, count));
+ offset_ += count;
+ return true;
+ }
+
+ bool Close() override {
+ if (!is_read_) {
+ TEST_AND_RETURN_FALSE(writer_->End());
+ }
+ return true;
+ }
+
+ private:
+ PuffinExtentStream(std::unique_ptr<ExtentReader> reader,
+ std::unique_ptr<ExtentWriter> writer,
+ size_t size)
+ : reader_(std::move(reader)),
+ writer_(std::move(writer)),
+ size_(size),
+ offset_(0),
+ is_read_(reader_ ? true : false) {}
+
+ std::unique_ptr<ExtentReader> reader_;
+ std::unique_ptr<ExtentWriter> writer_;
+ uint64_t size_;
+ uint64_t offset_;
+ bool is_read_;
+
+ DISALLOW_COPY_AND_ASSIGN(PuffinExtentStream);
+};
+
+} // namespace
+
+bool DeltaPerformer::PerformPuffDiffOperation(const InstallOperation& operation,
+ ErrorCode* error) {
+ // Since we delete data off the beginning of the buffer as we use it,
+ // the data we need should be exactly at the beginning of the buffer.
+ TEST_AND_RETURN_FALSE(buffer_offset_ == operation.data_offset());
+ TEST_AND_RETURN_FALSE(buffer_.size() >= operation.data_length());
+
+ if (operation.has_src_sha256_hash()) {
+ TEST_AND_RETURN_FALSE(CalculateAndValidateSourceHash(operation, error));
+ }
+
+ auto reader = std::make_unique<DirectExtentReader>();
+ TEST_AND_RETURN_FALSE(
+ reader->Init(source_fd_, operation.src_extents(), block_size_));
+ puffin::UniqueStreamPtr src_stream(new PuffinExtentStream(
+ std::move(reader),
+ utils::BlocksInExtents(operation.src_extents()) * block_size_));
+
+ auto writer = std::make_unique<DirectExtentWriter>();
+ TEST_AND_RETURN_FALSE(
+ writer->Init(target_fd_, operation.dst_extents(), block_size_));
+ puffin::UniqueStreamPtr dst_stream(new PuffinExtentStream(
+ std::move(writer),
+ utils::BlocksInExtents(operation.dst_extents()) * block_size_));
+
+ const size_t kMaxCacheSize = 5 * 1024 * 1024; // Total 5MB cache.
+ TEST_AND_RETURN_FALSE(puffin::PuffPatch(std::move(src_stream),
+ std::move(dst_stream),
+ buffer_.data(),
+ buffer_.size(),
+ kMaxCacheSize));
DiscardBuffer(true, buffer_.size());
return true;
}
diff --git a/payload_consumer/delta_performer.h b/payload_consumer/delta_performer.h
index f363a4c..731e7f1 100644
--- a/payload_consumer/delta_performer.h
+++ b/payload_consumer/delta_performer.h
@@ -19,6 +19,7 @@
#include <inttypes.h>
+#include <limits>
#include <string>
#include <vector>
@@ -79,13 +80,15 @@
HardwareInterface* hardware,
DownloadActionDelegate* download_delegate,
InstallPlan* install_plan,
- InstallPlan::Payload* payload)
+ InstallPlan::Payload* payload,
+ bool is_interactive)
: prefs_(prefs),
boot_control_(boot_control),
hardware_(hardware),
download_delegate_(download_delegate),
install_plan_(install_plan),
- payload_(payload) {}
+ payload_(payload),
+ is_interactive_(is_interactive) {}
// FileWriter's Write implementation where caller doesn't care about
// error codes.
@@ -162,9 +165,9 @@
public_key_path_ = public_key_path;
}
- // Set |*out_offset| to the byte offset where the size of the metadata signature
- // is stored in a payload. Return true on success, if this field is not
- // present in the payload, return false.
+ // Set |*out_offset| to the byte offset where the size of the metadata
+ // signature is stored in a payload. Return true on success, if this field is
+ // not present in the payload, return false.
bool GetMetadataSignatureSizeOffset(uint64_t* out_offset) const;
// Set |*out_offset| to the byte offset at which the manifest protobuf begins
@@ -242,6 +245,10 @@
// buffer.
ErrorCode ValidateMetadataSignature(const brillo::Blob& payload);
+ // Calculates and validates the source hash of the operation |operation|.
+ bool CalculateAndValidateSourceHash(const InstallOperation& operation,
+ ErrorCode* error);
+
// Returns true on success.
bool PerformInstallOperation(const InstallOperation& operation);
@@ -256,6 +263,8 @@
ErrorCode* error);
bool PerformSourceBsdiffOperation(const InstallOperation& operation,
ErrorCode* error);
+ bool PerformPuffDiffOperation(const InstallOperation& operation,
+ ErrorCode* error);
// Extracts the payload signature message from the blob on the |operation| if
// the offset matches the one specified by the manifest. Returns whether the
@@ -390,6 +399,9 @@
// The last progress chunk recorded.
unsigned last_progress_chunk_{0};
+ // If |true|, the update is user initiated (vs. periodic update checks).
+ bool is_interactive_{false};
+
// The timeout after which we should force emitting a progress log (constant),
// and the actual point in time for the next forced log to be emitted.
const base::TimeDelta forced_progress_log_wait_{
diff --git a/payload_consumer/delta_performer_integration_test.cc b/payload_consumer/delta_performer_integration_test.cc
index bc67d93..3572a6d 100644
--- a/payload_consumer/delta_performer_integration_test.cc
+++ b/payload_consumer/delta_performer_integration_test.cc
@@ -747,7 +747,8 @@
&state->fake_hardware_,
&state->mock_delegate_,
install_plan,
- &install_plan->payloads[0]);
+ &install_plan->payloads[0],
+ false /* is_interactive */);
string public_key_path = GetBuildArtifactsPath(kUnittestPublicKeyPath);
EXPECT_TRUE(utils::FileExists(public_key_path.c_str()));
(*performer)->set_public_key_path(public_key_path);
diff --git a/payload_consumer/delta_performer_unittest.cc b/payload_consumer/delta_performer_unittest.cc
index 4a86ac8..420efd2 100644
--- a/payload_consumer/delta_performer_unittest.cc
+++ b/payload_consumer/delta_performer_unittest.cc
@@ -87,6 +87,61 @@
0x00, 0x00, 0x59, 0x5a,
};
+const uint8_t src_deflates[] = {
+ /* raw 0 */ 0x11, 0x22,
+ /* deflate 2 */ 0x63, 0x64, 0x62, 0x66, 0x61, 0x05, 0x00,
+ /* raw 9 */ 0x33,
+ /* deflate 10 */ 0x03, 0x00,
+ /* raw 12 */
+ /* deflate 12 */ 0x63, 0x04, 0x00,
+ /* raw 15 */ 0x44, 0x55
+};
+
+const uint8_t dst_deflates[] = {
+ /* deflate 0 */ 0x63, 0x64, 0x62, 0x66, 0x61, 0x05, 0x00,
+ /* raw 7 */ 0x33, 0x66,
+ /* deflate 9 */ 0x01, 0x05, 0x00, 0xFA, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05,
+ /* deflate 19 */ 0x63, 0x04, 0x00
+};
+
+// To generate this patch either:
+// - Use puffin/src/patching_unittest.cc:TestPatching
+// Or
+// - Use the following approach:
+// * Make src_deflate a string of hex with only spaces. (e.g. "0XTE 0xST")
+// * echo "0XTE 0xST" | xxd -r -p > src.bin
+// * Find the location of deflates in src_deflates (in bytes) in the format of
+// "offset:length,...". (e.g. "2:7,10:2,12:3")
+// * Do previous three steps for dst_deflates.
+// * puffin --operation=puffdiff --src_file=src.bin --dst_file=dst.bin \
+// --src_deflates_byte="2:7,10:2,12:3" --dst_deflates_byte="0:7,9:10,19:3" \
+// --patch_file=patch.bin
+// * hexdump -ve '" " 12/1 "0x%02x, " "\n"' patch.bin
+const uint8_t puffdiff_patch[] = {
+ 0x50, 0x55, 0x46, 0x31, 0x00, 0x00, 0x00, 0x51, 0x08, 0x01, 0x12, 0x27,
+ 0x0A, 0x04, 0x08, 0x10, 0x10, 0x32, 0x0A, 0x04, 0x08, 0x50, 0x10, 0x0A,
+ 0x0A, 0x04, 0x08, 0x60, 0x10, 0x12, 0x12, 0x04, 0x08, 0x10, 0x10, 0x58,
+ 0x12, 0x04, 0x08, 0x78, 0x10, 0x28, 0x12, 0x05, 0x08, 0xA8, 0x01, 0x10,
+ 0x38, 0x18, 0x1F, 0x1A, 0x24, 0x0A, 0x02, 0x10, 0x32, 0x0A, 0x04, 0x08,
+ 0x48, 0x10, 0x50, 0x0A, 0x05, 0x08, 0x98, 0x01, 0x10, 0x12, 0x12, 0x02,
+ 0x10, 0x58, 0x12, 0x04, 0x08, 0x70, 0x10, 0x58, 0x12, 0x05, 0x08, 0xC8,
+ 0x01, 0x10, 0x38, 0x18, 0x21, 0x42, 0x53, 0x44, 0x49, 0x46, 0x46, 0x34,
+ 0x30, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x42, 0x5A, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0x65,
+ 0x29, 0x8C, 0x9B, 0x00, 0x00, 0x03, 0x60, 0x40, 0x7A, 0x0E, 0x08, 0x00,
+ 0x40, 0x00, 0x20, 0x00, 0x21, 0x22, 0x9A, 0x3D, 0x4F, 0x50, 0x40, 0x0C,
+ 0x3B, 0xC7, 0x9B, 0xB2, 0x21, 0x0E, 0xE9, 0x15, 0x98, 0x7A, 0x7C, 0x5D,
+ 0xC9, 0x14, 0xE1, 0x42, 0x41, 0x94, 0xA6, 0x32, 0x6C, 0x42, 0x5A, 0x68,
+ 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xF1, 0x20, 0x5F, 0x0D, 0x00,
+ 0x00, 0x02, 0x41, 0x15, 0x42, 0x08, 0x20, 0x00, 0x40, 0x00, 0x00, 0x02,
+ 0x40, 0x00, 0x20, 0x00, 0x22, 0x3D, 0x23, 0x10, 0x86, 0x03, 0x96, 0x54,
+ 0x11, 0x16, 0x5F, 0x17, 0x72, 0x45, 0x38, 0x50, 0x90, 0xF1, 0x20, 0x5F,
+ 0x0D, 0x42, 0x5A, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0x07,
+ 0xD4, 0xCB, 0x6E, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x20, 0x00,
+ 0x21, 0x18, 0x46, 0x82, 0xEE, 0x48, 0xA7, 0x0A, 0x12, 0x00, 0xFA, 0x99,
+ 0x6D, 0xC0};
+
} // namespace
class DeltaPerformerTest : public ::testing::Test {
@@ -334,7 +389,8 @@
&fake_hardware_,
&mock_delegate_,
&install_plan_,
- &payload_};
+ &payload_,
+ false /* is_interactive*/};
};
TEST_F(DeltaPerformerTest, FullPayloadWriteTest) {
@@ -375,7 +431,7 @@
testing::Mock::VerifyAndClearExpectations(&mock_delegate_);
EXPECT_CALL(mock_delegate_, ShouldCancel(_))
.WillOnce(
- testing::DoAll(testing::SetArgumentPointee<0>(ErrorCode::kError),
+ testing::DoAll(testing::SetArgPointee<0>(ErrorCode::kError),
testing::Return(true)));
ApplyPayload(payload_data, "/dev/null", false);
@@ -485,6 +541,32 @@
EXPECT_EQ(expected_data, ApplyPayload(payload_data, source_path, true));
}
+TEST_F(DeltaPerformerTest, PuffdiffOperationTest) {
+ AnnotatedOperation aop;
+ *(aop.op.add_src_extents()) = ExtentForRange(0, 1);
+ *(aop.op.add_dst_extents()) = ExtentForRange(0, 1);
+ brillo::Blob puffdiff_payload(std::begin(puffdiff_patch),
+ std::end(puffdiff_patch));
+ aop.op.set_data_offset(0);
+ aop.op.set_data_length(puffdiff_payload.size());
+ aop.op.set_type(InstallOperation::PUFFDIFF);
+ brillo::Blob src(std::begin(src_deflates), std::end(src_deflates));
+ src.resize(4096); // block size
+ brillo::Blob src_hash;
+ EXPECT_TRUE(HashCalculator::RawHashOfData(src, &src_hash));
+ aop.op.set_src_sha256_hash(src_hash.data(), src_hash.size());
+
+ brillo::Blob payload_data = GeneratePayload(puffdiff_payload, {aop}, false);
+
+ string source_path;
+ EXPECT_TRUE(utils::MakeTempFile("Source-XXXXXX", &source_path, nullptr));
+ ScopedPathUnlinker path_unlinker(source_path);
+ EXPECT_TRUE(utils::WriteFile(source_path.c_str(), src.data(), src.size()));
+
+ brillo::Blob dst(std::begin(dst_deflates), std::end(dst_deflates));
+ EXPECT_EQ(dst, ApplyPayload(payload_data, source_path, true));
+}
+
TEST_F(DeltaPerformerTest, SourceHashMismatchTest) {
brillo::Blob expected_data = {'f', 'o', 'o'};
brillo::Blob actual_data = {'b', 'a', 'r'};
diff --git a/payload_consumer/download_action.cc b/payload_consumer/download_action.cc
index cd25962..f1b6e33 100644
--- a/payload_consumer/download_action.cc
+++ b/payload_consumer/download_action.cc
@@ -20,9 +20,10 @@
#include <algorithm>
#include <string>
-#include <vector>
#include <base/files/file_path.h>
+#include <base/metrics/statistics_recorder.h>
+#include <base/strings/stringprintf.h>
#include "update_engine/common/action_pipe.h"
#include "update_engine/common/boot_control_interface.h"
@@ -35,7 +36,6 @@
using base::FilePath;
using std::string;
-using std::vector;
namespace chromeos_update_engine {
@@ -43,17 +43,21 @@
BootControlInterface* boot_control,
HardwareInterface* hardware,
SystemState* system_state,
- HttpFetcher* http_fetcher)
+ HttpFetcher* http_fetcher,
+ bool is_interactive)
: prefs_(prefs),
boot_control_(boot_control),
hardware_(hardware),
system_state_(system_state),
http_fetcher_(new MultiRangeHttpFetcher(http_fetcher)),
+ is_interactive_(is_interactive),
writer_(nullptr),
code_(ErrorCode::kSuccess),
delegate_(nullptr),
p2p_sharing_fd_(-1),
- p2p_visible_(true) {}
+ p2p_visible_(true) {
+ base::StatisticsRecorder::Initialize();
+}
DownloadAction::~DownloadAction() {}
@@ -241,8 +245,13 @@
if (writer_ && writer_ != delta_performer_.get()) {
LOG(INFO) << "Using writer for test.";
} else {
- delta_performer_.reset(new DeltaPerformer(
- prefs_, boot_control_, hardware_, delegate_, &install_plan_, payload_));
+ delta_performer_.reset(new DeltaPerformer(prefs_,
+ boot_control_,
+ hardware_,
+ delegate_,
+ &install_plan_,
+ payload_,
+ is_interactive_));
writer_ = delta_performer_.get();
}
if (system_state_ != nullptr) {
@@ -363,25 +372,33 @@
if (code == ErrorCode::kSuccess) {
if (delta_performer_ && !payload_->already_applied)
code = delta_performer_->VerifyPayload(payload_->hash, payload_->size);
- if (code != ErrorCode::kSuccess) {
+ if (code == ErrorCode::kSuccess) {
+ if (payload_ < &install_plan_.payloads.back() &&
+ system_state_->payload_state()->NextPayload()) {
+ LOG(INFO) << "Incrementing to next payload";
+ // No need to reset if this payload was already applied.
+ if (delta_performer_ && !payload_->already_applied)
+ DeltaPerformer::ResetUpdateProgress(prefs_, false);
+ // Start downloading next payload.
+ bytes_received_previous_payloads_ += payload_->size;
+ payload_++;
+ install_plan_.download_url =
+ system_state_->payload_state()->GetCurrentUrl();
+ StartDownloading();
+ return;
+ }
+ // Log UpdateEngine.DownloadAction.* histograms to help diagnose
+ // long-blocking oeprations.
+ std::string histogram_output;
+ base::StatisticsRecorder::WriteGraph(
+ "UpdateEngine.DownloadAction.", &histogram_output);
+ LOG(INFO) << histogram_output;
+ } else {
LOG(ERROR) << "Download of " << install_plan_.download_url
<< " failed due to payload verification error.";
// Delete p2p file, if applicable.
if (!p2p_file_id_.empty())
CloseP2PSharingFd(true);
- } else if (payload_ < &install_plan_.payloads.back() &&
- system_state_->payload_state()->NextPayload()) {
- LOG(INFO) << "Incrementing to next payload";
- // No need to reset if this payload was already applied.
- if (delta_performer_ && !payload_->already_applied)
- DeltaPerformer::ResetUpdateProgress(prefs_, false);
- // Start downloading next payload.
- bytes_received_previous_payloads_ += payload_->size;
- payload_++;
- install_plan_.download_url =
- system_state_->payload_state()->GetCurrentUrl();
- StartDownloading();
- return;
}
}
diff --git a/payload_consumer/download_action.h b/payload_consumer/download_action.h
index 786db20..81d7333 100644
--- a/payload_consumer/download_action.h
+++ b/payload_consumer/download_action.h
@@ -73,12 +73,13 @@
// Takes ownership of the passed in HttpFetcher. Useful for testing.
// A good calling pattern is:
// DownloadAction(prefs, boot_contol, hardware, system_state,
- // new WhateverHttpFetcher);
+ // new WhateverHttpFetcher, false);
DownloadAction(PrefsInterface* prefs,
BootControlInterface* boot_control,
HardwareInterface* hardware,
SystemState* system_state,
- HttpFetcher* http_fetcher);
+ HttpFetcher* http_fetcher,
+ bool is_interactive);
~DownloadAction() override;
// InstallPlanAction overrides.
@@ -154,6 +155,11 @@
// Pointer to the MultiRangeHttpFetcher that does the http work.
std::unique_ptr<MultiRangeHttpFetcher> http_fetcher_;
+ // If |true|, the update is user initiated (vs. periodic update checks). Hence
+ // the |delta_performer_| can decide not to use O_DSYNC flag for faster
+ // update.
+ bool is_interactive_;
+
// The FileWriter that downloaded data should be written to. It will
// either point to *decompressing_file_writer_ or *delta_performer_.
FileWriter* writer_;
diff --git a/payload_consumer/download_action_unittest.cc b/payload_consumer/download_action_unittest.cc
index b04db49..21ce461 100644
--- a/payload_consumer/download_action_unittest.cc
+++ b/payload_consumer/download_action_unittest.cc
@@ -22,7 +22,6 @@
#include <memory>
#include <string>
#include <utility>
-#include <vector>
#include <base/bind.h>
#include <base/files/file_path.h>
@@ -51,7 +50,6 @@
using base::WriteFile;
using std::string;
using std::unique_ptr;
-using std::vector;
using test_utils::ScopedTempFile;
using testing::AtLeast;
using testing::InSequence;
@@ -166,7 +164,8 @@
fake_system_state.boot_control(),
fake_system_state.hardware(),
&fake_system_state,
- http_fetcher);
+ http_fetcher,
+ false /* is_interactive */);
download_action.SetTestFileWriter(&writer);
BondActions(&feeder_action, &download_action);
MockDownloadActionDelegate download_delegate;
@@ -279,7 +278,8 @@
fake_system_state.boot_control(),
fake_system_state.hardware(),
&fake_system_state,
- http_fetcher);
+ http_fetcher,
+ false /* is_interactive */);
download_action.SetTestFileWriter(&mock_file_writer);
BondActions(&feeder_action, &download_action);
MockDownloadActionDelegate download_delegate;
@@ -368,7 +368,8 @@
fake_system_state_.boot_control(),
fake_system_state_.hardware(),
&fake_system_state_,
- new MockHttpFetcher(data.data(), data.size(), nullptr));
+ new MockHttpFetcher(data.data(), data.size(), nullptr),
+ false /* is_interactive */);
download_action.SetTestFileWriter(&writer);
MockDownloadActionDelegate download_delegate;
if (use_download_delegate) {
@@ -469,7 +470,8 @@
fake_system_state_.boot_control(),
fake_system_state_.hardware(),
&fake_system_state_,
- new MockHttpFetcher("x", 1, nullptr));
+ new MockHttpFetcher("x", 1, nullptr),
+ false /* is_interactive */);
download_action.SetTestFileWriter(&writer);
DownloadActionTestAction test_action;
@@ -558,7 +560,8 @@
fake_system_state_.boot_control(),
fake_system_state_.hardware(),
&fake_system_state_,
- http_fetcher_));
+ http_fetcher_,
+ false /* is_interactive */));
download_action_->SetTestFileWriter(&writer);
BondActions(&feeder_action, download_action_.get());
DownloadActionTestProcessorDelegate delegate(ErrorCode::kSuccess);
diff --git a/payload_consumer/extent_reader.cc b/payload_consumer/extent_reader.cc
new file mode 100644
index 0000000..96ea918
--- /dev/null
+++ b/payload_consumer/extent_reader.cc
@@ -0,0 +1,98 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/extent_reader.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/payload_constants.h"
+
+using google::protobuf::RepeatedPtrField;
+
+namespace chromeos_update_engine {
+
+bool DirectExtentReader::Init(FileDescriptorPtr fd,
+ const RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) {
+ fd_ = fd;
+ extents_ = extents;
+ block_size_ = block_size;
+ cur_extent_ = extents_.begin();
+
+ extents_upper_bounds_.reserve(extents_.size() + 1);
+ // We add this pad as the first element to not bother with boundary checks
+ // later.
+ extents_upper_bounds_.emplace_back(0);
+ for (const auto& extent : extents_) {
+ total_size_ += extent.num_blocks() * block_size_;
+ extents_upper_bounds_.emplace_back(total_size_);
+ }
+ return true;
+}
+
+bool DirectExtentReader::Seek(uint64_t offset) {
+ TEST_AND_RETURN_FALSE(offset <= total_size_);
+ if (offset_ == offset) {
+ return true;
+ }
+ // The first item is zero and upper_bound never returns it because it always
+ // return the item which is greater than the given value.
+ auto extent_idx = std::upper_bound(
+ extents_upper_bounds_.begin(), extents_upper_bounds_.end(), offset) -
+ extents_upper_bounds_.begin() - 1;
+ cur_extent_ = std::next(extents_.begin(), extent_idx);
+ offset_ = offset;
+ cur_extent_bytes_read_ = offset_ - extents_upper_bounds_[extent_idx];
+ return true;
+}
+
+bool DirectExtentReader::Read(void* buffer, size_t count) {
+ auto bytes = reinterpret_cast<uint8_t*>(buffer);
+ uint64_t bytes_read = 0;
+ while (bytes_read < count) {
+ if (cur_extent_ == extents_.end()) {
+ TEST_AND_RETURN_FALSE(bytes_read == count);
+ }
+ uint64_t cur_extent_bytes_left =
+ cur_extent_->num_blocks() * block_size_ - cur_extent_bytes_read_;
+ uint64_t bytes_to_read =
+ std::min(count - bytes_read, cur_extent_bytes_left);
+
+ ssize_t out_bytes_read;
+ TEST_AND_RETURN_FALSE(utils::PReadAll(
+ fd_,
+ bytes + bytes_read,
+ bytes_to_read,
+ cur_extent_->start_block() * block_size_ + cur_extent_bytes_read_,
+ &out_bytes_read));
+ TEST_AND_RETURN_FALSE(out_bytes_read ==
+ static_cast<ssize_t>(bytes_to_read));
+
+ bytes_read += bytes_to_read;
+ cur_extent_bytes_read_ += bytes_to_read;
+ offset_ += bytes_to_read;
+ if (cur_extent_bytes_read_ == cur_extent_->num_blocks() * block_size_) {
+ // We have to advance the cur_extent_;
+ cur_extent_++;
+ cur_extent_bytes_read_ = 0;
+ }
+ }
+ return true;
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/extent_reader.h b/payload_consumer/extent_reader.h
new file mode 100644
index 0000000..3f9e4c8
--- /dev/null
+++ b/payload_consumer/extent_reader.h
@@ -0,0 +1,82 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
+#define UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
+
+#include <vector>
+
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/update_metadata.pb.h"
+
+namespace chromeos_update_engine {
+
+// ExtentReader is an abstract class with reads from a given file descriptor at
+// the extents given.
+class ExtentReader {
+ public:
+ virtual ~ExtentReader() = default;
+
+ // Initializes |ExtentReader|
+ virtual bool Init(FileDescriptorPtr fd,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) = 0;
+
+ // Seeks to the given |offset| assuming all extents are concatenated together.
+ virtual bool Seek(uint64_t offset) = 0;
+
+ // Returns true on success.
+ virtual bool Read(void* buffer, size_t count) = 0;
+};
+
+// DirectExtentReader is probably the simplest ExtentReader implementation.
+// It reads the data directly from the extents.
+class DirectExtentReader : public ExtentReader {
+ public:
+ DirectExtentReader() = default;
+ ~DirectExtentReader() override = default;
+
+ bool Init(FileDescriptorPtr fd,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) override;
+ bool Seek(uint64_t offset) override;
+ bool Read(void* bytes, size_t count) override;
+
+ private:
+ FileDescriptorPtr fd_{nullptr};
+ google::protobuf::RepeatedPtrField<Extent> extents_;
+ size_t block_size_{0};
+
+ // Current extent being read from |fd_|.
+ google::protobuf::RepeatedPtrField<Extent>::iterator cur_extent_;
+
+ // Bytes read from |cur_extent_| thus far.
+ uint64_t cur_extent_bytes_read_{0};
+
+ // Offset assuming all extents are concatenated.
+ uint64_t offset_{0};
+
+ // The accelaring upper bounds for |extents_| if we assume all extents are
+ // concatenated.
+ std::vector<uint64_t> extents_upper_bounds_;
+ uint64_t total_size_{0};
+
+ DISALLOW_COPY_AND_ASSIGN(DirectExtentReader);
+};
+
+} // namespace chromeos_update_engine
+
+#endif // UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
diff --git a/payload_consumer/extent_reader_unittest.cc b/payload_consumer/extent_reader_unittest.cc
new file mode 100644
index 0000000..b7059bc
--- /dev/null
+++ b/payload_consumer/extent_reader_unittest.cc
@@ -0,0 +1,170 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/extent_reader.h"
+
+#include <fcntl.h>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include <brillo/secure_blob.h>
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/payload_constants.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+using chromeos_update_engine::test_utils::ExpectVectorsEq;
+using std::min;
+using std::string;
+using std::vector;
+
+namespace chromeos_update_engine {
+
+namespace {
+const size_t kBlockSize = 8;
+const size_t kRandomIterations = 1000;
+} // namespace
+
+class ExtentReaderTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ sample_.resize(4096 * 10);
+ srand(time(nullptr));
+ unsigned int rand_seed;
+ for (size_t i = 0; i < sample_.size(); i++) {
+ sample_[i] = rand_r(&rand_seed) % 256;
+ }
+ ASSERT_TRUE(utils::WriteFile(
+ temp_file_.path().c_str(), sample_.data(), sample_.size()));
+
+ fd_.reset(new EintrSafeFileDescriptor());
+ ASSERT_TRUE(fd_->Open(temp_file_.path().c_str(), O_RDONLY, 0600));
+ }
+ void TearDown() override { fd_->Close(); }
+
+ void ReadExtents(vector<Extent> extents, brillo::Blob* blob) {
+ blob->clear();
+ for (const auto& extent : extents) {
+ blob->insert(
+ blob->end(),
+ &sample_[extent.start_block() * kBlockSize],
+ &sample_[(extent.start_block() + extent.num_blocks()) * kBlockSize]);
+ }
+ }
+
+ FileDescriptorPtr fd_;
+ test_utils::ScopedTempFile temp_file_{"ExtentReaderTest-file.XXXXXX"};
+ brillo::Blob sample_;
+};
+
+TEST_F(ExtentReaderTest, SimpleTest) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob1(utils::BlocksInExtents(extents) * kBlockSize);
+ EXPECT_TRUE(reader.Read(blob1.data(), blob1.size()));
+ brillo::Blob blob2;
+ ReadExtents(extents, &blob2);
+ ExpectVectorsEq(blob1, blob2);
+}
+
+TEST_F(ExtentReaderTest, ZeroExtentLengthTest) {
+ vector<Extent> extents = {ExtentForRange(1, 0)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(1);
+ EXPECT_TRUE(reader.Read(blob.data(), 0));
+ EXPECT_FALSE(reader.Read(blob.data(), 1));
+}
+
+TEST_F(ExtentReaderTest, NoExtentTest) {
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(1);
+ EXPECT_TRUE(reader.Read(blob.data(), 0));
+ EXPECT_FALSE(reader.Read(blob.data(), 1));
+}
+
+TEST_F(ExtentReaderTest, OverflowExtentTest) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(utils::BlocksInExtents(extents) * kBlockSize + 1);
+ EXPECT_FALSE(reader.Read(blob.data(), blob.size()));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow1Test) {
+ vector<Extent> extents = {ExtentForRange(1, 0)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ EXPECT_FALSE(reader.Seek(1));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow2Test) {
+ DirectExtentReader reader;
+ reader.Init(fd_, {}, kBlockSize);
+ EXPECT_TRUE(reader.Seek(0));
+ EXPECT_FALSE(reader.Seek(1));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow3Test) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ // Seek to the end of the extents should be fine as long as nothing is read.
+ EXPECT_TRUE(reader.Seek(kBlockSize));
+ EXPECT_FALSE(reader.Seek(kBlockSize + 1));
+}
+
+TEST_F(ExtentReaderTest, RandomReadTest) {
+ vector<Extent> extents = {ExtentForRange(0, 0),
+ ExtentForRange(1, 1),
+ ExtentForRange(3, 0),
+ ExtentForRange(4, 2),
+ ExtentForRange(7, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+
+ brillo::Blob result;
+ ReadExtents(extents, &result);
+
+ brillo::Blob blob(utils::BlocksInExtents(extents) * kBlockSize);
+ srand(time(nullptr));
+ uint32_t rand_seed;
+ for (size_t idx = 0; idx < kRandomIterations; idx++) {
+ // zero to full size available.
+ size_t start = rand_r(&rand_seed) % blob.size();
+ size_t size = rand_r(&rand_seed) % (blob.size() - start);
+ EXPECT_TRUE(reader.Seek(start));
+ EXPECT_TRUE(reader.Read(blob.data(), size));
+ for (size_t i = 0; i < size; i++) {
+ ASSERT_EQ(blob[i], result[start + i]);
+ }
+ }
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/extent_writer.cc b/payload_consumer/extent_writer.cc
index 5501e22..c5776ec 100644
--- a/payload_consumer/extent_writer.cc
+++ b/payload_consumer/extent_writer.cc
@@ -34,21 +34,19 @@
return true;
const char* c_bytes = reinterpret_cast<const char*>(bytes);
size_t bytes_written = 0;
- while (count - bytes_written > 0) {
- TEST_AND_RETURN_FALSE(next_extent_index_ < extents_.size());
- uint64_t bytes_remaining_next_extent =
- extents_[next_extent_index_].num_blocks() * block_size_ -
- extent_bytes_written_;
- CHECK_NE(bytes_remaining_next_extent, static_cast<uint64_t>(0));
+ while (bytes_written < count) {
+ TEST_AND_RETURN_FALSE(cur_extent_ != extents_.end());
+ uint64_t bytes_remaining_cur_extent =
+ cur_extent_->num_blocks() * block_size_ - extent_bytes_written_;
+ CHECK_NE(bytes_remaining_cur_extent, static_cast<uint64_t>(0));
size_t bytes_to_write =
static_cast<size_t>(min(static_cast<uint64_t>(count - bytes_written),
- bytes_remaining_next_extent));
+ bytes_remaining_cur_extent));
TEST_AND_RETURN_FALSE(bytes_to_write > 0);
- if (extents_[next_extent_index_].start_block() != kSparseHole) {
+ if (cur_extent_->start_block() != kSparseHole) {
const off64_t offset =
- extents_[next_extent_index_].start_block() * block_size_ +
- extent_bytes_written_;
+ cur_extent_->start_block() * block_size_ + extent_bytes_written_;
TEST_AND_RETURN_FALSE_ERRNO(fd_->Seek(offset, SEEK_SET) !=
static_cast<off64_t>(-1));
TEST_AND_RETURN_FALSE(
@@ -56,13 +54,12 @@
}
bytes_written += bytes_to_write;
extent_bytes_written_ += bytes_to_write;
- if (bytes_remaining_next_extent == bytes_to_write) {
+ if (bytes_remaining_cur_extent == bytes_to_write) {
// We filled this extent
- CHECK_EQ(extent_bytes_written_,
- extents_[next_extent_index_].num_blocks() * block_size_);
+ CHECK_EQ(extent_bytes_written_, cur_extent_->num_blocks() * block_size_);
// move to next extent
extent_bytes_written_ = 0;
- next_extent_index_++;
+ cur_extent_++;
}
}
return true;
diff --git a/payload_consumer/extent_writer.h b/payload_consumer/extent_writer.h
index 6484ebf..2c15861 100644
--- a/payload_consumer/extent_writer.h
+++ b/payload_consumer/extent_writer.h
@@ -17,7 +17,8 @@
#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_WRITER_H_
#define UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_WRITER_H_
-#include <vector>
+#include <memory>
+#include <utility>
#include <base/logging.h>
#include <brillo/secure_blob.h>
@@ -40,7 +41,7 @@
// Returns true on success.
virtual bool Init(FileDescriptorPtr fd,
- const std::vector<Extent>& extents,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
uint32_t block_size) = 0;
// Returns true on success.
@@ -66,11 +67,12 @@
~DirectExtentWriter() override = default;
bool Init(FileDescriptorPtr fd,
- const std::vector<Extent>& extents,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
uint32_t block_size) override {
fd_ = fd;
block_size_ = block_size;
extents_ = extents;
+ cur_extent_ = extents_.begin();
return true;
}
bool Write(const void* bytes, size_t count) override;
@@ -80,11 +82,11 @@
FileDescriptorPtr fd_{nullptr};
size_t block_size_{0};
- // Bytes written into next_extent_index_ thus far
+ // Bytes written into |cur_extent_| thus far.
uint64_t extent_bytes_written_{0};
- std::vector<Extent> extents_;
- // The next call to write should correspond to extents_[next_extent_index_]
- std::vector<Extent>::size_type next_extent_index_{0};
+ google::protobuf::RepeatedPtrField<Extent> extents_;
+ // The next call to write should correspond to |cur_extents_|.
+ google::protobuf::RepeatedPtrField<Extent>::iterator cur_extent_;
};
// Takes an underlying ExtentWriter to which all operations are delegated.
@@ -100,7 +102,7 @@
~ZeroPadExtentWriter() override = default;
bool Init(FileDescriptorPtr fd,
- const std::vector<Extent>& extents,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
uint32_t block_size) override {
block_size_ = block_size;
return underlying_extent_writer_->Init(fd, extents, block_size);
diff --git a/payload_consumer/extent_writer_unittest.cc b/payload_consumer/extent_writer_unittest.cc
index 24d238e..48b27cb 100644
--- a/payload_consumer/extent_writer_unittest.cc
+++ b/payload_consumer/extent_writer_unittest.cc
@@ -19,16 +19,17 @@
#include <fcntl.h>
#include <algorithm>
+#include <memory>
#include <string>
#include <vector>
-#include <brillo/make_unique_ptr.h>
#include <brillo/secure_blob.h>
#include <gtest/gtest.h>
#include "update_engine/common/test_utils.h"
#include "update_engine/common/utils.h"
#include "update_engine/payload_consumer/payload_constants.h"
+#include "update_engine/payload_generator/extent_ranges.h"
using chromeos_update_engine::test_utils::ExpectVectorsEq;
using std::min;
@@ -65,16 +66,11 @@
};
TEST_F(ExtentWriterTest, SimpleTest) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(1);
- extent.set_num_blocks(1);
- extents.push_back(extent);
-
+ vector<Extent> extents = {ExtentForRange(1, 1)};
const string bytes = "1234";
-
DirectExtentWriter direct_writer;
- EXPECT_TRUE(direct_writer.Init(fd_, extents, kBlockSize));
+ EXPECT_TRUE(
+ direct_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
EXPECT_TRUE(direct_writer.Write(bytes.data(), bytes.size()));
EXPECT_TRUE(direct_writer.End());
@@ -91,14 +87,10 @@
}
TEST_F(ExtentWriterTest, ZeroLengthTest) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(1);
- extent.set_num_blocks(1);
- extents.push_back(extent);
-
+ vector<Extent> extents = {ExtentForRange(1, 1)};
DirectExtentWriter direct_writer;
- EXPECT_TRUE(direct_writer.Init(fd_, extents, kBlockSize));
+ EXPECT_TRUE(
+ direct_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
EXPECT_TRUE(direct_writer.Write(nullptr, 0));
EXPECT_TRUE(direct_writer.End());
}
@@ -117,23 +109,14 @@
void ExtentWriterTest::WriteAlignedExtents(size_t chunk_size,
size_t first_chunk_size) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(1);
- extent.set_num_blocks(1);
- extents.push_back(extent);
- extent.set_start_block(0);
- extent.set_num_blocks(1);
- extents.push_back(extent);
- extent.set_start_block(2);
- extent.set_num_blocks(1);
- extents.push_back(extent);
-
+ vector<Extent> extents = {
+ ExtentForRange(1, 1), ExtentForRange(0, 1), ExtentForRange(2, 1)};
brillo::Blob data(kBlockSize * 3);
test_utils::FillWithData(&data);
DirectExtentWriter direct_writer;
- EXPECT_TRUE(direct_writer.Init(fd_, extents, kBlockSize));
+ EXPECT_TRUE(
+ direct_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
size_t bytes_written = 0;
while (bytes_written < data.size()) {
@@ -172,22 +155,14 @@
}
void ExtentWriterTest::TestZeroPad(bool aligned_size) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(1);
- extent.set_num_blocks(1);
- extents.push_back(extent);
- extent.set_start_block(0);
- extent.set_num_blocks(1);
- extents.push_back(extent);
-
+ vector<Extent> extents = {ExtentForRange(1, 1), ExtentForRange(0, 1)};
brillo::Blob data(kBlockSize * 2);
test_utils::FillWithData(&data);
- ZeroPadExtentWriter zero_pad_writer(
- brillo::make_unique_ptr(new DirectExtentWriter()));
+ ZeroPadExtentWriter zero_pad_writer(std::make_unique<DirectExtentWriter>());
- EXPECT_TRUE(zero_pad_writer.Init(fd_, extents, kBlockSize));
+ EXPECT_TRUE(
+ zero_pad_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
size_t bytes_to_write = data.size();
const size_t missing_bytes = (aligned_size ? 0 : 9);
bytes_to_write -= missing_bytes;
@@ -216,17 +191,9 @@
}
TEST_F(ExtentWriterTest, SparseFileTest) {
- vector<Extent> extents;
- Extent extent;
- extent.set_start_block(1);
- extent.set_num_blocks(1);
- extents.push_back(extent);
- extent.set_start_block(kSparseHole);
- extent.set_num_blocks(2);
- extents.push_back(extent);
- extent.set_start_block(0);
- extent.set_num_blocks(1);
- extents.push_back(extent);
+ vector<Extent> extents = {ExtentForRange(1, 1),
+ ExtentForRange(kSparseHole, 2),
+ ExtentForRange(0, 1)};
const int block_count = 4;
const int on_disk_count = 2;
@@ -234,7 +201,8 @@
test_utils::FillWithData(&data);
DirectExtentWriter direct_writer;
- EXPECT_TRUE(direct_writer.Init(fd_, extents, kBlockSize));
+ EXPECT_TRUE(
+ direct_writer.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
size_t bytes_written = 0;
while (bytes_written < (block_count * kBlockSize)) {
diff --git a/payload_consumer/fake_extent_writer.h b/payload_consumer/fake_extent_writer.h
index 762c6d5..4418a9e 100644
--- a/payload_consumer/fake_extent_writer.h
+++ b/payload_consumer/fake_extent_writer.h
@@ -18,7 +18,6 @@
#define UPDATE_ENGINE_PAYLOAD_CONSUMER_FAKE_EXTENT_WRITER_H_
#include <memory>
-#include <vector>
#include <brillo/secure_blob.h>
@@ -35,7 +34,7 @@
// ExtentWriter overrides.
bool Init(FileDescriptorPtr /* fd */,
- const std::vector<Extent>& /* extents */,
+ const google::protobuf::RepeatedPtrField<Extent>& /* extents */,
uint32_t /* block_size */) override {
init_called_ = true;
return true;
diff --git a/payload_consumer/fake_file_descriptor.cc b/payload_consumer/fake_file_descriptor.cc
index 09bd2c9..d54856b 100644
--- a/payload_consumer/fake_file_descriptor.cc
+++ b/payload_consumer/fake_file_descriptor.cc
@@ -23,7 +23,7 @@
read_ops_.emplace_back(offset_, count);
// Check for the EOF condition first to avoid reporting it as a failure.
- if (offset_ >= static_cast<uint64_t>(size_) || !count)
+ if (offset_ >= static_cast<uint64_t>(size_) || count == 0)
return 0;
// Find the first offset greater or equal than the current position where a
// failure will occur. This will mark the end of the read chunk.
diff --git a/payload_consumer/fake_file_descriptor.h b/payload_consumer/fake_file_descriptor.h
index ad49373..f17820b 100644
--- a/payload_consumer/fake_file_descriptor.h
+++ b/payload_consumer/fake_file_descriptor.h
@@ -27,14 +27,14 @@
namespace chromeos_update_engine {
// A fake file descriptor with configurable errors. The file descriptor always
-// reads a fixed sequence of bytes, consisting on the concatenation of the
+// reads a fixed sequence of bytes, consisting of the concatenation of the
// numbers 0, 1, 2... each one encoded in 4 bytes as the big-endian 16-bit
// number encoded in hexadecimal. For example, the beginning of the stream in
// ASCII is 0000000100020003... which corresponds to the numbers 0, 1, 2 and 3.
class FakeFileDescriptor : public FileDescriptor {
public:
FakeFileDescriptor() = default;
- ~FakeFileDescriptor() = default;
+ ~FakeFileDescriptor() override = default;
// FileDescriptor override methods.
bool Open(const char* path, int flags, mode_t mode) override {
@@ -67,6 +67,10 @@
return false;
}
+ bool Flush() override {
+ return open_;
+ }
+
bool Close() override {
if (!open_)
return false;
@@ -86,7 +90,7 @@
// Marks the range starting from |offset| bytes into the file and |length|
// size as a failure range. Reads from this range will always fail.
void AddFailureRange(uint64_t offset, uint64_t length) {
- if (!length)
+ if (length == 0)
return;
failure_ranges_.emplace_back(offset, length);
}
diff --git a/payload_consumer/file_descriptor.cc b/payload_consumer/file_descriptor.cc
index ebe4428..4eabb8f 100644
--- a/payload_consumer/file_descriptor.cc
+++ b/payload_consumer/file_descriptor.cc
@@ -123,6 +123,11 @@
#endif // defined(BLKZEROOUT)
}
+bool EintrSafeFileDescriptor::Flush() {
+ CHECK_GE(fd_, 0);
+ return true;
+}
+
bool EintrSafeFileDescriptor::Close() {
CHECK_GE(fd_, 0);
if (IGNORE_EINTR(close(fd_)))
diff --git a/payload_consumer/file_descriptor.h b/payload_consumer/file_descriptor.h
index c8a5b15..5e524d9 100644
--- a/payload_consumer/file_descriptor.h
+++ b/payload_consumer/file_descriptor.h
@@ -87,6 +87,11 @@
uint64_t length,
int* result) = 0;
+ // Flushes any cached data. The descriptor must be opened prior to this
+ // call. Returns false if it fails to write data. Implementations may set
+ // errno accrodingly.
+ virtual bool Flush() = 0;
+
// Closes a file descriptor. The descriptor must be open prior to this call.
// Returns true on success, false otherwise. Specific implementations may set
// errno accordingly.
@@ -118,6 +123,7 @@
uint64_t start,
uint64_t length,
int* result) override;
+ bool Flush() override;
bool Close() override;
bool IsSettingErrno() override {
return true;
diff --git a/payload_consumer/file_descriptor_utils.cc b/payload_consumer/file_descriptor_utils.cc
index f7f61a5..73f86df 100644
--- a/payload_consumer/file_descriptor_utils.cc
+++ b/payload_consumer/file_descriptor_utils.cc
@@ -22,6 +22,7 @@
#include "update_engine/common/hash_calculator.h"
#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/extent_reader.h"
#include "update_engine/payload_consumer/extent_writer.h"
using google::protobuf::RepeatedPtrField;
@@ -34,15 +35,6 @@
// Size of the buffer used to copy blocks.
const int kMaxCopyBufferSize = 1024 * 1024;
-// Return the total number of blocks in the passed |extents| list.
-uint64_t GetBlockCount(const RepeatedPtrField<Extent>& extents) {
- uint64_t sum = 0;
- for (const Extent& ext : extents) {
- sum += ext.num_blocks();
- }
- return sum;
-}
-
} // namespace
namespace fd_utils {
@@ -53,53 +45,31 @@
const RepeatedPtrField<Extent>& tgt_extents,
uint32_t block_size,
brillo::Blob* hash_out) {
- HashCalculator source_hasher;
+ uint64_t total_blocks = utils::BlocksInExtents(src_extents);
+ TEST_AND_RETURN_FALSE(total_blocks == utils::BlocksInExtents(tgt_extents));
+
+ DirectExtentReader reader;
+ TEST_AND_RETURN_FALSE(reader.Init(source, src_extents, block_size));
+ DirectExtentWriter writer;
+ TEST_AND_RETURN_FALSE(writer.Init(target, tgt_extents, block_size));
uint64_t buffer_blocks = kMaxCopyBufferSize / block_size;
// Ensure we copy at least one block at a time.
if (buffer_blocks < 1)
buffer_blocks = 1;
-
- uint64_t total_blocks = GetBlockCount(src_extents);
- TEST_AND_RETURN_FALSE(total_blocks == GetBlockCount(tgt_extents));
-
brillo::Blob buf(buffer_blocks * block_size);
- DirectExtentWriter writer;
- std::vector<Extent> vec_tgt_extents;
- vec_tgt_extents.reserve(tgt_extents.size());
- for (const auto& ext : tgt_extents) {
- vec_tgt_extents.push_back(ext);
- }
- TEST_AND_RETURN_FALSE(writer.Init(target, vec_tgt_extents, block_size));
-
- for (const Extent& src_ext : src_extents) {
- for (uint64_t src_ext_block = 0; src_ext_block < src_ext.num_blocks();
- src_ext_block += buffer_blocks) {
- uint64_t iteration_blocks =
- min(buffer_blocks,
- static_cast<uint64_t>(src_ext.num_blocks() - src_ext_block));
- uint64_t src_start_block = src_ext.start_block() + src_ext_block;
-
- ssize_t bytes_read_this_iteration;
- TEST_AND_RETURN_FALSE(utils::PReadAll(source,
- buf.data(),
- iteration_blocks * block_size,
- src_start_block * block_size,
- &bytes_read_this_iteration));
-
+ HashCalculator source_hasher;
+ uint64_t blocks_left = total_blocks;
+ while (blocks_left > 0) {
+ uint64_t read_blocks = std::min(blocks_left, buffer_blocks);
+ TEST_AND_RETURN_FALSE(reader.Read(buf.data(), read_blocks * block_size));
+ if (hash_out) {
TEST_AND_RETURN_FALSE(
- bytes_read_this_iteration ==
- static_cast<ssize_t>(iteration_blocks * block_size));
-
- TEST_AND_RETURN_FALSE(
- writer.Write(buf.data(), iteration_blocks * block_size));
-
- if (hash_out) {
- TEST_AND_RETURN_FALSE(
- source_hasher.Update(buf.data(), iteration_blocks * block_size));
- }
+ source_hasher.Update(buf.data(), read_blocks * block_size));
}
+ TEST_AND_RETURN_FALSE(writer.Write(buf.data(), read_blocks * block_size));
+ blocks_left -= read_blocks;
}
TEST_AND_RETURN_FALSE(writer.End());
diff --git a/payload_consumer/file_descriptor_utils.h b/payload_consumer/file_descriptor_utils.h
index b73defb..d1289d6 100644
--- a/payload_consumer/file_descriptor_utils.h
+++ b/payload_consumer/file_descriptor_utils.h
@@ -17,8 +17,6 @@
#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_FILE_DESCRIPTOR_UTILS_H_
#define UPDATE_ENGINE_PAYLOAD_CONSUMER_FILE_DESCRIPTOR_UTILS_H_
-#include <vector>
-
#include <brillo/secure_blob.h>
#include "update_engine/payload_consumer/file_descriptor.h"
@@ -27,7 +25,7 @@
namespace chromeos_update_engine {
namespace fd_utils {
-// Copy a blocks from the |source| file to the |target| file and hash the
+// Copy blocks from the |source| file to the |target| file and hashes the
// contents. The blocks to copy from the |source| to the |target| files are
// specified by the |src_extents| and |tgt_extents| list of Extents, which
// must have the same length in number of blocks. Stores the hash of the
diff --git a/payload_consumer/file_descriptor_utils_unittest.cc b/payload_consumer/file_descriptor_utils_unittest.cc
index 9910239..8ba8ce6 100644
--- a/payload_consumer/file_descriptor_utils_unittest.cc
+++ b/payload_consumer/file_descriptor_utils_unittest.cc
@@ -16,13 +16,14 @@
#include "update_engine/payload_consumer/file_descriptor_utils.h"
+#include <fcntl.h>
+
#include <string>
#include <utility>
#include <vector>
-#include <gtest/gtest.h>
-
#include <brillo/data_encoding.h>
+#include <gtest/gtest.h>
#include "update_engine/common/hash_calculator.h"
#include "update_engine/common/test_utils.h"
@@ -31,13 +32,15 @@
#include "update_engine/payload_consumer/file_descriptor.h"
#include "update_engine/payload_generator/extent_ranges.h"
+using google::protobuf::RepeatedPtrField;
+
namespace chromeos_update_engine {
namespace {
-::google::protobuf::RepeatedPtrField<Extent> CreateExtentList(
+RepeatedPtrField<Extent> CreateExtentList(
const std::vector<std::pair<uint64_t, uint64_t>>& lst) {
- ::google::protobuf::RepeatedPtrField<Extent> result;
+ RepeatedPtrField<Extent> result;
for (const auto& item : lst) {
*result.Add() = ExtentForRange(item.first, item.second);
}
diff --git a/payload_consumer/file_writer_unittest.cc b/payload_consumer/file_writer_unittest.cc
index debb4c3..92837c8 100644
--- a/payload_consumer/file_writer_unittest.cc
+++ b/payload_consumer/file_writer_unittest.cc
@@ -21,7 +21,6 @@
#include <unistd.h>
#include <string>
-#include <vector>
#include <brillo/secure_blob.h>
#include <gtest/gtest.h>
@@ -30,7 +29,6 @@
#include "update_engine/common/utils.h"
using std::string;
-using std::vector;
namespace chromeos_update_engine {
diff --git a/payload_consumer/mtd_file_descriptor.cc b/payload_consumer/mtd_file_descriptor.cc
index 3f0a33f..5d7758a 100644
--- a/payload_consumer/mtd_file_descriptor.cc
+++ b/payload_consumer/mtd_file_descriptor.cc
@@ -18,11 +18,12 @@
#include <fcntl.h>
#include <mtd/ubi-user.h>
-#include <string>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
-#include <vector>
+
+#include <memory>
+#include <string>
#include <base/files/file_path.h>
#include <base/strings/string_number_conversions.h>
@@ -33,7 +34,6 @@
#include "update_engine/common/utils.h"
using std::string;
-using std::vector;
namespace {
diff --git a/payload_consumer/payload_constants.cc b/payload_consumer/payload_constants.cc
index 7d396b6..ad193a0 100644
--- a/payload_consumer/payload_constants.cc
+++ b/payload_consumer/payload_constants.cc
@@ -54,6 +54,8 @@
return "REPLACE_XZ";
case InstallOperation::PUFFDIFF:
return "PUFFDIFF";
+ case InstallOperation::BROTLI_BSDIFF:
+ return "BROTLI_BSDIFF";
}
return "<unknown_op>";
}
diff --git a/payload_consumer/payload_verifier.h b/payload_consumer/payload_verifier.h
index 22ced40..8caef35 100644
--- a/payload_consumer/payload_verifier.h
+++ b/payload_consumer/payload_verifier.h
@@ -18,7 +18,6 @@
#define UPDATE_ENGINE_PAYLOAD_CONSUMER_PAYLOAD_VERIFIER_H_
#include <string>
-#include <vector>
#include <base/macros.h>
#include <brillo/secure_blob.h>
diff --git a/payload_consumer/postinstall_runner_action_unittest.cc b/payload_consumer/postinstall_runner_action_unittest.cc
index 772270c..f15171b 100644
--- a/payload_consumer/postinstall_runner_action_unittest.cc
+++ b/payload_consumer/postinstall_runner_action_unittest.cc
@@ -22,7 +22,6 @@
#include <memory>
#include <string>
-#include <vector>
#include <base/bind.h>
#include <base/files/file_util.h>
@@ -43,7 +42,6 @@
using brillo::MessageLoop;
using chromeos_update_engine::test_utils::ScopedLoopbackDeviceBinder;
using std::string;
-using std::vector;
namespace chromeos_update_engine {
diff --git a/payload_consumer/xz_extent_writer.cc b/payload_consumer/xz_extent_writer.cc
index 4bd893d..343ed80 100644
--- a/payload_consumer/xz_extent_writer.cc
+++ b/payload_consumer/xz_extent_writer.cc
@@ -16,7 +16,7 @@
#include "update_engine/payload_consumer/xz_extent_writer.h"
-using std::vector;
+using google::protobuf::RepeatedPtrField;
namespace chromeos_update_engine {
@@ -47,7 +47,7 @@
return "<unknown xz error>";
}
#undef __XZ_ERROR_STRING_CASE
-};
+}
} // namespace
XzExtentWriter::~XzExtentWriter() {
@@ -55,7 +55,7 @@
}
bool XzExtentWriter::Init(FileDescriptorPtr fd,
- const vector<Extent>& extents,
+ const RepeatedPtrField<Extent>& extents,
uint32_t block_size) {
stream_ = xz_dec_init(XZ_DYNALLOC, kXzMaxDictSize);
TEST_AND_RETURN_FALSE(stream_ != nullptr);
diff --git a/payload_consumer/xz_extent_writer.h b/payload_consumer/xz_extent_writer.h
index a6b3257..5e50256 100644
--- a/payload_consumer/xz_extent_writer.h
+++ b/payload_consumer/xz_extent_writer.h
@@ -20,7 +20,7 @@
#include <xz.h>
#include <memory>
-#include <vector>
+#include <utility>
#include <brillo/secure_blob.h>
@@ -40,7 +40,7 @@
~XzExtentWriter() override;
bool Init(FileDescriptorPtr fd,
- const std::vector<Extent>& extents,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
uint32_t block_size) override;
bool Write(const void* bytes, size_t count) override;
bool EndImpl() override;
diff --git a/payload_consumer/xz_extent_writer_unittest.cc b/payload_consumer/xz_extent_writer_unittest.cc
index fb8bb40..c8bcdf9 100644
--- a/payload_consumer/xz_extent_writer_unittest.cc
+++ b/payload_consumer/xz_extent_writer_unittest.cc
@@ -22,19 +22,14 @@
#include <unistd.h>
#include <algorithm>
-#include <string>
-#include <vector>
-#include <brillo/make_unique_ptr.h>
+#include <base/memory/ptr_util.h>
#include <gtest/gtest.h>
#include "update_engine/common/test_utils.h"
#include "update_engine/common/utils.h"
#include "update_engine/payload_consumer/fake_extent_writer.h"
-using std::string;
-using std::vector;
-
namespace chromeos_update_engine {
namespace {
@@ -88,8 +83,7 @@
protected:
void SetUp() override {
fake_extent_writer_ = new FakeExtentWriter();
- xz_writer_.reset(
- new XzExtentWriter(brillo::make_unique_ptr(fake_extent_writer_)));
+ xz_writer_.reset(new XzExtentWriter(base::WrapUnique(fake_extent_writer_)));
}
void WriteAll(const brillo::Blob& compressed) {
diff --git a/payload_generator/ab_generator.cc b/payload_generator/ab_generator.cc
index 3b0d012..089dfd9 100644
--- a/payload_generator/ab_generator.cc
+++ b/payload_generator/ab_generator.cc
@@ -17,6 +17,7 @@
#include "update_engine/payload_generator/ab_generator.h"
#include <algorithm>
+#include <utility>
#include <base/strings/stringprintf.h>
@@ -173,7 +174,6 @@
InstallOperation new_op;
*(new_op.add_dst_extents()) = dst_ext;
uint32_t data_size = dst_ext.num_blocks() * kBlockSize;
- new_op.set_dst_length(data_size);
// If this is a REPLACE, attempt to reuse portions of the existing blob.
if (is_replace) {
new_op.set_type(InstallOperation::REPLACE);
@@ -238,15 +238,9 @@
if (is_delta_op) {
ExtendExtents(last_aop.op.mutable_src_extents(),
curr_aop.op.src_extents());
- if (curr_aop.op.src_length() > 0)
- last_aop.op.set_src_length(last_aop.op.src_length() +
- curr_aop.op.src_length());
}
ExtendExtents(last_aop.op.mutable_dst_extents(),
curr_aop.op.dst_extents());
- if (curr_aop.op.dst_length() > 0)
- last_aop.op.set_dst_length(last_aop.op.dst_length() +
- curr_aop.op.dst_length());
// Set the data length to zero so we know to add the blob later.
if (is_a_replace)
last_aop.op.set_data_length(0);
@@ -276,9 +270,9 @@
BlobFileWriter* blob_file) {
TEST_AND_RETURN_FALSE(IsAReplaceOperation(aop->op.type()));
- brillo::Blob data(aop->op.dst_length());
vector<Extent> dst_extents;
ExtentsToVector(aop->op.dst_extents(), &dst_extents);
+ brillo::Blob data(utils::BlocksInExtents(dst_extents) * kBlockSize);
TEST_AND_RETURN_FALSE(utils::ReadExtents(target_part_path,
dst_extents,
&data,
@@ -312,7 +306,7 @@
uint64_t src_length =
aop.op.has_src_length()
? aop.op.src_length()
- : BlocksInExtents(aop.op.src_extents()) * kBlockSize;
+ : utils::BlocksInExtents(aop.op.src_extents()) * kBlockSize;
TEST_AND_RETURN_FALSE(utils::ReadExtents(
source_part_path, src_extents, &src_data, src_length, kBlockSize));
TEST_AND_RETURN_FALSE(HashCalculator::RawHashOfData(src_data, &src_hash));
diff --git a/payload_generator/ab_generator.h b/payload_generator/ab_generator.h
index 77afb87..343b546 100644
--- a/payload_generator/ab_generator.h
+++ b/payload_generator/ab_generator.h
@@ -55,13 +55,13 @@
BlobFileWriter* blob_file,
std::vector<AnnotatedOperation>* aops) override;
- // Split the operations in the vector of AnnotatedOperations |aops|
- // such that for every operation there is only one dst extent and updates
- // |aops| with the new list of operations. All kinds of operations are
- // fragmented except BSDIFF and SOURCE_BSDIFF operations.
- // The |target_part_path| is the filename of the new image, where the
- // destination extents refer to. The blobs of the operations in |aops| should
- // reference |blob_file|. |blob_file| are updated if needed.
+ // Split the operations in the vector of AnnotatedOperations |aops| such that
+ // for every operation there is only one dst extent and updates |aops| with
+ // the new list of operations. All kinds of operations are fragmented except
+ // BSDIFF and SOURCE_BSDIFF, PUFFDIFF and BROTLI_BSDIFF operations. The
+ // |target_part_path| is the filename of the new image, where the destination
+ // extents refer to. The blobs of the operations in |aops| should reference
+ // |blob_file|. |blob_file| are updated if needed.
static bool FragmentOperations(const PayloadVersion& version,
std::vector<AnnotatedOperation>* aops,
const std::string& target_part_path,
diff --git a/payload_generator/ab_generator_unittest.cc b/payload_generator/ab_generator_unittest.cc
index ab4b164..25609c7 100644
--- a/payload_generator/ab_generator_unittest.cc
+++ b/payload_generator/ab_generator_unittest.cc
@@ -42,7 +42,9 @@
namespace {
-bool ExtentEquals(const Extent& ext, uint64_t start_block, uint64_t num_blocks) {
+bool ExtentEquals(const Extent& ext,
+ uint64_t start_block,
+ uint64_t num_blocks) {
return ext.start_block() == start_block && ext.num_blocks() == num_blocks;
}
@@ -85,7 +87,6 @@
op_ex1_num_blocks);
*(op.add_dst_extents()) = ExtentForRange(op_ex2_start_block,
op_ex2_num_blocks);
- op.set_dst_length(op_ex1_num_blocks + op_ex2_num_blocks);
brillo::Blob op_data;
op_data.insert(op_data.end(),
@@ -136,7 +137,8 @@
EXPECT_EQ("SplitTestOp:0", result_ops[0].name);
InstallOperation first_op = result_ops[0].op;
EXPECT_EQ(expected_type, first_op.type());
- EXPECT_EQ(op_ex1_size, first_op.dst_length());
+ EXPECT_FALSE(first_op.has_src_length());
+ EXPECT_FALSE(first_op.has_dst_length());
EXPECT_EQ(1, first_op.dst_extents().size());
EXPECT_TRUE(ExtentEquals(first_op.dst_extents(0), op_ex1_start_block,
op_ex1_num_blocks));
@@ -165,7 +167,8 @@
EXPECT_EQ("SplitTestOp:1", result_ops[1].name);
InstallOperation second_op = result_ops[1].op;
EXPECT_EQ(expected_type, second_op.type());
- EXPECT_EQ(op_ex2_size, second_op.dst_length());
+ EXPECT_FALSE(second_op.has_src_length());
+ EXPECT_FALSE(second_op.has_dst_length());
EXPECT_EQ(1, second_op.dst_extents().size());
EXPECT_TRUE(ExtentEquals(second_op.dst_extents(0), op_ex2_start_block,
op_ex2_num_blocks));
@@ -235,7 +238,6 @@
InstallOperation first_op;
first_op.set_type(orig_type);
const size_t first_op_size = first_op_num_blocks * kBlockSize;
- first_op.set_dst_length(first_op_size);
*(first_op.add_dst_extents()) = ExtentForRange(0, first_op_num_blocks);
brillo::Blob first_op_data(part_data.begin(),
part_data.begin() + first_op_size);
@@ -255,8 +257,6 @@
InstallOperation second_op;
second_op.set_type(orig_type);
- const size_t second_op_size = second_op_num_blocks * kBlockSize;
- second_op.set_dst_length(second_op_size);
*(second_op.add_dst_extents()) = ExtentForRange(first_op_num_blocks,
second_op_num_blocks);
brillo::Blob second_op_data(part_data.begin() + first_op_size,
@@ -302,7 +302,7 @@
InstallOperation new_op = aops[0].op;
EXPECT_EQ(expected_op_type, new_op.type());
EXPECT_FALSE(new_op.has_src_length());
- EXPECT_EQ(total_op_num_blocks * kBlockSize, new_op.dst_length());
+ EXPECT_FALSE(new_op.has_dst_length());
EXPECT_EQ(1, new_op.dst_extents().size());
EXPECT_TRUE(ExtentEquals(new_op.dst_extents(0), 0, total_op_num_blocks));
EXPECT_EQ("first,second", aops[0].name);
diff --git a/payload_generator/deflate_utils.cc b/payload_generator/deflate_utils.cc
new file mode 100644
index 0000000..88e42e0
--- /dev/null
+++ b/payload_generator/deflate_utils.cc
@@ -0,0 +1,295 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/deflate_utils.h"
+
+#include <algorithm>
+#include <string>
+#include <utility>
+
+#include <base/files/file_util.h>
+#include <base/logging.h>
+#include <base/strings/string_util.h>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+#include "update_engine/update_metadata.pb.h"
+
+using std::string;
+using std::vector;
+using puffin::BitExtent;
+using puffin::ByteExtent;
+
+namespace chromeos_update_engine {
+namespace deflate_utils {
+namespace {
+
+// The minimum size for a squashfs image to be processed.
+const uint64_t kMinimumSquashfsImageSize = 1 * 1024 * 1024; // bytes
+
+// TODO(*): Optimize this so we don't have to read all extents into memory in
+// case it is large.
+bool CopyExtentsToFile(const string& in_path,
+ const vector<Extent> extents,
+ const string& out_path,
+ size_t block_size) {
+ brillo::Blob data(utils::BlocksInExtents(extents) * block_size);
+ TEST_AND_RETURN_FALSE(
+ utils::ReadExtents(in_path, extents, &data, data.size(), block_size));
+ TEST_AND_RETURN_FALSE(
+ utils::WriteFile(out_path.c_str(), data.data(), data.size()));
+ return true;
+}
+
+bool IsSquashfsImage(const string& part_path,
+ const FilesystemInterface::File& file) {
+ // Only check for files with img postfix.
+ if (base::EndsWith(file.name, ".img", base::CompareCase::SENSITIVE) &&
+ utils::BlocksInExtents(file.extents) >=
+ kMinimumSquashfsImageSize / kBlockSize) {
+ brillo::Blob super_block;
+ TEST_AND_RETURN_FALSE(
+ utils::ReadFileChunk(part_path,
+ file.extents[0].start_block() * kBlockSize,
+ 100,
+ &super_block));
+ return SquashfsFilesystem::IsSquashfsImage(super_block);
+ }
+ return false;
+}
+
+// Realigns subfiles |files| of a splitted file |file| into its correct
+// positions. This can be used for squashfs, zip, apk, etc.
+bool RealignSplittedFiles(const FilesystemInterface::File& file,
+ vector<FilesystemInterface::File>* files) {
+ // We have to shift all the Extents in |files|, based on the Extents of the
+ // |file| itself.
+ size_t num_blocks = 0;
+ for (auto& in_file : *files) { // We need to modify so no constant.
+ TEST_AND_RETURN_FALSE(
+ ShiftExtentsOverExtents(file.extents, &in_file.extents));
+ TEST_AND_RETURN_FALSE(
+ ShiftBitExtentsOverExtents(file.extents, &in_file.deflates));
+
+ in_file.name = file.name + "/" + in_file.name;
+ num_blocks += utils::BlocksInExtents(in_file.extents);
+ }
+
+ // Check that all files in |in_files| cover the entire image.
+ TEST_AND_RETURN_FALSE(utils::BlocksInExtents(file.extents) == num_blocks);
+ return true;
+}
+
+bool IsBitExtentInExtent(const Extent& extent, const BitExtent& bit_extent) {
+ return (bit_extent.offset / 8) >= (extent.start_block() * kBlockSize) &&
+ ((bit_extent.offset + bit_extent.length + 7) / 8) <=
+ ((extent.start_block() + extent.num_blocks()) * kBlockSize);
+}
+
+} // namespace
+
+ByteExtent ExpandToByteExtent(const BitExtent& extent) {
+ uint64_t offset = extent.offset / 8;
+ uint64_t length = ((extent.offset + extent.length + 7) / 8) - offset;
+ return {offset, length};
+}
+
+bool ShiftExtentsOverExtents(const vector<Extent>& base_extents,
+ vector<Extent>* over_extents) {
+ if (utils::BlocksInExtents(base_extents) <
+ utils::BlocksInExtents(*over_extents)) {
+ LOG(ERROR) << "over_extents have more blocks than base_extents! Invalid!";
+ return false;
+ }
+ for (size_t idx = 0; idx < over_extents->size(); idx++) {
+ auto over_ext = &over_extents->at(idx);
+ auto gap_blocks = base_extents[0].start_block();
+ auto last_end_block = base_extents[0].start_block();
+ for (auto base_ext : base_extents) { // We need to modify |base_ext|, so we
+ // use copy.
+ gap_blocks += base_ext.start_block() - last_end_block;
+ last_end_block = base_ext.start_block() + base_ext.num_blocks();
+ base_ext.set_start_block(base_ext.start_block() - gap_blocks);
+ if (over_ext->start_block() >= base_ext.start_block() &&
+ over_ext->start_block() <
+ base_ext.start_block() + base_ext.num_blocks()) {
+ if (over_ext->start_block() + over_ext->num_blocks() <=
+ base_ext.start_block() + base_ext.num_blocks()) {
+ // |over_ext| is inside |base_ext|, increase its start block.
+ over_ext->set_start_block(over_ext->start_block() + gap_blocks);
+ } else {
+ // |over_ext| spills over this |base_ext|, split it into two.
+ auto new_blocks = base_ext.start_block() + base_ext.num_blocks() -
+ over_ext->start_block();
+ vector<Extent> new_extents = {
+ ExtentForRange(gap_blocks + over_ext->start_block(), new_blocks),
+ ExtentForRange(over_ext->start_block() + new_blocks,
+ over_ext->num_blocks() - new_blocks)};
+ *over_ext = new_extents[0];
+ over_extents->insert(std::next(over_extents->begin(), idx + 1),
+ new_extents[1]);
+ }
+ break; // We processed |over_ext|, so break the loop;
+ }
+ }
+ }
+ return true;
+}
+
+bool ShiftBitExtentsOverExtents(const vector<Extent>& base_extents,
+ vector<BitExtent>* over_extents) {
+ if (over_extents->empty()) {
+ return true;
+ }
+
+ // This check is needed to make sure the number of bytes in |over_extents|
+ // does not exceed |base_extents|.
+ auto last_extent = ExpandToByteExtent(over_extents->back());
+ TEST_AND_RETURN_FALSE(last_extent.offset + last_extent.length <=
+ utils::BlocksInExtents(base_extents) * kBlockSize);
+
+ for (auto o_ext = over_extents->begin(); o_ext != over_extents->end();) {
+ size_t gap_blocks = base_extents[0].start_block();
+ size_t last_end_block = base_extents[0].start_block();
+ bool o_ext_processed = false;
+ for (auto b_ext : base_extents) { // We need to modify |b_ext|, so we copy.
+ gap_blocks += b_ext.start_block() - last_end_block;
+ last_end_block = b_ext.start_block() + b_ext.num_blocks();
+ b_ext.set_start_block(b_ext.start_block() - gap_blocks);
+ auto byte_o_ext = ExpandToByteExtent(*o_ext);
+ if (byte_o_ext.offset >= b_ext.start_block() * kBlockSize &&
+ byte_o_ext.offset <
+ (b_ext.start_block() + b_ext.num_blocks()) * kBlockSize) {
+ if ((byte_o_ext.offset + byte_o_ext.length) <=
+ (b_ext.start_block() + b_ext.num_blocks()) * kBlockSize) {
+ // |o_ext| is inside |b_ext|, increase its start block.
+ o_ext->offset += gap_blocks * kBlockSize * 8;
+ ++o_ext;
+ } else {
+ // |o_ext| spills over this |b_ext|, remove it.
+ o_ext = over_extents->erase(o_ext);
+ }
+ o_ext_processed = true;
+ break; // We processed o_ext, so break the loop;
+ }
+ }
+ TEST_AND_RETURN_FALSE(o_ext_processed);
+ }
+ return true;
+}
+
+vector<BitExtent> FindDeflates(const vector<Extent>& extents,
+ const vector<BitExtent>& in_deflates) {
+ vector<BitExtent> result;
+ // TODO(ahassani): Replace this with binary_search style search.
+ for (const auto& deflate : in_deflates) {
+ for (const auto& extent : extents) {
+ if (IsBitExtentInExtent(extent, deflate)) {
+ result.push_back(deflate);
+ break;
+ }
+ }
+ }
+ return result;
+}
+
+bool CompactDeflates(const vector<Extent>& extents,
+ const vector<BitExtent>& in_deflates,
+ vector<BitExtent>* out_deflates) {
+ size_t bytes_passed = 0;
+ out_deflates->reserve(in_deflates.size());
+ for (const auto& extent : extents) {
+ size_t gap_bytes = extent.start_block() * kBlockSize - bytes_passed;
+ for (const auto& deflate : in_deflates) {
+ if (IsBitExtentInExtent(extent, deflate)) {
+ out_deflates->emplace_back(deflate.offset - (gap_bytes * 8),
+ deflate.length);
+ }
+ }
+ bytes_passed += extent.num_blocks() * kBlockSize;
+ }
+
+ // All given |in_deflates| items should've been inside one of the extents in
+ // |extents|.
+ TEST_AND_RETURN_FALSE(in_deflates.size() == out_deflates->size());
+
+ // Make sure all outgoing deflates are ordered and non-overlapping.
+ auto result = std::adjacent_find(out_deflates->begin(),
+ out_deflates->end(),
+ [](const BitExtent& a, const BitExtent& b) {
+ return (a.offset + a.length) > b.offset;
+ });
+ TEST_AND_RETURN_FALSE(result == out_deflates->end());
+ return true;
+}
+
+bool FindAndCompactDeflates(const vector<Extent>& extents,
+ const vector<BitExtent>& in_deflates,
+ vector<BitExtent>* out_deflates) {
+ auto found_deflates = FindDeflates(extents, in_deflates);
+ TEST_AND_RETURN_FALSE(CompactDeflates(extents, found_deflates, out_deflates));
+ return true;
+}
+
+bool PreprocessParitionFiles(const PartitionConfig& part,
+ vector<FilesystemInterface::File>* result_files,
+ bool extract_deflates) {
+ // Get the file system files.
+ vector<FilesystemInterface::File> tmp_files;
+ part.fs_interface->GetFiles(&tmp_files);
+ result_files->reserve(tmp_files.size());
+
+ for (const auto& file : tmp_files) {
+ if (IsSquashfsImage(part.path, file)) {
+ // Read the image into a file.
+ base::FilePath path;
+ TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&path));
+ ScopedPathUnlinker old_unlinker(path.value());
+ TEST_AND_RETURN_FALSE(
+ CopyExtentsToFile(part.path, file.extents, path.value(), kBlockSize));
+ // Test if it is actually a Squashfs file.
+ auto sqfs =
+ SquashfsFilesystem::CreateFromFile(path.value(), extract_deflates);
+ if (sqfs) {
+ // It is an squashfs file. Get its files to replace with itself.
+ vector<FilesystemInterface::File> files;
+ sqfs->GetFiles(&files);
+
+ // Replace squashfs file with its files only if |files| has at least two
+ // files or if it has some deflates (since it is better to replace it to
+ // take advantage of the deflates.)
+ if (files.size() > 1 ||
+ (files.size() == 1 && !files[0].deflates.empty())) {
+ TEST_AND_RETURN_FALSE(RealignSplittedFiles(file, &files));
+ result_files->insert(result_files->end(), files.begin(), files.end());
+ continue;
+ }
+ } else {
+ LOG(WARNING) << "We thought file: " << file.name
+ << " was a Squashfs file, but it was not.";
+ }
+ }
+ // TODO(ahassani): Process other types of files like apk, zip, etc.
+ result_files->push_back(file);
+ }
+ return true;
+}
+
+} // namespace deflate_utils
+} // namespace chromeos_update_engine
diff --git a/payload_generator/deflate_utils.h b/payload_generator/deflate_utils.h
new file mode 100644
index 0000000..798ce25
--- /dev/null
+++ b/payload_generator/deflate_utils.h
@@ -0,0 +1,97 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_DEFLATE_UTILS_H_
+#define UPDATE_ENGINE_PAYLOAD_GENERATOR_DEFLATE_UTILS_H_
+
+#include <puffin/puffdiff.h>
+#include <vector>
+
+#include "update_engine/payload_generator/filesystem_interface.h"
+#include "update_engine/payload_generator/payload_generation_config.h"
+
+namespace chromeos_update_engine {
+namespace deflate_utils {
+
+// Gets the files from the partition and processes all its files. Processing
+// includes:
+// - splitting large Squashfs containers into its smaller files.
+bool PreprocessParitionFiles(const PartitionConfig& part,
+ std::vector<FilesystemInterface::File>* result,
+ bool extract_deflates);
+
+// Spreads all extents in |over_extents| over |base_extents|. Here we assume the
+// |over_extents| are non-overlapping and sorted by their offset.
+//
+// |base_extents|:
+// | ----------------------- ------ --------------
+// |over_extents|:
+// | ========== ==== ========== ======
+// |over_extents| is transforms to:
+// | ========== ==== = ====== === ======
+//
+bool ShiftExtentsOverExtents(const std::vector<Extent>& base_extents,
+ std::vector<Extent>* over_extents);
+
+// Spreads all extents in |over_extents| over |base_extents|. Here we assume the
+// |over_extents| are non-overlapping and sorted by their offset. An item in
+// |over_extents| is removed if it is spread in two or more extents in
+// |base_extents|.
+//
+// |base_extents|:
+// | ----------------------- ------ --------------
+// |over_extents|:
+// | ========== ==== ========== ======
+// |over_extents| is transforms to:
+// | ========== ==== ======
+//
+bool ShiftBitExtentsOverExtents(const std::vector<Extent>& base_extents,
+ std::vector<puffin::BitExtent>* over_extents);
+
+// Finds all deflate locations in |deflates| that are inside an Extent in
+// |extents|. This function should not change the order of deflates.
+std::vector<puffin::BitExtent> FindDeflates(
+ const std::vector<Extent>& extents,
+ const std::vector<puffin::BitExtent>& deflates);
+
+// Creates a new list of deflate locations (|out_deflates|) from |in_deflates|
+// by assuming all extents in the |extents| have been put together
+// linearly. This function assumes that all deflate locations given in
+// |in_deflates| are located somewhere in the |extents|. |out_deflates| should
+// be empty on call.
+//
+// |extents|:
+// | ----------------------- ------ --------------
+// |in_deflates|:
+// | ======== ==== ==== ======
+// |out_deflates|:
+// | ======== ==== ==== ======
+//
+bool CompactDeflates(const std::vector<Extent>& extents,
+ const std::vector<puffin::BitExtent>& in_deflates,
+ std::vector<puffin::BitExtent>* out_deflates);
+
+// Combines |FindDeflates| and |CompcatDeflates| for ease of use.
+bool FindAndCompactDeflates(const std::vector<Extent>& extents,
+ const std::vector<puffin::BitExtent>& in_deflates,
+ std::vector<puffin::BitExtent>* out_deflates);
+
+// Expands a BitExtents to a ByteExtent.
+puffin::ByteExtent ExpandToByteExtent(const puffin::BitExtent& extent);
+
+} // namespace deflate_utils
+} // namespace chromeos_update_engine
+#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_DEFLATE_UTILS_H_
diff --git a/payload_generator/deflate_utils_unittest.cc b/payload_generator/deflate_utils_unittest.cc
new file mode 100644
index 0000000..cb9476a
--- /dev/null
+++ b/payload_generator/deflate_utils_unittest.cc
@@ -0,0 +1,190 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/deflate_utils.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+using std::vector;
+using puffin::BitExtent;
+using puffin::ByteExtent;
+
+namespace chromeos_update_engine {
+namespace deflate_utils {
+
+// This creates a sudo-random BitExtents from ByteExtents for simpler testing.
+vector<BitExtent> ByteToBitExtent(const vector<ByteExtent>& byte_extents) {
+ vector<BitExtent> bit_extents;
+ for (auto& byte_extent : byte_extents) {
+ bit_extents.emplace_back(byte_extent.offset * 8 + (byte_extent.offset & 7),
+ byte_extent.length * 8 - (byte_extent.length & 7));
+ }
+ return bit_extents;
+}
+
+TEST(DeflateUtilsTest, ExtentsShiftTest) {
+ vector<Extent> base_extents = {ExtentForRange(10, 10),
+ ExtentForRange(70, 10),
+ ExtentForRange(50, 10),
+ ExtentForRange(30, 10),
+ ExtentForRange(90, 10)};
+ vector<Extent> over_extents = {ExtentForRange(2, 2),
+ ExtentForRange(5, 2),
+ ExtentForRange(7, 3),
+ ExtentForRange(13, 10),
+ ExtentForRange(25, 20),
+ ExtentForRange(47, 3)};
+ vector<Extent> out_over_extents = {ExtentForRange(12, 2),
+ ExtentForRange(15, 2),
+ ExtentForRange(17, 3),
+ ExtentForRange(73, 7),
+ ExtentForRange(50, 3),
+ ExtentForRange(55, 5),
+ ExtentForRange(30, 10),
+ ExtentForRange(90, 5),
+ ExtentForRange(97, 3)};
+ EXPECT_TRUE(ShiftExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+
+ // Failure case
+ base_extents = {ExtentForRange(10, 10)};
+ over_extents = {ExtentForRange(2, 12)};
+ EXPECT_FALSE(ShiftExtentsOverExtents(base_extents, &over_extents));
+}
+
+TEST(DeflateUtilsTest, ShiftBitExtentsOverExtentsTest) {
+ vector<Extent> base_extents = {ExtentForRange(3, 1),
+ ExtentForRange(1, 1),
+ ExtentForRange(5, 1),
+ ExtentForRange(7, 1),
+ ExtentForRange(9, 1)};
+ vector<BitExtent> over_extents =
+ ByteToBitExtent({{0, 0}, {100, 2000}, {4096, 0}, {5000, 5000}});
+ vector<BitExtent> out_over_extents =
+ ByteToBitExtent({{12288, 0}, {12388, 2000}, {4096, 0}});
+ ASSERT_TRUE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+}
+
+TEST(DeflateUtilsTest, ShiftBitExtentsOverExtentsBoundaryTest) {
+ vector<Extent> base_extents = {ExtentForRange(1, 1)};
+ vector<BitExtent> over_extents = ByteToBitExtent({{2, 4096}});
+ vector<BitExtent> out_over_extents = {};
+ EXPECT_FALSE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+
+ base_extents = {ExtentForRange(1, 1)};
+ over_extents = {};
+ out_over_extents = {};
+ EXPECT_TRUE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+
+ base_extents = {};
+ over_extents = {};
+ out_over_extents = {};
+ EXPECT_TRUE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+
+ base_extents = {};
+ over_extents = ByteToBitExtent({{0, 1}});
+ out_over_extents = ByteToBitExtent({{0, 1}});
+ EXPECT_FALSE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+
+ base_extents = {ExtentForRange(1, 2)};
+ over_extents = ByteToBitExtent({{0, 3 * 4096}, {4 * 4096, 4096}});
+ out_over_extents = ByteToBitExtent({{0, 3 * 4096}, {4 * 4096, 4096}});
+ EXPECT_FALSE(ShiftBitExtentsOverExtents(base_extents, &over_extents));
+ EXPECT_EQ(over_extents, out_over_extents);
+}
+
+TEST(DeflateUtilsTest, FindDeflatesTest) {
+ vector<Extent> extents = {
+ ExtentForRange(1, 1), ExtentForRange(3, 1), ExtentForRange(5, 1)};
+ vector<BitExtent> in_deflates = ByteToBitExtent({{0, 0},
+ {10, 400},
+ {4096, 0},
+ {3000, 2000},
+ {4096, 100},
+ {4097, 100},
+ {8100, 92},
+ {8100, 93},
+ {8100, 6000},
+ {25000, 1}});
+ vector<BitExtent> expected_out_deflates =
+ ByteToBitExtent({{4096, 0}, {4096, 100}, {4097, 100}, {8100, 92}});
+ vector<BitExtent> out_deflates;
+ out_deflates = FindDeflates(extents, in_deflates);
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+}
+
+TEST(DeflateUtilsTest, FindDeflatesBoundaryTest) {
+ vector<Extent> extents = {};
+ vector<BitExtent> in_deflates = ByteToBitExtent({{0, 0}, {8100, 93}});
+ vector<BitExtent> expected_out_deflates = {};
+ vector<BitExtent> out_deflates;
+ out_deflates = FindDeflates(extents, in_deflates);
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+
+ extents = {};
+ in_deflates = {};
+ out_deflates = FindDeflates(extents, in_deflates);
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+}
+
+TEST(DeflateUtilsTest, CompactTest) {
+ vector<Extent> extents = {
+ ExtentForRange(1, 1), ExtentForRange(5, 1), ExtentForRange(3, 1)};
+ vector<BitExtent> in_deflates =
+ ByteToBitExtent({{4096, 0}, {12288, 4096}, {4096, 100}, {20480, 100}});
+ vector<BitExtent> expected_out_deflates =
+ ByteToBitExtent({{0, 0}, {0, 100}, {4096, 100}, {8192, 4096}});
+ vector<BitExtent> out_deflates;
+ ASSERT_TRUE(CompactDeflates(extents, in_deflates, &out_deflates));
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+}
+
+TEST(DeflateUtilsTest, CompactBoundaryTest) {
+ vector<Extent> extents = {};
+ vector<BitExtent> in_deflates = ByteToBitExtent({{4096, 0}});
+ vector<BitExtent> expected_out_deflates = {};
+ vector<BitExtent> out_deflates;
+ EXPECT_FALSE(CompactDeflates(extents, in_deflates, &out_deflates));
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+
+ extents = {};
+ in_deflates = {};
+ ASSERT_TRUE(CompactDeflates(extents, in_deflates, &out_deflates));
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+
+ extents = {ExtentForRange(1, 1)};
+ in_deflates = {};
+ ASSERT_TRUE(CompactDeflates(extents, in_deflates, &out_deflates));
+ EXPECT_EQ(out_deflates, expected_out_deflates);
+}
+
+} // namespace deflate_utils
+} // namespace chromeos_update_engine
diff --git a/payload_generator/delta_diff_utils.cc b/payload_generator/delta_diff_utils.cc
index 1664960..bcbc3a5 100644
--- a/payload_generator/delta_diff_utils.cc
+++ b/payload_generator/delta_diff_utils.cc
@@ -17,18 +17,25 @@
#include "update_engine/payload_generator/delta_diff_utils.h"
#include <endian.h>
-// TODO: Remove these pragmas when b/35721782 is fixed.
+#if defined(__clang__)
+// TODO(*): Remove these pragmas when b/35721782 is fixed.
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmacro-redefined"
+#endif
#include <ext2fs/ext2fs.h>
+#if defined(__clang__)
#pragma clang diagnostic pop
+#endif
#include <unistd.h>
#include <algorithm>
#include <map>
+#include <memory>
+#include <utility>
#include <base/files/file_util.h>
#include <base/format_macros.h>
+#include <base/strings/string_util.h>
#include <base/strings/stringprintf.h>
#include <base/threading/simple_thread.h>
#include <brillo/data_encoding.h>
@@ -37,11 +44,14 @@
#include "update_engine/common/hash_calculator.h"
#include "update_engine/common/subprocess.h"
#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/payload_constants.h"
#include "update_engine/payload_generator/block_mapping.h"
#include "update_engine/payload_generator/bzip.h"
+#include "update_engine/payload_generator/deflate_utils.h"
#include "update_engine/payload_generator/delta_diff_generator.h"
#include "update_engine/payload_generator/extent_ranges.h"
#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/squashfs_filesystem.h"
#include "update_engine/payload_generator/xz.h"
using std::map;
@@ -60,8 +70,8 @@
// The maximum destination size allowed for puffdiff. In general, puffdiff
// should work for arbitrary big files, but the payload application is quite
-// memory intensive, so we limit these operations to 50 MiB.
-const uint64_t kMaxPuffdiffDestinationSize = 50 * 1024 * 1024; // bytes
+// memory intensive, so we limit these operations to 150 MiB.
+const uint64_t kMaxPuffdiffDestinationSize = 150 * 1024 * 1024; // bytes
// Process a range of blocks from |range_start| to |range_end| in the extent at
// position |*idx_p| of |extents|. If |do_remove| is true, this range will be
@@ -172,6 +182,8 @@
const PayloadVersion& version,
const vector<Extent>& old_extents,
const vector<Extent>& new_extents,
+ const vector<puffin::BitExtent>& old_deflates,
+ const vector<puffin::BitExtent>& new_deflates,
const string& name,
ssize_t chunk_blocks,
BlobFileWriter* blob_file)
@@ -180,6 +192,8 @@
version_(version),
old_extents_(old_extents),
new_extents_(new_extents),
+ old_deflates_(old_deflates),
+ new_deflates_(new_deflates),
name_(name),
chunk_blocks_(chunk_blocks),
blob_file_(blob_file) {}
@@ -204,6 +218,8 @@
// The block ranges of the old/new file within the src/tgt image
const vector<Extent> old_extents_;
const vector<Extent> new_extents_;
+ const vector<puffin::BitExtent> old_deflates_;
+ const vector<puffin::BitExtent> new_deflates_;
const string name_;
// Block limit of one aop.
ssize_t chunk_blocks_;
@@ -218,17 +234,22 @@
void FileDeltaProcessor::Run() {
TEST_AND_RETURN(blob_file_ != nullptr);
+ LOG(INFO) << "Encoding file " << name_ << " ("
+ << utils::BlocksInExtents(new_extents_) << " blocks)";
+
if (!DeltaReadFile(&file_aops_,
old_part_,
new_part_,
old_extents_,
new_extents_,
+ old_deflates_,
+ new_deflates_,
name_,
chunk_blocks_,
version_,
blob_file_)) {
LOG(ERROR) << "Failed to generate delta for " << name_ << " ("
- << BlocksInExtents(new_extents_) << " blocks)";
+ << utils::BlocksInExtents(new_extents_) << " blocks)";
}
}
@@ -259,17 +280,20 @@
&old_visited_blocks,
&new_visited_blocks));
- map<string, vector<Extent>> old_files_map;
+ bool puffdiff_allowed = version.OperationAllowed(InstallOperation::PUFFDIFF);
+ map<string, FilesystemInterface::File> old_files_map;
if (old_part.fs_interface) {
vector<FilesystemInterface::File> old_files;
- old_part.fs_interface->GetFiles(&old_files);
+ TEST_AND_RETURN_FALSE(deflate_utils::PreprocessParitionFiles(
+ old_part, &old_files, puffdiff_allowed));
for (const FilesystemInterface::File& file : old_files)
- old_files_map[file.name] = file.extents;
+ old_files_map[file.name] = file;
}
TEST_AND_RETURN_FALSE(new_part.fs_interface);
vector<FilesystemInterface::File> new_files;
- new_part.fs_interface->GetFiles(&new_files);
+ TEST_AND_RETURN_FALSE(deflate_utils::PreprocessParitionFiles(
+ new_part, &new_files, puffdiff_allowed));
vector<FileDeltaProcessor> file_delta_processors;
@@ -292,9 +316,6 @@
if (new_file_extents.empty())
continue;
- LOG(INFO) << "Encoding file " << new_file.name << " ("
- << BlocksInExtents(new_file_extents) << " blocks)";
-
// We can't visit each dst image inode more than once, as that would
// duplicate work. Here, we avoid visiting each source image inode
// more than once. Technically, we could have multiple operations
@@ -303,8 +324,9 @@
// from using a graph/cycle detection/etc to generate diffs, and at that
// time, it will be easy (non-complex) to have many operations read
// from the same source blocks. At that time, this code can die. -adlr
- vector<Extent> old_file_extents = FilterExtentRanges(
- old_files_map[new_file.name], old_visited_blocks);
+ auto old_file = old_files_map[new_file.name];
+ vector<Extent> old_file_extents =
+ FilterExtentRanges(old_file.extents, old_visited_blocks);
old_visited_blocks.AddExtents(old_file_extents);
file_delta_processors.emplace_back(old_part.path,
@@ -312,6 +334,8 @@
version,
std::move(old_file_extents),
std::move(new_file_extents),
+ old_file.deflates,
+ new_file.deflates,
new_file.name, // operation name
hard_chunk_blocks,
blob_file);
@@ -344,9 +368,9 @@
old_unvisited = FilterExtentRanges(old_unvisited, old_visited_blocks);
}
- LOG(INFO) << "Scanning " << BlocksInExtents(new_unvisited)
- << " unwritten blocks using chunk size of "
- << soft_chunk_blocks << " blocks.";
+ LOG(INFO) << "Scanning " << utils::BlocksInExtents(new_unvisited)
+ << " unwritten blocks using chunk size of " << soft_chunk_blocks
+ << " blocks.";
// We use the soft_chunk_blocks limit for the <non-file-data> as we don't
// really know the structure of this data and we should not expect it to have
// redundancy between partitions.
@@ -355,6 +379,8 @@
new_part.path,
old_unvisited,
new_unvisited,
+ {}, // old_deflates,
+ {}, // new_deflates
"<non-file-data>", // operation name
soft_chunk_blocks,
version,
@@ -459,13 +485,15 @@
new_part,
vector<Extent>(), // old_extents
vector<Extent>{extent}, // new_extents
+ {}, // old_deflates
+ {}, // new_deflates
"<zeros>",
chunk_blocks,
version,
blob_file));
}
LOG(INFO) << "Produced " << (aops->size() - num_ops) << " operations for "
- << BlocksInExtents(new_zeros) << " zeroed blocks";
+ << utils::BlocksInExtents(new_zeros) << " zeroed blocks";
// Produce MOVE/SOURCE_COPY operations for the moved blocks.
num_ops = aops->size();
@@ -519,6 +547,8 @@
const string& new_part,
const vector<Extent>& old_extents,
const vector<Extent>& new_extents,
+ const vector<puffin::BitExtent>& old_deflates,
+ const vector<puffin::BitExtent>& new_deflates,
const string& name,
ssize_t chunk_blocks,
const PayloadVersion& version,
@@ -526,7 +556,7 @@
brillo::Blob data;
InstallOperation operation;
- uint64_t total_blocks = BlocksInExtents(new_extents);
+ uint64_t total_blocks = utils::BlocksInExtents(new_extents);
if (chunk_blocks == -1)
chunk_blocks = total_blocks;
@@ -547,6 +577,8 @@
new_part,
old_extents_chunk,
new_extents_chunk,
+ old_deflates,
+ new_deflates,
version,
&data,
&operation));
@@ -637,14 +669,16 @@
const string& new_part,
const vector<Extent>& old_extents,
const vector<Extent>& new_extents,
+ const vector<puffin::BitExtent>& old_deflates,
+ const vector<puffin::BitExtent>& new_deflates,
const PayloadVersion& version,
brillo::Blob* out_data,
InstallOperation* out_op) {
InstallOperation operation;
// We read blocks from old_extents and write blocks to new_extents.
- uint64_t blocks_to_read = BlocksInExtents(old_extents);
- uint64_t blocks_to_write = BlocksInExtents(new_extents);
+ uint64_t blocks_to_read = utils::BlocksInExtents(old_extents);
+ uint64_t blocks_to_write = utils::BlocksInExtents(new_extents);
// Disable bsdiff, and puffdiff when the data is too big.
bool bsdiff_allowed =
@@ -725,32 +759,91 @@
}
}
if (puffdiff_allowed) {
- LOG(ERROR) << "puffdiff is not supported yet!";
- return false;
+ // Find all deflate positions inside the given extents and then put all
+ // deflates together because we have already read all the extents into
+ // one buffer.
+ vector<puffin::BitExtent> src_deflates;
+ TEST_AND_RETURN_FALSE(deflate_utils::FindAndCompactDeflates(
+ src_extents, old_deflates, &src_deflates));
+
+ vector<puffin::BitExtent> dst_deflates;
+ TEST_AND_RETURN_FALSE(deflate_utils::FindAndCompactDeflates(
+ dst_extents, new_deflates, &dst_deflates));
+
+ // Remove equal deflates. TODO(*): We can do a N*N check using
+ // hashing. It will not reduce the payload size, but it will speeds up
+ // the puffing on the client device.
+ auto src = src_deflates.begin();
+ auto dst = dst_deflates.begin();
+ for (; src != src_deflates.end() && dst != dst_deflates.end();) {
+ auto src_in_bytes = deflate_utils::ExpandToByteExtent(*src);
+ auto dst_in_bytes = deflate_utils::ExpandToByteExtent(*dst);
+ if (src_in_bytes.length == dst_in_bytes.length &&
+ !memcmp(old_data.data() + src_in_bytes.offset,
+ new_data.data() + dst_in_bytes.offset,
+ src_in_bytes.length)) {
+ src = src_deflates.erase(src);
+ dst = dst_deflates.erase(dst);
+ } else {
+ src++;
+ dst++;
+ }
+ }
+
+ // Only Puffdiff if both files have at least one deflate left.
+ if (!src_deflates.empty() && !dst_deflates.empty()) {
+ brillo::Blob puffdiff_delta;
+ string temp_file_path;
+ TEST_AND_RETURN_FALSE(utils::MakeTempFile(
+ "puffdiff-delta.XXXXXX", &temp_file_path, nullptr));
+ ScopedPathUnlinker temp_file_unlinker(temp_file_path);
+
+ // Perform PuffDiff operation.
+ TEST_AND_RETURN_FALSE(puffin::PuffDiff(old_data,
+ new_data,
+ src_deflates,
+ dst_deflates,
+ temp_file_path,
+ &puffdiff_delta));
+ TEST_AND_RETURN_FALSE(puffdiff_delta.size() > 0);
+ if (puffdiff_delta.size() < data_blob.size()) {
+ operation.set_type(InstallOperation::PUFFDIFF);
+ data_blob = std::move(puffdiff_delta);
+ }
+ }
}
}
}
- size_t removed_bytes = 0;
// Remove identical src/dst block ranges in MOVE operations.
if (operation.type() == InstallOperation::MOVE) {
- removed_bytes = RemoveIdenticalBlockRanges(
+ auto removed_bytes = RemoveIdenticalBlockRanges(
&src_extents, &dst_extents, new_data.size());
+ operation.set_src_length(old_data.size() - removed_bytes);
+ operation.set_dst_length(new_data.size() - removed_bytes);
}
- // Set legacy src_length and dst_length fields.
- operation.set_src_length(old_data.size() - removed_bytes);
- operation.set_dst_length(new_data.size() - removed_bytes);
- // Embed extents in the operation.
- StoreExtents(src_extents, operation.mutable_src_extents());
+ // WARNING: We always set legacy |src_length| and |dst_length| fields for
+ // BSDIFF. For SOURCE_BSDIFF we only set them for minor version 3 and
+ // lower. This is needed because we used to use these two parameters in the
+ // SOURCE_BSDIFF for minor version 3 and lower, but we do not need them
+ // anymore in higher minor versions. This means if we stop adding these
+ // parameters for those minor versions, the delta payloads will be invalid.
+ if (operation.type() == InstallOperation::BSDIFF ||
+ (operation.type() == InstallOperation::SOURCE_BSDIFF &&
+ version.minor <= kOpSrcHashMinorPayloadVersion)) {
+ operation.set_src_length(old_data.size());
+ operation.set_dst_length(new_data.size());
+ }
+
+ // Embed extents in the operation. Replace (all variants), zero and discard
+ // operations should not have source extents.
+ if (!IsNoSourceOperation(operation.type())) {
+ StoreExtents(src_extents, operation.mutable_src_extents());
+ }
+ // All operations have dst_extents.
StoreExtents(dst_extents, operation.mutable_dst_extents());
- // Replace operations should not have source extents.
- if (IsAReplaceOperation(operation.type())) {
- operation.clear_src_extents();
- operation.clear_src_length();
- }
-
*out_data = std::move(data_blob);
*out_op = operation;
return true;
@@ -762,6 +855,12 @@
op_type == InstallOperation::REPLACE_XZ);
}
+bool IsNoSourceOperation(InstallOperation_Type op_type) {
+ return (IsAReplaceOperation(op_type) ||
+ op_type == InstallOperation::ZERO ||
+ op_type == InstallOperation::DISCARD);
+}
+
// Returns true if |op| is a no-op operation that doesn't do any useful work
// (e.g., a move operation that copies blocks onto themselves).
bool IsNoopOperation(const InstallOperation& op) {
diff --git a/payload_generator/delta_diff_utils.h b/payload_generator/delta_diff_utils.h
index 2d49459..dea8535 100644
--- a/payload_generator/delta_diff_utils.h
+++ b/payload_generator/delta_diff_utils.h
@@ -21,6 +21,7 @@
#include <vector>
#include <brillo/secure_blob.h>
+#include <puffin/puffdiff.h>
#include "update_engine/payload_generator/annotated_operation.h"
#include "update_engine/payload_generator/extent_ranges.h"
@@ -76,12 +77,15 @@
// stored in |new_part| in the blocks described by |new_extents| and, if it
// exists, the old version exists in |old_part| in the blocks described by
// |old_extents|. The operations added to |aops| reference the data blob
-// in the |blob_file|. Returns true on success.
+// in the |blob_file|. |old_deflates| and |new_deflates| are all deflate
+// locations in |old_part| and |new_part|. Returns true on success.
bool DeltaReadFile(std::vector<AnnotatedOperation>* aops,
const std::string& old_part,
const std::string& new_part,
const std::vector<Extent>& old_extents,
const std::vector<Extent>& new_extents,
+ const std::vector<puffin::BitExtent>& old_deflates,
+ const std::vector<puffin::BitExtent>& new_deflates,
const std::string& name,
ssize_t chunk_blocks,
const PayloadVersion& version,
@@ -94,11 +98,14 @@
// MOVE or SOURCE_COPY operation. If there is a change, the smallest of the
// operations allowed in the given |version| (REPLACE, REPLACE_BZ, BSDIFF,
// SOURCE_BSDIFF, or PUFFDIFF) wins.
-// |new_extents| must not be empty. Returns true on success.
+// |new_extents| must not be empty. |old_deflates| and |new_deflates| are all
+// the deflate locations in |old_part| and |new_part|. Returns true on success.
bool ReadExtentsToDiff(const std::string& old_part,
const std::string& new_part,
const std::vector<Extent>& old_extents,
const std::vector<Extent>& new_extents,
+ const std::vector<puffin::BitExtent>& old_deflates,
+ const std::vector<puffin::BitExtent>& new_deflates,
const PayloadVersion& version,
brillo::Blob* out_data,
InstallOperation* out_op);
@@ -112,9 +119,12 @@
brillo::Blob* out_blob,
InstallOperation_Type* out_type);
-// Returns whether op_type is one of the REPLACE full operations.
+// Returns whether |op_type| is one of the REPLACE full operations.
bool IsAReplaceOperation(InstallOperation_Type op_type);
+// Returns true if an operation with type |op_type| has no |src_extents|.
+bool IsNoSourceOperation(InstallOperation_Type op_type);
+
// Returns true if |op| is a no-op operation that doesn't do any useful work
// (e.g., a move operation that copies blocks onto themselves).
bool IsNoopOperation(const InstallOperation& op);
diff --git a/payload_generator/delta_diff_utils_unittest.cc b/payload_generator/delta_diff_utils_unittest.cc
index bb83942..a83cea2 100644
--- a/payload_generator/delta_diff_utils_unittest.cc
+++ b/payload_generator/delta_diff_utils_unittest.cc
@@ -179,6 +179,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion, kInPlaceMinorPayloadVersion),
&data,
&op));
@@ -192,9 +194,9 @@
EXPECT_EQ(kBlockSize, op.src_length());
EXPECT_EQ(1, op.dst_extents_size());
EXPECT_EQ(kBlockSize, op.dst_length());
- EXPECT_EQ(BlocksInExtents(op.src_extents()),
- BlocksInExtents(op.dst_extents()));
- EXPECT_EQ(1U, BlocksInExtents(op.dst_extents()));
+ EXPECT_EQ(utils::BlocksInExtents(op.src_extents()),
+ utils::BlocksInExtents(op.dst_extents()));
+ EXPECT_EQ(1U, utils::BlocksInExtents(op.dst_extents()));
}
TEST_F(DeltaDiffUtilsTest, MoveWithSameBlock) {
@@ -218,8 +220,8 @@
ExtentForRange(24, 3),
ExtentForRange(29, 1) };
- uint64_t num_blocks = BlocksInExtents(old_extents);
- EXPECT_EQ(num_blocks, BlocksInExtents(new_extents));
+ uint64_t num_blocks = utils::BlocksInExtents(old_extents);
+ EXPECT_EQ(num_blocks, utils::BlocksInExtents(new_extents));
// The size of the data should match the total number of blocks. Each block
// has a different content.
@@ -238,6 +240,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion, kInPlaceMinorPayloadVersion),
&data,
&op));
@@ -258,7 +262,7 @@
ExtentForRange(18, 1),
ExtentForRange(20, 1),
ExtentForRange(26, 1) };
- num_blocks = BlocksInExtents(old_extents);
+ num_blocks = utils::BlocksInExtents(old_extents);
EXPECT_EQ(num_blocks * kBlockSize, op.src_length());
EXPECT_EQ(num_blocks * kBlockSize, op.dst_length());
@@ -301,6 +305,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion, kInPlaceMinorPayloadVersion),
&data,
&op));
@@ -315,9 +321,9 @@
EXPECT_EQ(kBlockSize, op.src_length());
EXPECT_EQ(1, op.dst_extents_size());
EXPECT_EQ(kBlockSize, op.dst_length());
- EXPECT_EQ(BlocksInExtents(op.src_extents()),
- BlocksInExtents(op.dst_extents()));
- EXPECT_EQ(1U, BlocksInExtents(op.dst_extents()));
+ EXPECT_EQ(utils::BlocksInExtents(op.src_extents()),
+ utils::BlocksInExtents(op.dst_extents()));
+ EXPECT_EQ(1U, utils::BlocksInExtents(op.dst_extents()));
}
TEST_F(DeltaDiffUtilsTest, ReplaceSmallTest) {
@@ -349,6 +355,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion,
kInPlaceMinorPayloadVersion),
&data,
@@ -364,8 +372,8 @@
EXPECT_EQ(0, op.src_extents_size());
EXPECT_FALSE(op.has_src_length());
EXPECT_EQ(1, op.dst_extents_size());
- EXPECT_EQ(data_to_test.size(), op.dst_length());
- EXPECT_EQ(1U, BlocksInExtents(op.dst_extents()));
+ EXPECT_FALSE(op.has_dst_length());
+ EXPECT_EQ(1U, utils::BlocksInExtents(op.dst_extents()));
}
}
@@ -390,6 +398,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion, kSourceMinorPayloadVersion),
&data,
&op));
@@ -422,6 +432,8 @@
new_part_.path,
old_extents,
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
PayloadVersion(kChromeOSMajorPayloadVersion, kSourceMinorPayloadVersion),
&data,
&op));
@@ -622,7 +634,8 @@
// The last range is split since the old image has zeros in part of it.
ExtentForRange(30, 20),
};
- brillo::Blob zeros_data(BlocksInExtents(new_zeros) * block_size_, '\0');
+ brillo::Blob zeros_data(utils::BlocksInExtents(new_zeros) * block_size_,
+ '\0');
EXPECT_TRUE(WriteExtents(new_part_.path, new_zeros, block_size_, zeros_data));
vector<Extent> old_zeros = vector<Extent>{ExtentForRange(43, 7)};
diff --git a/payload_generator/ext2_filesystem.cc b/payload_generator/ext2_filesystem.cc
index ee2f8c2..07ec371 100644
--- a/payload_generator/ext2_filesystem.cc
+++ b/payload_generator/ext2_filesystem.cc
@@ -17,12 +17,16 @@
#include "update_engine/payload_generator/ext2_filesystem.h"
#include <et/com_err.h>
-// TODO: Remove these pragmas when b/35721782 is fixed.
+#if defined(__clang__)
+// TODO(*): Remove these pragmas when b/35721782 is fixed.
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmacro-redefined"
+#endif
#include <ext2fs/ext2_io.h>
#include <ext2fs/ext2fs.h>
+#if defined(__clang__)
#pragma clang diagnostic pop
+#endif
#include <map>
#include <set>
@@ -344,7 +348,7 @@
return false;
brillo::Blob blob;
- uint64_t physical_size = BlocksInExtents(extents) * filsys_->blocksize;
+ uint64_t physical_size = utils::BlocksInExtents(extents) * filsys_->blocksize;
// Sparse holes in the settings file are not supported.
if (EXT2_I_SIZE(&ino_data) > physical_size)
return false;
diff --git a/payload_generator/ext2_filesystem.h b/payload_generator/ext2_filesystem.h
index 1a4e1a1..c0562d0 100644
--- a/payload_generator/ext2_filesystem.h
+++ b/payload_generator/ext2_filesystem.h
@@ -23,11 +23,15 @@
#include <string>
#include <vector>
+#if defined(__clang__)
// TODO: Remove these pragmas when b/35721782 is fixed.
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wmacro-redefined"
+#endif
#include <ext2fs/ext2fs.h>
+#if defined(__clang__)
#pragma clang diagnostic pop
+#endif
namespace chromeos_update_engine {
diff --git a/payload_generator/ext2_filesystem_unittest.cc b/payload_generator/ext2_filesystem_unittest.cc
index a3c7731..5360e6c 100644
--- a/payload_generator/ext2_filesystem_unittest.cc
+++ b/payload_generator/ext2_filesystem_unittest.cc
@@ -158,7 +158,8 @@
// Small symlinks don't actually have data blocks.
EXPECT_TRUE(map_files["/link-short_symlink"].extents.empty());
- EXPECT_EQ(1U, BlocksInExtents(map_files["/link-long_symlink"].extents));
+ EXPECT_EQ(1U,
+ utils::BlocksInExtents(map_files["/link-long_symlink"].extents));
// Hard-links report the same list of blocks.
EXPECT_EQ(map_files["/link-hard-regular-16k"].extents,
@@ -168,14 +169,19 @@
// The number of blocks in these files doesn't depend on the
// block size.
EXPECT_TRUE(map_files["/empty-file"].extents.empty());
- EXPECT_EQ(1U, BlocksInExtents(map_files["/regular-small"].extents));
- EXPECT_EQ(1U, BlocksInExtents(map_files["/regular-with_net_cap"].extents));
+ EXPECT_EQ(1U, utils::BlocksInExtents(map_files["/regular-small"].extents));
+ EXPECT_EQ(
+ 1U, utils::BlocksInExtents(map_files["/regular-with_net_cap"].extents));
EXPECT_TRUE(map_files["/sparse_empty-10k"].extents.empty());
EXPECT_TRUE(map_files["/sparse_empty-2blocks"].extents.empty());
- EXPECT_EQ(1U, BlocksInExtents(map_files["/sparse-16k-last_block"].extents));
- EXPECT_EQ(1U,
- BlocksInExtents(map_files["/sparse-16k-first_block"].extents));
- EXPECT_EQ(2U, BlocksInExtents(map_files["/sparse-16k-holes"].extents));
+ EXPECT_EQ(
+ 1U,
+ utils::BlocksInExtents(map_files["/sparse-16k-last_block"].extents));
+ EXPECT_EQ(
+ 1U,
+ utils::BlocksInExtents(map_files["/sparse-16k-first_block"].extents));
+ EXPECT_EQ(2U,
+ utils::BlocksInExtents(map_files["/sparse-16k-holes"].extents));
}
}
diff --git a/payload_generator/extent_ranges.cc b/payload_generator/extent_ranges.cc
index 0e0cdf7..c1d3d63 100644
--- a/payload_generator/extent_ranges.cc
+++ b/payload_generator/extent_ranges.cc
@@ -23,6 +23,7 @@
#include <base/logging.h>
+#include "update_engine/common/utils.h"
#include "update_engine/payload_consumer/payload_constants.h"
#include "update_engine/payload_generator/extent_utils.h"
@@ -250,7 +251,7 @@
out.back().set_num_blocks(blocks_needed);
break;
}
- CHECK(out_blocks == BlocksInExtents(out));
+ CHECK(out_blocks == utils::BlocksInExtents(out));
return out;
}
diff --git a/payload_generator/extent_utils.cc b/payload_generator/extent_utils.cc
index 89ccca2..47073f9 100644
--- a/payload_generator/extent_utils.cc
+++ b/payload_generator/extent_utils.cc
@@ -53,16 +53,6 @@
extents->push_back(new_extent);
}
-Extent GetElement(const vector<Extent>& collection, size_t index) {
- return collection[index];
-}
-
-Extent GetElement(
- const google::protobuf::RepeatedPtrField<Extent>& collection,
- size_t index) {
- return collection.Get(index);
-}
-
void ExtendExtents(
google::protobuf::RepeatedPtrField<Extent>* extents,
const google::protobuf::RepeatedPtrField<Extent>& extents_to_add) {
diff --git a/payload_generator/extent_utils.h b/payload_generator/extent_utils.h
index 3e45264..f5fbb0e 100644
--- a/payload_generator/extent_utils.h
+++ b/payload_generator/extent_utils.h
@@ -32,31 +32,12 @@
// into an arbitrary place in the extents.
void AppendBlockToExtents(std::vector<Extent>* extents, uint64_t block);
-// Get/SetElement are intentionally overloaded so that templated functions
-// can accept either type of collection of Extents.
-Extent GetElement(const std::vector<Extent>& collection, size_t index);
-Extent GetElement(
- const google::protobuf::RepeatedPtrField<Extent>& collection,
- size_t index);
-
-// Return the total number of blocks in a collection (vector or
-// RepeatedPtrField) of Extents.
-template<typename T>
-uint64_t BlocksInExtents(const T& collection) {
- uint64_t ret = 0;
- for (size_t i = 0; i < static_cast<size_t>(collection.size()); ++i) {
- ret += GetElement(collection, i).num_blocks();
- }
- return ret;
-}
-
// Takes a collection (vector or RepeatedPtrField) of Extent and
// returns a vector of the blocks referenced, in order.
template<typename T>
std::vector<uint64_t> ExpandExtents(const T& extents) {
std::vector<uint64_t> ret;
- for (size_t i = 0, e = static_cast<size_t>(extents.size()); i != e; ++i) {
- const Extent extent = GetElement(extents, i);
+ for (const auto& extent : extents) {
if (extent.start_block() == kSparseHole) {
ret.resize(ret.size() + extent.num_blocks(), kSparseHole);
} else {
diff --git a/payload_generator/extent_utils_unittest.cc b/payload_generator/extent_utils_unittest.cc
index d470e7b..eef4385 100644
--- a/payload_generator/extent_utils_unittest.cc
+++ b/payload_generator/extent_utils_unittest.cc
@@ -54,23 +54,23 @@
TEST(ExtentUtilsTest, BlocksInExtentsTest) {
{
vector<Extent> extents;
- EXPECT_EQ(0U, BlocksInExtents(extents));
+ EXPECT_EQ(0U, utils::BlocksInExtents(extents));
extents.push_back(ExtentForRange(0, 1));
- EXPECT_EQ(1U, BlocksInExtents(extents));
+ EXPECT_EQ(1U, utils::BlocksInExtents(extents));
extents.push_back(ExtentForRange(23, 55));
- EXPECT_EQ(56U, BlocksInExtents(extents));
+ EXPECT_EQ(56U, utils::BlocksInExtents(extents));
extents.push_back(ExtentForRange(1, 2));
- EXPECT_EQ(58U, BlocksInExtents(extents));
+ EXPECT_EQ(58U, utils::BlocksInExtents(extents));
}
{
google::protobuf::RepeatedPtrField<Extent> extents;
- EXPECT_EQ(0U, BlocksInExtents(extents));
+ EXPECT_EQ(0U, utils::BlocksInExtents(extents));
*extents.Add() = ExtentForRange(0, 1);
- EXPECT_EQ(1U, BlocksInExtents(extents));
+ EXPECT_EQ(1U, utils::BlocksInExtents(extents));
*extents.Add() = ExtentForRange(23, 55);
- EXPECT_EQ(56U, BlocksInExtents(extents));
+ EXPECT_EQ(56U, utils::BlocksInExtents(extents));
*extents.Add() = ExtentForRange(1, 2);
- EXPECT_EQ(58U, BlocksInExtents(extents));
+ EXPECT_EQ(58U, utils::BlocksInExtents(extents));
}
}
diff --git a/payload_generator/filesystem_interface.h b/payload_generator/filesystem_interface.h
index 866c46b..b1506e4 100644
--- a/payload_generator/filesystem_interface.h
+++ b/payload_generator/filesystem_interface.h
@@ -33,6 +33,7 @@
#include <base/macros.h>
#include <brillo/key_value_store.h>
+#include <puffin/utils.h>
#include "update_engine/update_metadata.pb.h"
@@ -62,6 +63,10 @@
// between 0 and GetBlockCount() - 1. The blocks are encoded in extents,
// indicating the starting block, and the number of consecutive blocks.
std::vector<Extent> extents;
+
+ // All the deflate locations in the file. These locations are not relative
+ // to the extents. They are relative to the file system itself.
+ std::vector<puffin::BitExtent> deflates;
};
virtual ~FilesystemInterface() = default;
diff --git a/payload_generator/full_update_generator_unittest.cc b/payload_generator/full_update_generator_unittest.cc
index 9e62de2..6da4d10 100644
--- a/payload_generator/full_update_generator_unittest.cc
+++ b/payload_generator/full_update_generator_unittest.cc
@@ -16,6 +16,7 @@
#include "update_engine/payload_generator/full_update_generator.h"
+#include <memory>
#include <string>
#include <vector>
@@ -116,9 +117,9 @@
// new_part has one chunk and a half.
EXPECT_EQ(2U, aops.size());
EXPECT_EQ(config_.hard_chunk_size / config_.block_size,
- BlocksInExtents(aops[0].op.dst_extents()));
+ utils::BlocksInExtents(aops[0].op.dst_extents()));
EXPECT_EQ((new_part.size() - config_.hard_chunk_size) / config_.block_size,
- BlocksInExtents(aops[1].op.dst_extents()));
+ utils::BlocksInExtents(aops[1].op.dst_extents()));
}
// Test that if the image size is much smaller than the chunk size, it handles
@@ -138,7 +139,7 @@
// new_part has less than one chunk.
EXPECT_EQ(1U, aops.size());
EXPECT_EQ(new_part.size() / config_.block_size,
- BlocksInExtents(aops[0].op.dst_extents()));
+ utils::BlocksInExtents(aops[0].op.dst_extents()));
}
} // namespace chromeos_update_engine
diff --git a/payload_generator/generate_delta_main.cc b/payload_generator/generate_delta_main.cc
index e2ae776..2729bc4 100644
--- a/payload_generator/generate_delta_main.cc
+++ b/payload_generator/generate_delta_main.cc
@@ -225,7 +225,8 @@
&fake_hardware,
nullptr,
&install_plan,
- &payload);
+ &payload,
+ true); // is_interactive
brillo::Blob buf(1024 * 1024);
int fd = open(payload_file.c_str(), O_RDONLY, 0);
diff --git a/payload_generator/graph_utils.cc b/payload_generator/graph_utils.cc
index 2d5fb63..4829b21 100644
--- a/payload_generator/graph_utils.cc
+++ b/payload_generator/graph_utils.cc
@@ -104,9 +104,9 @@
template<typename T>
void DumpExtents(const T& field, int prepend_space_count) {
string header(prepend_space_count, ' ');
- for (int i = 0, e = field.size(); i != e; ++i) {
- LOG(INFO) << header << "(" << GetElement(field, i).start_block() << ", "
- << GetElement(field, i).num_blocks() << ")";
+ for (const auto& extent : field) {
+ LOG(INFO) << header << "(" << extent.start_block() << ", "
+ << extent.num_blocks() << ")";
}
}
diff --git a/payload_generator/inplace_generator.cc b/payload_generator/inplace_generator.cc
index bc140e8..b858c2b 100644
--- a/payload_generator/inplace_generator.cc
+++ b/payload_generator/inplace_generator.cc
@@ -299,8 +299,7 @@
template<typename T>
bool TempBlocksExistInExtents(const T& extents) {
- for (int i = 0, e = extents.size(); i < e; ++i) {
- Extent extent = GetElement(extents, i);
+ for (const auto& extent : extents) {
uint64_t start = extent.start_block();
uint64_t num = extent.num_blocks();
if (start >= kTempBlockStart || (start + num) >= kTempBlockStart) {
@@ -573,6 +572,8 @@
new_part,
vector<Extent>(), // old_extents
new_extents,
+ {}, // old_deflates
+ {}, // new_deflates
(*graph)[cut.old_dst].aop.name,
-1, // chunk_blocks, forces to have a single operation.
kInPlacePayloadVersion,
diff --git a/payload_generator/mapfile_filesystem.cc b/payload_generator/mapfile_filesystem.cc
index f4f0804..5264a9c 100644
--- a/payload_generator/mapfile_filesystem.cc
+++ b/payload_generator/mapfile_filesystem.cc
@@ -21,9 +21,9 @@
#include <base/files/file_util.h>
#include <base/logging.h>
+#include <base/memory/ptr_util.h>
#include <base/strings/string_number_conversions.h>
#include <base/strings/string_split.h>
-#include <brillo/make_unique_ptr.h>
#include "update_engine/common/utils.h"
#include "update_engine/payload_generator/extent_ranges.h"
@@ -61,8 +61,7 @@
return nullptr;
}
- return brillo::make_unique_ptr(
- new MapfileFilesystem(mapfile_filename, num_blocks));
+ return base::WrapUnique(new MapfileFilesystem(mapfile_filename, num_blocks));
}
MapfileFilesystem::MapfileFilesystem(const string& mapfile_filename,
diff --git a/payload_generator/payload_file.cc b/payload_generator/payload_file.cc
index 4cb117d..f48d2a2 100644
--- a/payload_generator/payload_file.cc
+++ b/payload_generator/payload_file.cc
@@ -19,6 +19,9 @@
#include <endian.h>
#include <algorithm>
+#include <map>
+
+#include <base/strings/stringprintf.h>
#include "update_engine/common/hash_calculator.h"
#include "update_engine/payload_consumer/delta_performer.h"
@@ -320,38 +323,39 @@
}
void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
- vector<DeltaObject> objects;
+ std::map<DeltaObject, int> object_counts;
off_t total_size = 0;
for (const auto& part : part_vec_) {
for (const AnnotatedOperation& aop : part.aops) {
- objects.push_back(DeltaObject(aop.name,
- aop.op.type(),
- aop.op.data_length()));
+ DeltaObject delta(aop.name, aop.op.type(), aop.op.data_length());
+ object_counts[delta]++;
total_size += aop.op.data_length();
}
}
- objects.push_back(DeltaObject("<manifest-metadata>",
- -1,
- metadata_size));
+ object_counts[DeltaObject("<manifest-metadata>", -1, metadata_size)] = 1;
total_size += metadata_size;
- std::sort(objects.begin(), objects.end());
-
- static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
- for (const DeltaObject& object : objects) {
- fprintf(
- stderr, kFormatString,
+ static const char kFormatString[] = "%6.2f%% %10jd %-13s %s %d";
+ for (const auto& object_count : object_counts) {
+ const DeltaObject& object = object_count.first;
+ LOG(INFO) << base::StringPrintf(
+ kFormatString,
object.size * 100.0 / total_size,
static_cast<intmax_t>(object.size),
(object.type >= 0 ? InstallOperationTypeName(
static_cast<InstallOperation_Type>(object.type))
: "-"),
- object.name.c_str());
+ object.name.c_str(),
+ object_count.second);
}
- fprintf(stderr, kFormatString,
- 100.0, static_cast<intmax_t>(total_size), "", "<total>");
+ LOG(INFO) << base::StringPrintf(kFormatString,
+ 100.0,
+ static_cast<intmax_t>(total_size),
+ "",
+ "<total>",
+ 1);
}
} // namespace chromeos_update_engine
diff --git a/payload_generator/payload_generation_config.cc b/payload_generator/payload_generation_config.cc
index e2fec21..836b481 100644
--- a/payload_generator/payload_generation_config.cc
+++ b/payload_generator/payload_generation_config.cc
@@ -165,6 +165,7 @@
case InstallOperation::SOURCE_BSDIFF:
return minor >= kSourceMinorPayloadVersion;
+ case InstallOperation::BROTLI_BSDIFF:
case InstallOperation::PUFFDIFF:
return minor >= kPuffdiffMinorPayloadVersion;
}
diff --git a/payload_generator/squashfs_filesystem.cc b/payload_generator/squashfs_filesystem.cc
new file mode 100644
index 0000000..c98ad12
--- /dev/null
+++ b/payload_generator/squashfs_filesystem.cc
@@ -0,0 +1,332 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+
+#include <fcntl.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+
+#include <base/files/file_util.h>
+#include <base/logging.h>
+#include <base/strings/string_number_conversions.h>
+#include <base/strings/string_split.h>
+#include <brillo/streams/file_stream.h>
+
+#include "update_engine/common/subprocess.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/deflate_utils.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/update_metadata.pb.h"
+
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace chromeos_update_engine {
+
+namespace {
+
+Extent ExtentForBytes(uint64_t block_size,
+ uint64_t start_bytes,
+ uint64_t size_bytes) {
+ uint64_t start_block = start_bytes / block_size;
+ uint64_t end_block = (start_bytes + size_bytes + block_size - 1) / block_size;
+ return ExtentForRange(start_block, end_block - start_block);
+}
+
+// The size of the squashfs super block.
+constexpr size_t kSquashfsSuperBlockSize = 96;
+constexpr uint64_t kSquashfsCompressedBit = 1 << 24;
+constexpr uint32_t kSquashfsZlibCompression = 1;
+
+bool ReadSquashfsHeader(const brillo::Blob blob,
+ SquashfsFilesystem::SquashfsHeader* header) {
+ if (blob.size() < kSquashfsSuperBlockSize) {
+ return false;
+ }
+
+ memcpy(&header->magic, blob.data(), 4);
+ memcpy(&header->block_size, blob.data() + 12, 4);
+ memcpy(&header->compression_type, blob.data() + 20, 2);
+ memcpy(&header->major_version, blob.data() + 28, 2);
+ return true;
+}
+
+bool CheckHeader(const SquashfsFilesystem::SquashfsHeader& header) {
+ return header.magic == 0x73717368 && header.major_version == 4;
+}
+
+bool GetFileMapContent(const string& sqfs_path, string* map) {
+ // Create a tmp file
+ string map_file;
+ TEST_AND_RETURN_FALSE(
+ utils::MakeTempFile("squashfs_file_map.XXXXXX", &map_file, nullptr));
+ ScopedPathUnlinker map_unlinker(map_file);
+
+ // Run unsquashfs to get the system file map.
+ // unsquashfs -m <map-file> <squashfs-file>
+ vector<string> cmd = {"unsquashfs", "-m", map_file, sqfs_path};
+ string stdout;
+ int exit_code;
+ if (!Subprocess::SynchronousExec(cmd, &exit_code, &stdout) ||
+ exit_code != 0) {
+ LOG(ERROR) << "Failed to run unsquashfs -m. The stdout content was: "
+ << stdout;
+ return false;
+ }
+ TEST_AND_RETURN_FALSE(utils::ReadFile(map_file, map));
+ return true;
+}
+
+} // namespace
+
+bool SquashfsFilesystem::Init(const string& map,
+ const string& sqfs_path,
+ size_t size,
+ const SquashfsHeader& header,
+ bool extract_deflates) {
+ size_ = size;
+
+ bool is_zlib = header.compression_type == kSquashfsZlibCompression;
+ if (!is_zlib) {
+ LOG(WARNING) << "Filesystem is not Gzipped. Not filling deflates!";
+ }
+ vector<puffin::ByteExtent> zlib_blks;
+
+ // Reading files map. For the format of the file map look at the comments for
+ // |CreateFromFileMap()|.
+ auto lines = base::SplitStringPiece(map,
+ "\n",
+ base::WhitespaceHandling::KEEP_WHITESPACE,
+ base::SplitResult::SPLIT_WANT_NONEMPTY);
+ for (const auto& line : lines) {
+ auto splits =
+ base::SplitStringPiece(line,
+ " \t",
+ base::WhitespaceHandling::TRIM_WHITESPACE,
+ base::SplitResult::SPLIT_WANT_NONEMPTY);
+ // Only filename is invalid.
+ TEST_AND_RETURN_FALSE(splits.size() > 1);
+ uint64_t start;
+ TEST_AND_RETURN_FALSE(base::StringToUint64(splits[1], &start));
+ uint64_t cur_offset = start;
+ for (size_t i = 2; i < splits.size(); ++i) {
+ uint64_t blk_size;
+ TEST_AND_RETURN_FALSE(base::StringToUint64(splits[i], &blk_size));
+ // TODO(ahassani): For puffin push it into a proper list if uncompressed.
+ auto new_blk_size = blk_size & ~kSquashfsCompressedBit;
+ TEST_AND_RETURN_FALSE(new_blk_size <= header.block_size);
+ if (new_blk_size > 0 && !(blk_size & kSquashfsCompressedBit)) {
+ // Compressed block
+ if (is_zlib && extract_deflates) {
+ zlib_blks.emplace_back(cur_offset, new_blk_size);
+ }
+ }
+ cur_offset += new_blk_size;
+ }
+
+ // If size is zero do not add the file.
+ if (cur_offset - start > 0) {
+ File file;
+ file.name = splits[0].as_string();
+ file.extents = {ExtentForBytes(kBlockSize, start, cur_offset - start)};
+ files_.emplace_back(file);
+ }
+ }
+
+ // Sort all files by their offset in the squashfs.
+ std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
+ return a.extents[0].start_block() < b.extents[0].start_block();
+ });
+ // If there is any overlap between two consecutive extents, remove them. Here
+ // we are assuming all files have exactly one extent. If this assumption
+ // changes then this implementation needs to change too.
+ for (auto first = files_.begin(), second = first + 1;
+ first != files_.end() && second != files_.end();
+ second = first + 1) {
+ auto first_begin = first->extents[0].start_block();
+ auto first_end = first_begin + first->extents[0].num_blocks();
+ auto second_begin = second->extents[0].start_block();
+ auto second_end = second_begin + second->extents[0].num_blocks();
+ // Remove the first file if the size is zero.
+ if (first_end == first_begin) {
+ first = files_.erase(first);
+ } else if (first_end > second_begin) { // We found a collision.
+ if (second_end <= first_end) {
+ // Second file is inside the first file, remove the second file.
+ second = files_.erase(second);
+ } else if (first_begin == second_begin) {
+ // First file is inside the second file, remove the first file.
+ first = files_.erase(first);
+ } else {
+ // Remove overlapping extents from the first file.
+ first->extents[0].set_num_blocks(second_begin - first_begin);
+ ++first;
+ }
+ } else {
+ ++first;
+ }
+ }
+
+ // Find all the metadata including superblock and add them to the list of
+ // files.
+ ExtentRanges file_extents;
+ for (const auto& file : files_) {
+ file_extents.AddExtents(file.extents);
+ }
+ vector<Extent> full = {
+ ExtentForRange(0, (size_ + kBlockSize - 1) / kBlockSize)};
+ auto metadata_extents = FilterExtentRanges(full, file_extents);
+ // For now there should be at most two extents. One for superblock and one for
+ // metadata at the end. Just create appropriate files with <metadata-i> name.
+ // We can add all these extents as one metadata too, but that violates the
+ // contiguous write optimization.
+ for (size_t i = 0; i < metadata_extents.size(); i++) {
+ File file;
+ file.name = "<metadata-" + std::to_string(i) + ">";
+ file.extents = {metadata_extents[i]};
+ files_.emplace_back(file);
+ }
+
+ // Do one last sort before returning.
+ std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
+ return a.extents[0].start_block() < b.extents[0].start_block();
+ });
+
+ if (is_zlib && extract_deflates) {
+ // If it is infact gzipped, then the sqfs_path should be valid to read its
+ // content.
+ TEST_AND_RETURN_FALSE(!sqfs_path.empty());
+ if (zlib_blks.empty()) {
+ return true;
+ }
+
+ // Sort zlib blocks.
+ std::sort(zlib_blks.begin(),
+ zlib_blks.end(),
+ [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) {
+ return a.offset < b.offset;
+ });
+
+ // Sanity check. Make sure zlib blocks are not overlapping.
+ auto result = std::adjacent_find(
+ zlib_blks.begin(),
+ zlib_blks.end(),
+ [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) {
+ return (a.offset + a.length) > b.offset;
+ });
+ TEST_AND_RETURN_FALSE(result == zlib_blks.end());
+
+ vector<puffin::BitExtent> deflates;
+ TEST_AND_RETURN_FALSE(
+ puffin::LocateDeflatesInZlibBlocks(sqfs_path, zlib_blks, &deflates));
+
+ // Add deflates for each file.
+ for (auto& file : files_) {
+ file.deflates = deflate_utils::FindDeflates(file.extents, deflates);
+ }
+ }
+ return true;
+}
+
+unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile(
+ const string& sqfs_path, bool extract_deflates) {
+ if (sqfs_path.empty())
+ return nullptr;
+
+ brillo::StreamPtr sqfs_file =
+ brillo::FileStream::Open(base::FilePath(sqfs_path),
+ brillo::Stream::AccessMode::READ,
+ brillo::FileStream::Disposition::OPEN_EXISTING,
+ nullptr);
+ if (!sqfs_file) {
+ LOG(ERROR) << "Unable to open " << sqfs_path << " for reading.";
+ return nullptr;
+ }
+
+ SquashfsHeader header;
+ brillo::Blob blob(kSquashfsSuperBlockSize);
+ if (!sqfs_file->ReadAllBlocking(blob.data(), blob.size(), nullptr)) {
+ LOG(ERROR) << "Unable to read from file: " << sqfs_path;
+ return nullptr;
+ }
+ if (!ReadSquashfsHeader(blob, &header) || !CheckHeader(header)) {
+ // This is not necessary an error.
+ return nullptr;
+ }
+
+ // Read the map file.
+ string filemap;
+ if (!GetFileMapContent(sqfs_path, &filemap)) {
+ LOG(ERROR) << "Failed to produce squashfs map file: " << sqfs_path;
+ return nullptr;
+ }
+
+ unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
+ if (!sqfs->Init(
+ filemap, sqfs_path, sqfs_file->GetSize(), header, extract_deflates)) {
+ LOG(ERROR) << "Failed to initialized the Squashfs file system";
+ return nullptr;
+ }
+
+ return sqfs;
+}
+
+unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFileMap(
+ const string& filemap, size_t size, const SquashfsHeader& header) {
+ if (!CheckHeader(header)) {
+ LOG(ERROR) << "Invalid Squashfs super block!";
+ return nullptr;
+ }
+
+ unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
+ if (!sqfs->Init(filemap, "", size, header, false)) {
+ LOG(ERROR) << "Failed to initialize the Squashfs file system using filemap";
+ return nullptr;
+ }
+ // TODO(ahassani): Add a function that initializes the puffin related extents.
+ return sqfs;
+}
+
+size_t SquashfsFilesystem::GetBlockSize() const {
+ return kBlockSize;
+}
+
+size_t SquashfsFilesystem::GetBlockCount() const {
+ return size_ / kBlockSize;
+}
+
+bool SquashfsFilesystem::GetFiles(vector<File>* files) const {
+ files->insert(files->end(), files_.begin(), files_.end());
+ return true;
+}
+
+bool SquashfsFilesystem::LoadSettings(brillo::KeyValueStore* store) const {
+ // Settings not supported in squashfs.
+ LOG(ERROR) << "squashfs doesn't support LoadSettings().";
+ return false;
+}
+
+bool SquashfsFilesystem::IsSquashfsImage(const brillo::Blob& blob) {
+ SquashfsHeader header;
+ return ReadSquashfsHeader(blob, &header) && CheckHeader(header);
+}
+} // namespace chromeos_update_engine
diff --git a/payload_generator/squashfs_filesystem.h b/payload_generator/squashfs_filesystem.h
new file mode 100644
index 0000000..b79f8c7
--- /dev/null
+++ b/payload_generator/squashfs_filesystem.h
@@ -0,0 +1,120 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// This class implements a FilesystemInterface, which lets the caller obtain
+// basic information about what files are in the filesystem and where are they
+// located in the disk, but not full access to the uncompressed contents of
+// these files. This class uses the definitions found in
+// fs/squashfs/squashfs_fs.h in the kernel header tree. This class supports
+// squashfs version 4 in little-endian format only.
+
+#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
+#define UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <brillo/secure_blob.h>
+
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/filesystem_interface.h"
+
+namespace chromeos_update_engine {
+
+class SquashfsFilesystem : public FilesystemInterface {
+ public:
+ // From an squashfs image we need: (offset, bytes)
+ // - magic: (0, 4)
+ // * Acceptable value is: 0x73717368
+ // - block size: (12, 4)
+ // - compression type: (20, 2)
+ // * 1 is for zlib, gzip
+ // - major number: (28, 2)
+ // * We only support version 4 for now.
+ struct SquashfsHeader {
+ uint32_t magic;
+ uint32_t block_size;
+ uint16_t compression_type;
+ uint16_t major_version;
+ };
+
+ ~SquashfsFilesystem() override = default;
+
+ // Creates the file system from the Squashfs file itself. If
+ // |extract_deflates| is true, it will process files to find location of all
+ // deflate streams.
+ static std::unique_ptr<SquashfsFilesystem> CreateFromFile(
+ const std::string& sqfs_path, bool extract_deflates);
+
+ // Creates the file system from a file map |filemap| which is a multi-line
+ // string with each line with the following format:
+ //
+ // file_path start_byte compressed_size_1 ... compressed_size_2
+ //
+ // file_path: The name of the file inside the Squashfs File.
+ // start_byte: The byte address of the start of the file.
+ // compressed_size_i: The compressed size of the ith block of the file.
+ //
+ // The 25th bit of compressed_size_i is set if the block is uncompressed.
+ // |size| is the size of the Squashfs image.
+ static std::unique_ptr<SquashfsFilesystem> CreateFromFileMap(
+ const std::string& filemap, size_t size, const SquashfsHeader& header);
+
+ // FilesystemInterface overrides.
+ size_t GetBlockSize() const override;
+ size_t GetBlockCount() const override;
+
+ // Returns one FilesystemInterface::File for every file (that is not added to
+ // fragments) in the squashfs image.
+ //
+ // It also returns the following metadata files:
+ // <fragment-i>: The ith fragment in the Sqauashfs file.
+ // <metadata-i>: The part of the file system that does not belong to any
+ // file. Normally, there is only two: one for superblock and
+ // one for the metadata at the end.
+ bool GetFiles(std::vector<File>* files) const override;
+
+ // Squashfs image does not support this yet.
+ bool LoadSettings(brillo::KeyValueStore* store) const override;
+
+ // Returns true if the first few bytes of a file indicates a valid Squashfs
+ // image. The size of the |blob| should be at least
+ // sizeof(SquashfsHeader) or for now 96 bytes.
+ static bool IsSquashfsImage(const brillo::Blob& blob);
+
+ private:
+ SquashfsFilesystem() = default;
+
+ // Initialize and populates the files in the file system.
+ bool Init(const std::string& map,
+ const std::string& sqfs_path,
+ size_t size,
+ const SquashfsHeader& header,
+ bool extract_deflates);
+
+ // The size of the image in bytes.
+ size_t size_;
+
+ // All the files in the filesystem.
+ std::vector<File> files_;
+
+ DISALLOW_COPY_AND_ASSIGN(SquashfsFilesystem);
+};
+
+} // namespace chromeos_update_engine
+#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_SQUASHFS_FILESYSTEM_H_
diff --git a/payload_generator/squashfs_filesystem_unittest.cc b/payload_generator/squashfs_filesystem_unittest.cc
new file mode 100644
index 0000000..29fcf1c
--- /dev/null
+++ b/payload_generator/squashfs_filesystem_unittest.cc
@@ -0,0 +1,318 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <base/format_macros.h>
+#include <base/logging.h>
+#include <base/strings/string_number_conversions.h>
+#include <base/strings/string_util.h>
+#include <base/strings/stringprintf.h>
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+namespace chromeos_update_engine {
+
+using std::map;
+using std::set;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using test_utils::GetBuildArtifactsPath;
+
+namespace {
+
+constexpr uint64_t kTestBlockSize = 4096;
+constexpr uint64_t kTestSqfsBlockSize = 1 << 15;
+
+// Checks that all the blocks in |extents| are in the range [0, total_blocks).
+void ExpectBlocksInRange(const vector<Extent>& extents, uint64_t total_blocks) {
+ for (const Extent& extent : extents) {
+ EXPECT_LE(0U, extent.start_block());
+ EXPECT_LE(extent.start_block() + extent.num_blocks(), total_blocks);
+ }
+}
+
+SquashfsFilesystem::SquashfsHeader GetSimpleHeader() {
+ // These properties are enough for now. Add more as needed.
+ return {
+ .magic = 0x73717368,
+ .block_size = kTestSqfsBlockSize,
+ .compression_type = 1, // For gzip.
+ .major_version = 4,
+ };
+}
+
+} // namespace
+
+class SquashfsFilesystemTest : public ::testing::Test {
+ public:
+ void CheckSquashfs(const unique_ptr<SquashfsFilesystem>& fs) {
+ ASSERT_TRUE(fs);
+ EXPECT_EQ(kTestBlockSize, fs->GetBlockSize());
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+
+ map<string, FilesystemInterface::File> map_files;
+ for (const auto& file : files) {
+ EXPECT_EQ(map_files.end(), map_files.find(file.name))
+ << "File " << file.name << " repeated in the list.";
+ map_files[file.name] = file;
+ ExpectBlocksInRange(file.extents, fs->GetBlockCount());
+ }
+
+ // Checking the sortness.
+ EXPECT_TRUE(std::is_sorted(files.begin(),
+ files.end(),
+ [](const FilesystemInterface::File& a,
+ const FilesystemInterface::File& b) {
+ return a.extents[0].start_block() <
+ b.extents[0].start_block();
+ }));
+
+ auto overlap_check = [](const FilesystemInterface::File& a,
+ const FilesystemInterface::File& b) {
+ // Return true if overlapping.
+ return a.extents[0].start_block() + a.extents[0].num_blocks() >
+ b.extents[0].start_block();
+ };
+ // Check files are not overlapping.
+ EXPECT_EQ(std::adjacent_find(files.begin(), files.end(), overlap_check),
+ files.end());
+ }
+};
+
+// CreateFromFile() depends on unsquashfs -m, which only exists in Chrome OS.
+#ifdef __CHROMEOS__
+TEST_F(SquashfsFilesystemTest, EmptyFilesystemTest) {
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
+ GetBuildArtifactsPath("gen/disk_sqfs_empty.img"), true);
+ CheckSquashfs(fs);
+
+ // Even an empty squashfs filesystem is rounded up to 4K.
+ EXPECT_EQ(4096 / kTestBlockSize, fs->GetBlockCount());
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+
+ FilesystemInterface::File file;
+ file.name = "<metadata-0>";
+ file.extents.emplace_back();
+ file.extents[0].set_start_block(0);
+ file.extents[0].set_num_blocks(1);
+ EXPECT_EQ(files[0].name, file.name);
+ EXPECT_EQ(files[0].extents, file.extents);
+}
+
+TEST_F(SquashfsFilesystemTest, DefaultFilesystemTest) {
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFile(
+ GetBuildArtifactsPath("gen/disk_sqfs_default.img"), true);
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+
+ FilesystemInterface::File file;
+ file.name = "<fragment-0>";
+ file.extents.emplace_back();
+ file.extents[0].set_start_block(0);
+ file.extents[0].set_num_blocks(1);
+ EXPECT_EQ(files[0].name, file.name);
+ EXPECT_EQ(files[0].extents, file.extents);
+}
+#endif // __CHROMEOS__
+
+TEST_F(SquashfsFilesystemTest, SimpleFileMapTest) {
+ string filemap = R"(dir1/file1 96 4000
+ dir1/file2 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ EXPECT_EQ(files.size(), 2u);
+}
+
+TEST_F(SquashfsFilesystemTest, FileMapZeroSizeFileTest) {
+ // The second file's size is zero.
+ string filemap = R"(dir1/file1 96 4000
+ dir1/file2 4096
+ dir1/file3 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ // The second and third files are removed. The file with size zero is removed.
+ EXPECT_EQ(files.size(), 2u);
+}
+
+// Testing the compressed bit.
+TEST_F(SquashfsFilesystemTest, CompressedBitTest) {
+ string filemap = "dir1/file1 0 " + std::to_string(4000 | (1 << 24)) + "\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 1u);
+}
+
+// Test overlap.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles1Test) {
+ string filemap = R"(file1 0 6000
+ file2 5000 5000)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 3, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 2u);
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 1u);
+ EXPECT_EQ(files[1].extents[0].num_blocks(), 2u);
+}
+
+// Test overlap, first inside second.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles2Test) {
+ string filemap = R"(file1 0 4000
+ file2 0 6000)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].name, "file2");
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 2u);
+}
+
+// Test overlap, second inside first.
+TEST_F(SquashfsFilesystemTest, OverlapingFiles3Test) {
+ string filemap = R"(file1 0 8000
+ file2 100 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ CheckSquashfs(fs);
+
+ vector<FilesystemInterface::File> files;
+ ASSERT_TRUE(fs->GetFiles(&files));
+ ASSERT_EQ(files.size(), 1u);
+ EXPECT_EQ(files[0].name, "file1");
+ EXPECT_EQ(files[0].extents[0].num_blocks(), 2u);
+}
+
+// Fail a line with only one argument.
+TEST_F(SquashfsFilesystemTest, FailOnlyFileNameTest) {
+ string filemap = "dir1/file1\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail a line with space separated filen name
+TEST_F(SquashfsFilesystemTest, FailSpaceInFileNameTest) {
+ string filemap = "dir1 file1 0 10\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail empty line
+TEST_F(SquashfsFilesystemTest, FailEmptyLineTest) {
+ // The second file's size is zero.
+ string filemap = R"(
+ /t
+ dir1/file3 4096 100)";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize * 2, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Fail on bad magic or major
+TEST_F(SquashfsFilesystemTest, FailBadMagicOrMajorTest) {
+ string filemap = "dir1/file1 0 10\n";
+ auto header = GetSimpleHeader();
+ header.magic = 1;
+ EXPECT_FALSE(
+ SquashfsFilesystem::CreateFromFileMap(filemap, kTestBlockSize, header));
+
+ header = GetSimpleHeader();
+ header.major_version = 3;
+ EXPECT_FALSE(
+ SquashfsFilesystem::CreateFromFileMap(filemap, kTestBlockSize, header));
+}
+
+// Fail size with larger than block_size
+TEST_F(SquashfsFilesystemTest, FailLargerThanBlockSizeTest) {
+ string filemap = "file1 0 " + std::to_string(kTestSqfsBlockSize + 1) + "\n";
+ unique_ptr<SquashfsFilesystem> fs = SquashfsFilesystem::CreateFromFileMap(
+ filemap, kTestBlockSize, GetSimpleHeader());
+ EXPECT_FALSE(fs);
+}
+
+// Test is squashfs image.
+TEST_F(SquashfsFilesystemTest, IsSquashfsImageTest) {
+ // Some sample from a recent squashfs file.
+ brillo::Blob super_block = {
+ 0x68, 0x73, 0x71, 0x73, 0x59, 0x05, 0x00, 0x00, 0x09, 0x3a, 0x89, 0x58,
+ 0x00, 0x00, 0x02, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x01, 0x00, 0x11, 0x00,
+ 0xc0, 0x00, 0x06, 0x00, 0x04, 0x00, 0x00, 0x00, 0x89, 0x18, 0xf7, 0x7c,
+ 0x00, 0x00, 0x00, 0x00, 0x2e, 0x33, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00,
+ 0x3a, 0x30, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00, 0x16, 0x33, 0xcd, 0x16,
+ 0x00, 0x00, 0x00, 0x00, 0x07, 0x62, 0xcc, 0x16, 0x00, 0x00, 0x00, 0x00,
+ 0x77, 0xe6, 0xcc, 0x16, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x25, 0xcd, 0x16,
+ 0x00, 0x00, 0x00, 0x00, 0x10, 0x30, 0xcd, 0x16, 0x00, 0x00, 0x00, 0x00};
+
+ EXPECT_TRUE(SquashfsFilesystem::IsSquashfsImage(super_block));
+
+ // Bad magic
+ auto bad_super_block = super_block;
+ bad_super_block[1] = 0x02;
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+
+ // Bad major
+ bad_super_block = super_block;
+ bad_super_block[28] = 0x03;
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+
+ // Small size;
+ bad_super_block = super_block;
+ bad_super_block.resize(10);
+ EXPECT_FALSE(SquashfsFilesystem::IsSquashfsImage(bad_super_block));
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_generator/zip_unittest.cc b/payload_generator/zip_unittest.cc
index 54adfcb..c750eb7 100644
--- a/payload_generator/zip_unittest.cc
+++ b/payload_generator/zip_unittest.cc
@@ -17,10 +17,10 @@
#include <string.h>
#include <unistd.h>
+#include <memory>
#include <string>
#include <vector>
-#include <brillo/make_unique_ptr.h>
#include <gtest/gtest.h>
#include "update_engine/common/test_utils.h"
@@ -31,6 +31,7 @@
#include "update_engine/payload_generator/xz.h"
using chromeos_update_engine::test_utils::kRandomString;
+using google::protobuf::RepeatedPtrField;
using std::string;
using std::vector;
@@ -50,7 +51,7 @@
~MemoryExtentWriter() override = default;
bool Init(FileDescriptorPtr fd,
- const vector<Extent>& extents,
+ const RepeatedPtrField<Extent>& extents,
uint32_t block_size) override {
return true;
}
@@ -70,7 +71,7 @@
template <typename W>
bool DecompressWithWriter(const brillo::Blob& in, brillo::Blob* out) {
std::unique_ptr<ExtentWriter> writer(
- new W(brillo::make_unique_ptr(new MemoryExtentWriter(out))));
+ new W(std::make_unique<MemoryExtentWriter>(out)));
// Init() parameters are ignored by the testing MemoryExtentWriter.
bool ok = writer->Init(nullptr, {}, 1);
ok = writer->Write(in.data(), in.size()) && ok;
diff --git a/payload_state_unittest.cc b/payload_state_unittest.cc
index e469d2f..f1c3835 100644
--- a/payload_state_unittest.cc
+++ b/payload_state_unittest.cc
@@ -42,7 +42,7 @@
using testing::Mock;
using testing::NiceMock;
using testing::Return;
-using testing::SetArgumentPointee;
+using testing::SetArgPointee;
using testing::_;
namespace chromeos_update_engine {
@@ -611,7 +611,7 @@
EXPECT_CALL(*prefs2, GetInt64(kPrefsFullPayloadAttemptNumber, _))
.Times(AtLeast(1));
EXPECT_CALL(*prefs2, GetInt64(kPrefsCurrentUrlIndex, _))
- .WillRepeatedly(DoAll(SetArgumentPointee<1>(2), Return(true)));
+ .WillRepeatedly(DoAll(SetArgPointee<1>(2), Return(true)));
EXPECT_CALL(*prefs2, GetInt64(kPrefsCurrentUrlFailureCount, _))
.Times(AtLeast(1));
EXPECT_CALL(*prefs2, GetInt64(kPrefsUrlSwitchCount, _))
@@ -1012,7 +1012,7 @@
// Let's verify we can reload it correctly.
EXPECT_CALL(*mock_powerwash_safe_prefs, GetString(
kPrefsRollbackVersion, _)).WillOnce(DoAll(
- SetArgumentPointee<1>(rollback_version), Return(true)));
+ SetArgPointee<1>(rollback_version), Return(true)));
EXPECT_CALL(*mock_powerwash_safe_prefs, SetString(kPrefsRollbackVersion,
rollback_version));
payload_state.LoadRollbackVersion();
@@ -1245,7 +1245,7 @@
// Test with device policy not allowing http updates.
EXPECT_CALL(disable_http_policy, GetHttpDownloadsEnabled(_))
- .WillRepeatedly(DoAll(SetArgumentPointee<0>(false), Return(true)));
+ .WillRepeatedly(DoAll(SetArgPointee<0>(false), Return(true)));
// Reset state and set again.
SetupPayloadStateWith2Urls(
@@ -1274,7 +1274,7 @@
policy::MockDevicePolicy enable_http_policy;
fake_system_state.set_device_policy(&enable_http_policy);
EXPECT_CALL(enable_http_policy, GetHttpDownloadsEnabled(_))
- .WillRepeatedly(DoAll(SetArgumentPointee<0>(true), Return(true)));
+ .WillRepeatedly(DoAll(SetArgPointee<0>(true), Return(true)));
// Now, set the same response using the same hash
// so that we can test that the state is reset not because of the
diff --git a/power_manager_chromeos.cc b/power_manager_chromeos.cc
index e175f95..23fb032 100644
--- a/power_manager_chromeos.cc
+++ b/power_manager_chromeos.cc
@@ -16,6 +16,8 @@
#include "update_engine/power_manager_chromeos.h"
+#include <memory>
+
#include <power_manager/dbus-constants.h>
#include <power_manager/dbus-proxies.h>
@@ -37,7 +39,9 @@
<< ::power_manager::kRequestRestartMethod;
brillo::ErrorPtr error;
return power_manager_proxy_.RequestRestart(
- ::power_manager::REQUEST_RESTART_FOR_UPDATE, &error);
+ ::power_manager::REQUEST_RESTART_FOR_UPDATE,
+ "update_engine applying update",
+ &error);
}
} // namespace chromeos_update_engine
diff --git a/real_system_state.cc b/real_system_state.cc
index b9ffa2c..8e7ad51 100644
--- a/real_system_state.cc
+++ b/real_system_state.cc
@@ -16,17 +16,17 @@
#include "update_engine/real_system_state.h"
+#include <memory>
#include <string>
#include <base/bind.h>
#include <base/files/file_util.h>
#include <base/location.h>
#include <base/time/time.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/message_loop.h>
-#if USE_CHROME_KIOSK_APP || USE_CHROME_NETWORK_PROXY
+#if USE_CHROME_KIOSK_APP
#include <chromeos/dbus/service_constants.h>
-#endif // USE_CHROME_KIOSK_APP || USE_CHROME_NETWORK_PROXY
+#endif // USE_CHROME_KIOSK_APP
#include "update_engine/common/boot_control.h"
#include "update_engine/common/boot_control_stub.h"
@@ -57,7 +57,7 @@
if (!boot_control_) {
LOG(WARNING) << "Unable to create BootControl instance, using stub "
<< "instead. All update attempts will fail.";
- boot_control_ = brillo::make_unique_ptr(new BootControlStub());
+ boot_control_ = std::make_unique<BootControlStub>();
}
hardware_ = hardware::CreateHardware();
@@ -70,12 +70,6 @@
libcros_proxy_.reset(new org::chromium::LibCrosServiceInterfaceProxy(
DBusConnection::Get()->GetDBus(), chromeos::kLibCrosServiceName));
#endif // USE_CHROME_KIOSK_APP
-#if USE_CHROME_NETWORK_PROXY
- network_proxy_service_proxy_.reset(
- new org::chromium::NetworkProxyServiceInterfaceProxy(
- DBusConnection::Get()->GetDBus(),
- chromeos::kNetworkProxyServiceName));
-#endif // USE_CHROME_NETWORK_PROXY
LOG_IF(INFO, !hardware_->IsNormalBootMode()) << "Booted in dev mode.";
LOG_IF(INFO, !hardware_->IsOfficialBuild()) << "Booted non-official build.";
@@ -146,14 +140,8 @@
new CertificateChecker(prefs_.get(), &openssl_wrapper_));
certificate_checker_->Init();
- update_attempter_.reset(
- new UpdateAttempter(this,
- certificate_checker_.get(),
-#if USE_CHROME_NETWORK_PROXY
- network_proxy_service_proxy_.get()));
-#else
- nullptr));
-#endif // USE_CHROME_NETWORK_PROXY
+ update_attempter_.reset(new UpdateAttempter(this,
+ certificate_checker_.get()));
// Initialize the UpdateAttempter before the UpdateManager.
update_attempter_->Init();
diff --git a/real_system_state.h b/real_system_state.h
index c95458e..49f7c31 100644
--- a/real_system_state.h
+++ b/real_system_state.h
@@ -27,9 +27,6 @@
#if USE_CHROME_KIOSK_APP
#include <libcros/dbus-proxies.h>
#endif // USE_CHROME_KIOSK_APP
-#if USE_CHROME_NETWORK_PROXY
-#include <network_proxy/dbus-proxies.h>
-#endif // USE_CHROME_NETWORK_PROXY
#include "update_engine/certificate_checker.h"
#include "update_engine/common/boot_control_interface.h"
@@ -134,10 +131,6 @@
#if USE_CHROME_KIOSK_APP
std::unique_ptr<org::chromium::LibCrosServiceInterfaceProxy> libcros_proxy_;
#endif // USE_CHROME_KIOSK_APP
-#if USE_CHROME_NETWORK_PROXY
- std::unique_ptr<org::chromium::NetworkProxyServiceInterfaceProxy>
- network_proxy_service_proxy_;
-#endif // USE_CHROME_NETWORK_PROXY
// Interface for the power manager.
std::unique_ptr<PowerManagerInterface> power_manager_;
diff --git a/sample_images/generate_images.sh b/sample_images/generate_images.sh
index 6a0d1ea..8478682 100755
--- a/sample_images/generate_images.sh
+++ b/sample_images/generate_images.sh
@@ -26,12 +26,15 @@
# cleanup <path>
# Unmount and remove the mountpoint <path>
cleanup() {
- if ! sudo umount "$1" 2>/dev/null; then
- if mountpoint -q "$1"; then
- sync && sudo umount "$1"
+ local path="$1"
+ if ! sudo umount "${path}" 2>/dev/null; then
+ if mountpoint -q "${path}"; then
+ sync && sudo umount "${path}"
fi
fi
- rmdir "$1"
+ if [ -n "${path}" ]; then
+ sudo rm -rf "${path}"
+ fi
}
# add_files_default <mntdir> <block_size>
@@ -203,10 +206,11 @@
# generate_fs <filename> <kind> <size> [block_size] [block_groups]
generate_fs() {
local filename="$1"
- local kind="$2"
- local size="$3"
- local block_size="${4:-4096}"
- local block_groups="${5:-}"
+ local type="$2"
+ local kind="$3"
+ local size="$4"
+ local block_size="${5:-4096}"
+ local block_groups="${6:-}"
local mkfs_opts=( -q -F -b "${block_size}" -L "ROOT-TEST" -t ext2 )
if [[ -n "${block_groups}" ]]; then
@@ -215,16 +219,17 @@
local mntdir=$(mktemp --tmpdir -d generate_ext2.XXXXXX)
trap 'cleanup "${mntdir}"; rm -f "${filename}"' INT TERM EXIT
-
# Cleanup old image.
if [[ -e "${filename}" ]]; then
rm -f "${filename}"
fi
- truncate --size="${size}" "${filename}"
- mkfs.ext2 "${mkfs_opts[@]}" "${filename}"
- sudo mount "${filename}" "${mntdir}" -o loop
+ if [[ "${type}" == "ext2" ]]; then
+ truncate --size="${size}" "${filename}"
+ mkfs.ext2 "${mkfs_opts[@]}" "${filename}"
+ sudo mount "${filename}" "${mntdir}" -o loop
+ fi
case "${kind}" in
unittest)
add_files_ue_settings "${mntdir}" "${block_size}"
@@ -237,6 +242,10 @@
;;
esac
+ if [[ "${type}" == "sqfs" ]]; then
+ mksquashfs "${mntdir}" "${filename}"
+ fi
+
cleanup "${mntdir}"
trap - INT TERM EXIT
}
@@ -253,10 +262,14 @@
main() {
# Add more sample images here.
- generate_image disk_ext2_1k default $((1024 * 1024)) 1024
- generate_image disk_ext2_4k default $((1024 * 4096)) 4096
- generate_image disk_ext2_4k_empty empty $((1024 * 4096)) 4096
- generate_image disk_ext2_unittest unittest $((1024 * 4096)) 4096
+ generate_image disk_ext2_1k ext2 default $((1024 * 1024)) 1024
+ generate_image disk_ext2_4k ext2 default $((1024 * 4096)) 4096
+ generate_image disk_ext2_4k_empty ext2 empty $((1024 * 4096)) 4096
+ generate_image disk_ext2_unittest ext2 unittest $((1024 * 4096)) 4096
+
+ # Add squashfs sample images.
+ generate_image disk_sqfs_empty sqfs empty $((1024 * 4096)) 4096
+ generate_image disk_sqfs_default sqfs default $((1024 * 4096)) 4096
# Generate the tarball and delete temporary images.
echo "Packing tar file sample_images.tar.bz2"
diff --git a/sample_images/sample_images.tar.bz2 b/sample_images/sample_images.tar.bz2
index 72f4eb5..6215482 100644
--- a/sample_images/sample_images.tar.bz2
+++ b/sample_images/sample_images.tar.bz2
Binary files differ
diff --git a/scripts/blockdiff.py b/scripts/blockdiff.py
new file mode 100755
index 0000000..1dc60a6
--- /dev/null
+++ b/scripts/blockdiff.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python2
+#
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Block diff utility."""
+
+from __future__ import print_function
+
+import optparse
+import sys
+
+
+class BlockDiffError(Exception):
+ pass
+
+
+def BlockDiff(block_size, file1, file2, name1, name2, max_length=-1):
+ """Performs a binary diff of two files by blocks.
+
+ Args:
+ block_size: the size of a block to diff by
+ file1: first file object
+ file2: second file object
+ name1: name of first file (for error reporting)
+ name2: name of second file (for error reporting)
+ max_length: the maximum length to read/diff in bytes (optional)
+ Returns:
+ A list of (start, length) pairs representing block extents that differ
+ between the two files.
+ Raises:
+ BlockDiffError if there were errors while diffing.
+
+ """
+ if max_length < 0:
+ max_length = sys.maxint
+ diff_list = []
+ num_blocks = extent_start = extent_length = 0
+ while max_length or extent_length:
+ read_length = min(max_length, block_size)
+ data1 = file1.read(read_length)
+ data2 = file2.read(read_length)
+ if len(data1) != len(data2):
+ raise BlockDiffError('read %d bytes from %s but %d bytes from %s' %
+ (len(data1), name1, len(data2), name2))
+
+ if data1 != data2:
+ # Data is different, mark it down.
+ if extent_length:
+ # Stretch the current diff extent.
+ extent_length += 1
+ else:
+ # Start a new diff extent.
+ extent_start = num_blocks
+ extent_length = 1
+ elif extent_length:
+ # Record the previous extent.
+ diff_list.append((extent_start, extent_length))
+ extent_length = 0
+
+ # Are we done reading?
+ if not data1:
+ break
+
+ max_length -= len(data1)
+ num_blocks += 1
+
+ return diff_list
+
+
+def main(argv):
+ # Parse command-line arguments.
+ parser = optparse.OptionParser(
+ usage='Usage: %prog FILE1 FILE2',
+ description='Compare FILE1 and FILE2 by blocks.')
+
+ parser.add_option('-b', '--block-size', metavar='NUM', type=int, default=4096,
+ help='the block size to use (default: %default)')
+ parser.add_option('-m', '--max-length', metavar='NUM', type=int, default=-1,
+ help='maximum number of bytes to compared')
+
+ opts, args = parser.parse_args(argv[1:])
+
+ try:
+ name1, name2 = args
+ except ValueError:
+ parser.error('unexpected number of arguments')
+
+ # Perform the block diff.
+ try:
+ with open(name1) as file1:
+ with open(name2) as file2:
+ diff_list = BlockDiff(opts.block_size, file1, file2, name1, name2,
+ opts.max_length)
+ except BlockDiffError as e:
+ print('Error: ' % e, file=sys.stderr)
+ return 2
+
+ # Print the diff, if such was found.
+ if diff_list:
+ total_diff_blocks = 0
+ for extent_start, extent_length in diff_list:
+ total_diff_blocks += extent_length
+ print('%d->%d (%d)' %
+ (extent_start, extent_start + extent_length, extent_length))
+
+ print('total diff: %d blocks' % total_diff_blocks)
+ return 1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/scripts/paycheck.py b/scripts/paycheck.py
index 0195f53..8df1bf0 100755
--- a/scripts/paycheck.py
+++ b/scripts/paycheck.py
@@ -12,7 +12,6 @@
import os
import sys
-# pylint: disable=F0401
lib_dir = os.path.join(os.path.dirname(__file__), 'lib')
if os.path.exists(lib_dir) and os.path.isdir(lib_dir):
sys.path.insert(1, lib_dir)
@@ -91,6 +90,8 @@
'operations (not in-place)'))
trace_opts.add_option('--bspatch-path', metavar='FILE',
help=('use the specified bspatch binary'))
+ trace_opts.add_option('--puffpatch-path', metavar='FILE',
+ help=('use the specified puffpatch binary'))
parser.add_option_group(trace_opts)
trace_opts = optparse.OptionGroup(parser, 'Block tracing')
@@ -147,6 +148,8 @@
parser.error('--extract-bsdiff can only be used when applying payloads')
if opts.bspatch_path:
parser.error('--bspatch-path can only be used when applying payloads')
+ if opts.puffpatch_path:
+ parser.error('--puffpatch-path can only be used when applying payloads')
else:
parser.error('unexpected number of arguments')
@@ -215,6 +218,8 @@
dargs = {'bsdiff_in_place': not options.extract_bsdiff}
if options.bspatch_path:
dargs['bspatch_path'] = options.bspatch_path
+ if options.puffpatch_path:
+ dargs['puffpatch_path'] = options.puffpatch_path
if options.assert_type == _TYPE_DELTA:
dargs['old_kernel_part'] = extra_args[2]
dargs['old_rootfs_part'] = extra_args[3]
diff --git a/scripts/run_unittests b/scripts/run_unittests
new file mode 100755
index 0000000..c8e713d
--- /dev/null
+++ b/scripts/run_unittests
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Runs update_payload unittests
+
+set -e
+
+# Invoke unittest scripts.
+for unittest_script in update_payload/*_unittest.py; do
+ filename=$(basename "${unittest_script}")
+ python -m update_payload."${filename%.*}"
+done
+
+exit 0
diff --git a/scripts/update_payload/__init__.py b/scripts/update_payload/__init__.py
index 1906a16..e4a5588 100644
--- a/scripts/update_payload/__init__.py
+++ b/scripts/update_payload/__init__.py
@@ -5,7 +5,6 @@
"""Library for processing, verifying and applying Chrome OS update payloads."""
# Just raise the interface classes to the root namespace.
-# pylint: disable=W0401
-from checker import CHECKS_TO_DISABLE
-from error import PayloadError
-from payload import Payload
+from update_payload.checker import CHECKS_TO_DISABLE
+from update_payload.error import PayloadError
+from update_payload.payload import Payload
diff --git a/scripts/update_payload/applier.py b/scripts/update_payload/applier.py
index fa419bd..e470ac4 100644
--- a/scripts/update_payload/applier.py
+++ b/scripts/update_payload/applier.py
@@ -24,8 +24,8 @@
import sys
import tempfile
-import common
-from error import PayloadError
+from update_payload import common
+from update_payload.error import PayloadError
#
@@ -44,7 +44,6 @@
PayloadError if computed hash doesn't match expected one, or if fails to
read the specified length of data.
"""
- # pylint: disable=E1101
hasher = hashlib.sha256()
block_length = 1024 * 1024
max_length = length if length >= 0 else sys.maxint
@@ -213,7 +212,7 @@
self.minor_version = payload.manifest.minor_version
self.bsdiff_in_place = bsdiff_in_place
self.bspatch_path = bspatch_path or 'bspatch'
- self.puffpatch_path = puffpatch_path or 'imgpatch'
+ self.puffpatch_path = puffpatch_path or 'puffin'
self.truncate_to_expected_size = truncate_to_expected_size
def _ApplyReplaceOperation(self, op, op_name, out_data, part_file, part_size):
@@ -301,22 +300,27 @@
_WriteExtents(part_file, in_data, op.dst_extents, block_size,
'%s.dst_extents' % op_name)
- def _ApplyBsdiffOperation(self, op, op_name, patch_data, new_part_file):
- """Applies a BSDIFF operation.
+ def _ApplyZeroOperation(self, op, op_name, part_file):
+ """Applies a ZERO operation.
Args:
op: the operation object
op_name: name string for error reporting
- patch_data: the binary patch content
- new_part_file: the target partition file object
+ part_file: the partition file object
Raises:
PayloadError if something goes wrong.
"""
- # Implemented using a SOURCE_BSDIFF operation with the source and target
- # partition set to the new partition.
- self._ApplyDiffOperation(op, op_name, patch_data, new_part_file,
- new_part_file)
+ block_size = self.block_size
+ base_name = '%s.dst_extents' % op_name
+
+ # Iterate over the extents and write zero.
+ # pylint: disable=unused-variable
+ for ex, ex_name in common.ExtentIter(op.dst_extents, base_name):
+ # Only do actual writing if this is not a pseudo-extent.
+ if ex.start_block != common.PSEUDO_EXTENT_MARKER:
+ part_file.seek(ex.start_block * block_size)
+ part_file.write('\0' * (ex.num_blocks * block_size))
def _ApplySourceCopyOperation(self, op, op_name, old_part_file,
new_part_file):
@@ -345,9 +349,26 @@
_WriteExtents(new_part_file, in_data, op.dst_extents, block_size,
'%s.dst_extents' % op_name)
+ def _BytesInExtents(self, extents, base_name):
+ """Counts the length of extents in bytes.
+
+ Args:
+ extents: The list of Extents.
+ base_name: For error reporting.
+
+ Returns:
+ The number of bytes in extents.
+ """
+
+ length = 0
+ # pylint: disable=unused-variable
+ for ex, ex_name in common.ExtentIter(extents, base_name):
+ length += ex.num_blocks * self.block_size
+ return length
+
def _ApplyDiffOperation(self, op, op_name, patch_data, old_part_file,
new_part_file):
- """Applies a SOURCE_BSDIFF or PUFFDIFF operation.
+ """Applies a SOURCE_BSDIFF, BROTLI_BSDIFF or PUFFDIFF operation.
Args:
op: the operation object
@@ -372,24 +393,40 @@
patch_file.write(patch_data)
if (hasattr(new_part_file, 'fileno') and
- ((not old_part_file) or hasattr(old_part_file, 'fileno')) and
- op.type != common.OpType.PUFFDIFF):
+ ((not old_part_file) or hasattr(old_part_file, 'fileno'))):
# Construct input and output extents argument for bspatch.
+
in_extents_arg, _, _ = _ExtentsToBspatchArg(
op.src_extents, block_size, '%s.src_extents' % op_name,
- data_length=op.src_length)
+ data_length=op.src_length if op.src_length else
+ self._BytesInExtents(op.src_extents, "%s.src_extents"))
out_extents_arg, pad_off, pad_len = _ExtentsToBspatchArg(
op.dst_extents, block_size, '%s.dst_extents' % op_name,
- data_length=op.dst_length)
+ data_length=op.dst_length if op.dst_length else
+ self._BytesInExtents(op.dst_extents, "%s.dst_extents"))
new_file_name = '/dev/fd/%d' % new_part_file.fileno()
# Diff from source partition.
old_file_name = '/dev/fd/%d' % old_part_file.fileno()
- # Invoke bspatch on partition file with extents args.
- bspatch_cmd = [self.bspatch_path, old_file_name, new_file_name,
- patch_file_name, in_extents_arg, out_extents_arg]
- subprocess.check_call(bspatch_cmd)
+ if op.type in (common.OpType.BSDIFF, common.OpType.SOURCE_BSDIFF,
+ common.OpType.BROTLI_BSDIFF):
+ # Invoke bspatch on partition file with extents args.
+ bspatch_cmd = [self.bspatch_path, old_file_name, new_file_name,
+ patch_file_name, in_extents_arg, out_extents_arg]
+ subprocess.check_call(bspatch_cmd)
+ elif op.type == common.OpType.PUFFDIFF:
+ # Invoke puffpatch on partition file with extents args.
+ puffpatch_cmd = [self.puffpatch_path,
+ "--operation=puffpatch",
+ "--src_file=%s" % old_file_name,
+ "--dst_file=%s" % new_file_name,
+ "--patch_file=%s" % patch_file_name,
+ "--src_extents=%s" % in_extents_arg,
+ "--dst_extents=%s" % out_extents_arg]
+ subprocess.check_call(puffpatch_cmd)
+ else:
+ raise PayloadError("Unknown operation %s", op.type)
# Pad with zeros past the total output length.
if pad_len:
@@ -399,7 +436,9 @@
# Gather input raw data and write to a temp file.
input_part_file = old_part_file if old_part_file else new_part_file
in_data = _ReadExtents(input_part_file, op.src_extents, block_size,
- max_length=op.src_length)
+ max_length=op.src_length if op.src_length else
+ self._BytesInExtents(op.src_extents,
+ "%s.src_extents"))
with tempfile.NamedTemporaryFile(delete=False) as in_file:
in_file_name = in_file.name
in_file.write(in_data)
@@ -408,12 +447,22 @@
with tempfile.NamedTemporaryFile(delete=False) as out_file:
out_file_name = out_file.name
- # Invoke bspatch.
- patch_cmd = [self.bspatch_path, in_file_name, out_file_name,
- patch_file_name]
- if op.type == common.OpType.PUFFDIFF:
- patch_cmd[0] = self.puffpatch_path
- subprocess.check_call(patch_cmd)
+ if op.type in (common.OpType.BSDIFF, common.OpType.SOURCE_BSDIFF,
+ common.OpType.BROTLI_BSDIFF):
+ # Invoke bspatch.
+ bspatch_cmd = [self.bspatch_path, in_file_name, out_file_name,
+ patch_file_name]
+ subprocess.check_call(bspatch_cmd)
+ elif op.type == common.OpType.PUFFDIFF:
+ # Invoke puffpatch.
+ puffpatch_cmd = [self.puffpatch_path,
+ "--operation=puffpatch",
+ "--src_file=%s" % in_file_name,
+ "--dst_file=%s" % out_file_name,
+ "--patch_file=%s" % patch_file_name]
+ subprocess.check_call(puffpatch_cmd)
+ else:
+ raise PayloadError("Unknown operation %s", op.type)
# Read output.
with open(out_file_name, 'rb') as out_file:
@@ -463,12 +512,16 @@
self._ApplyReplaceOperation(op, op_name, data, new_part_file, part_size)
elif op.type == common.OpType.MOVE:
self._ApplyMoveOperation(op, op_name, new_part_file)
+ elif op.type == common.OpType.ZERO:
+ self._ApplyZeroOperation(op, op_name, new_part_file)
elif op.type == common.OpType.BSDIFF:
- self._ApplyBsdiffOperation(op, op_name, data, new_part_file)
+ self._ApplyDiffOperation(op, op_name, data, new_part_file,
+ new_part_file)
elif op.type == common.OpType.SOURCE_COPY:
self._ApplySourceCopyOperation(op, op_name, old_part_file,
new_part_file)
- elif op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.PUFFDIFF):
+ elif op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.PUFFDIFF,
+ common.OpType.BROTLI_BSDIFF):
self._ApplyDiffOperation(op, op_name, data, old_part_file,
new_part_file)
else:
diff --git a/scripts/update_payload/block_tracer.py b/scripts/update_payload/block_tracer.py
index f222b21..5caf7e3 100644
--- a/scripts/update_payload/block_tracer.py
+++ b/scripts/update_payload/block_tracer.py
@@ -16,7 +16,7 @@
from __future__ import print_function
-import common
+from update_payload import common
#
diff --git a/scripts/update_payload/checker.py b/scripts/update_payload/checker.py
index 3144395..e4cb845 100644
--- a/scripts/update_payload/checker.py
+++ b/scripts/update_payload/checker.py
@@ -21,11 +21,11 @@
import os
import subprocess
-import common
-import error
-import format_utils
-import histogram
-import update_metadata_pb2
+from update_payload import common
+from update_payload import error
+from update_payload import format_utils
+from update_payload import histogram
+from update_payload import update_metadata_pb2
#
@@ -815,10 +815,30 @@
if dst_extent:
raise error.PayloadError('%s: excess dst blocks.' % op_name)
- def _CheckAnyDiffOperation(self, data_length, total_dst_blocks, op_name):
- """Specific checks for BSDIFF, SOURCE_BSDIFF and PUFFDIFF operations.
+ def _CheckZeroOperation(self, op, op_name):
+ """Specific checks for ZERO operations.
Args:
+ op: The operation object from the manifest.
+ op_name: Operation name for error reporting.
+
+ Raises:
+ error.PayloadError if any check fails.
+ """
+ # Check: Does not contain src extents, data_length and data_offset.
+ if op.src_extents:
+ raise error.PayloadError('%s: contains src_extents.' % op_name)
+ if op.data_length:
+ raise error.PayloadError('%s: contains data_length.' % op_name)
+ if op.data_offset:
+ raise error.PayloadError('%s: contains data_offset.' % op_name)
+
+ def _CheckAnyDiffOperation(self, op, data_length, total_dst_blocks, op_name):
+ """Specific checks for BSDIFF, SOURCE_BSDIFF, PUFFDIFF and BROTLI_BSDIFF
+ operations.
+
+ Args:
+ op: The operation.
data_length: The length of the data blob associated with the operation.
total_dst_blocks: Total number of blocks in dst_extents.
op_name: Operation name for error reporting.
@@ -838,6 +858,15 @@
(op_name, data_length, total_dst_blocks, self.block_size,
total_dst_blocks * self.block_size))
+ # Check the existence of src_length and dst_length for legacy bsdiffs.
+ if (op.type == common.OpType.BSDIFF or
+ (op.type == common.OpType.SOURCE_BSDIFF and self.minor_version <= 3)):
+ if not op.HasField('src_length') or not op.HasField('dst_length'):
+ raise error.PayloadError('%s: require {src,dst}_length.' % op_name)
+ else:
+ if op.HasField('src_length') or op.HasField('dst_length'):
+ raise error.PayloadError('%s: unneeded {src,dst}_length.' % op_name)
+
def _CheckSourceCopyOperation(self, data_offset, total_src_blocks,
total_dst_blocks, op_name):
"""Specific checks for SOURCE_COPY.
@@ -941,8 +970,6 @@
op_name)
# Check: Hash verifies correctly.
- # pylint cannot find the method in hashlib, for some reason.
- # pylint: disable=E1101
actual_hash = hashlib.sha256(self.payload.ReadDataBlob(data_offset,
data_length))
if op.data_sha256_hash != actual_hash.digest():
@@ -972,17 +999,20 @@
elif op.type == common.OpType.MOVE and self.minor_version == 1:
self._CheckMoveOperation(op, data_offset, total_src_blocks,
total_dst_blocks, op_name)
+ elif op.type == common.OpType.ZERO and self.minor_version >= 4:
+ self._CheckZeroOperation(op, op_name)
elif op.type == common.OpType.BSDIFF and self.minor_version == 1:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
elif op.type == common.OpType.SOURCE_COPY and self.minor_version >= 2:
self._CheckSourceCopyOperation(data_offset, total_src_blocks,
total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
elif op.type == common.OpType.SOURCE_BSDIFF and self.minor_version >= 2:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
- elif op.type == common.OpType.PUFFDIFF and self.minor_version >= 4:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ elif (op.type in (common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF) and
+ self.minor_version >= 4):
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
else:
raise error.PayloadError(
@@ -1011,8 +1041,8 @@
itertools.repeat(0, self._SizeToNumBlocks(total_size)))
def _CheckOperations(self, operations, report, base_name, old_fs_size,
- new_fs_size, new_usable_size, prev_data_offset,
- allow_signature):
+ new_fs_size, old_usable_size, new_usable_size,
+ prev_data_offset, allow_signature):
"""Checks a sequence of update operations.
Args:
@@ -1021,6 +1051,7 @@
base_name: The name of the operation block.
old_fs_size: The old filesystem size in bytes.
new_fs_size: The new filesystem size in bytes.
+ old_usable_size: The overall usable size of the old partition in bytes.
new_usable_size: The overall usable size of the new partition in bytes.
prev_data_offset: Offset of last used data bytes.
allow_signature: Whether this sequence may contain signature operations.
@@ -1038,10 +1069,12 @@
common.OpType.REPLACE: 0,
common.OpType.REPLACE_BZ: 0,
common.OpType.MOVE: 0,
+ common.OpType.ZERO: 0,
common.OpType.BSDIFF: 0,
common.OpType.SOURCE_COPY: 0,
common.OpType.SOURCE_BSDIFF: 0,
common.OpType.PUFFDIFF: 0,
+ common.OpType.BROTLI_BSDIFF: 0,
}
# Total blob sizes for each operation type.
op_blob_totals = {
@@ -1052,6 +1085,7 @@
# SOURCE_COPY operations don't have blobs.
common.OpType.SOURCE_BSDIFF: 0,
common.OpType.PUFFDIFF: 0,
+ common.OpType.BROTLI_BSDIFF: 0,
}
# Counts of hashed vs unhashed operations.
blob_hash_counts = {
@@ -1062,7 +1096,7 @@
blob_hash_counts['signature'] = 0
# Allocate old and new block counters.
- old_block_counters = (self._AllocBlockCounters(new_usable_size)
+ old_block_counters = (self._AllocBlockCounters(old_usable_size)
if old_fs_size else None)
new_block_counters = self._AllocBlockCounters(new_usable_size)
@@ -1079,7 +1113,7 @@
is_last = op_num == len(operations)
curr_data_used = self._CheckOperation(
op, op_name, is_last, old_block_counters, new_block_counters,
- new_usable_size if old_fs_size else 0, new_usable_size,
+ old_usable_size, new_usable_size,
prev_data_offset + total_data_used, allow_signature,
blob_hash_counts)
if curr_data_used:
@@ -1132,8 +1166,6 @@
report.AddSection('signatures')
# Check: At least one signature present.
- # pylint cannot see through the protobuf object, it seems.
- # pylint: disable=E1101
if not sigs.signatures:
raise error.PayloadError('Signature block is empty.')
@@ -1229,10 +1261,13 @@
# exceed the filesystem size when moving data blocks around.
# - Otherwise, use the encoded filesystem size.
new_rootfs_usable_size = self.new_rootfs_fs_size
+ old_rootfs_usable_size = self.old_rootfs_fs_size
if rootfs_part_size:
new_rootfs_usable_size = rootfs_part_size
+ old_rootfs_usable_size = rootfs_part_size
elif self.payload_type == _TYPE_DELTA and self.minor_version in (None, 1):
new_rootfs_usable_size = _OLD_DELTA_USABLE_PART_SIZE
+ old_rootfs_usable_size = _OLD_DELTA_USABLE_PART_SIZE
# Part 3: Examine rootfs operations.
# TODO(garnold)(chromium:243559) only default to the filesystem size if
@@ -1242,7 +1277,8 @@
total_blob_size = self._CheckOperations(
self.payload.manifest.install_operations, report,
'install_operations', self.old_rootfs_fs_size,
- self.new_rootfs_fs_size, new_rootfs_usable_size, 0, False)
+ self.new_rootfs_fs_size, old_rootfs_usable_size,
+ new_rootfs_usable_size, 0, False)
# Part 4: Examine kernel operations.
# TODO(garnold)(chromium:243559) as above.
@@ -1251,6 +1287,7 @@
self.payload.manifest.kernel_install_operations, report,
'kernel_install_operations', self.old_kernel_fs_size,
self.new_kernel_fs_size,
+ kernel_part_size if kernel_part_size else self.old_kernel_fs_size,
kernel_part_size if kernel_part_size else self.new_kernel_fs_size,
total_blob_size, True)
diff --git a/scripts/update_payload/checker_unittest.py b/scripts/update_payload/checker_unittest.py
index a0a5056..974519d 100755
--- a/scripts/update_payload/checker_unittest.py
+++ b/scripts/update_payload/checker_unittest.py
@@ -20,14 +20,16 @@
# pylint: disable=F0401
import mox
-import checker
-import common
-import payload as update_payload # Avoid name conflicts later.
-import test_utils
-import update_metadata_pb2
+from update_payload import checker
+from update_payload import common
+from update_payload import test_utils
+from update_payload import update_metadata_pb2
+from update_payload.error import PayloadError
+from update_payload.payload import Payload # Avoid name conflicts later.
def _OpTypeByName(op_name):
+ """Returns the type of an operation from itsname."""
op_name_to_type = {
'REPLACE': common.OpType.REPLACE,
'REPLACE_BZ': common.OpType.REPLACE_BZ,
@@ -39,6 +41,7 @@
'DISCARD': common.OpType.DISCARD,
'REPLACE_XZ': common.OpType.REPLACE_XZ,
'PUFFDIFF': common.OpType.PUFFDIFF,
+ 'BROTLI_BSDIFF': common.OpType.BROTLI_BSDIFF,
}
return op_name_to_type[op_name]
@@ -54,7 +57,7 @@
payload_file = cStringIO.StringIO()
payload_gen_write_to_file_func(payload_file, **payload_gen_dargs)
payload_file.seek(0)
- payload = update_payload.Payload(payload_file)
+ payload = Payload(payload_file)
payload.Init()
return checker.PayloadChecker(payload, **checker_init_dargs)
@@ -64,7 +67,7 @@
payload_file = cStringIO.StringIO()
payload_gen.WriteToFile(payload_file)
payload_file.seek(0)
- payload = update_payload.Payload(payload_file)
+ payload = Payload(payload_file)
payload.Init()
return checker.PayloadChecker(payload)
@@ -90,7 +93,7 @@
def MockPayload(self):
"""Create a mock payload object, complete with a mock manifest."""
- payload = self.mox.CreateMock(update_payload.Payload)
+ payload = self.mox.CreateMock(Payload)
payload.is_init = True
payload.manifest = self.mox.CreateMock(
update_metadata_pb2.DeltaArchiveManifest)
@@ -194,7 +197,7 @@
args = (msg, name, report, is_mandatory, is_submsg)
kwargs = {'convert': convert, 'linebreak': linebreak, 'indent': indent}
if is_mandatory and not is_present:
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckElem, *args, **kwargs)
else:
ret_val, ret_subreport = checker.PayloadChecker._CheckElem(*args,
@@ -228,8 +231,7 @@
# Test the method call.
if is_mandatory and not is_present:
- self.assertRaises(update_payload.PayloadError, tested_func, *args,
- **kwargs)
+ self.assertRaises(PayloadError, tested_func, *args, **kwargs)
else:
ret_val = tested_func(*args, **kwargs)
self.assertEquals(val if is_present else None, ret_val)
@@ -253,7 +255,7 @@
# Test the method call.
if is_mandatory and not is_present:
- self.assertRaises(update_payload.PayloadError, tested_func, *args)
+ self.assertRaises(PayloadError, tested_func, *args)
else:
ret_val, ret_subreport = tested_func(*args)
self.assertEquals(val if is_present else None, ret_val)
@@ -265,11 +267,9 @@
None, None, 'foo', 'bar', 'baz'))
self.assertIsNone(checker.PayloadChecker._CheckPresentIff(
'a', 'b', 'foo', 'bar', 'baz'))
- self.assertRaises(update_payload.PayloadError,
- checker.PayloadChecker._CheckPresentIff,
+ self.assertRaises(PayloadError, checker.PayloadChecker._CheckPresentIff,
'a', None, 'foo', 'bar', 'baz')
- self.assertRaises(update_payload.PayloadError,
- checker.PayloadChecker._CheckPresentIff,
+ self.assertRaises(PayloadError, checker.PayloadChecker._CheckPresentIff,
None, 'b', 'foo', 'bar', 'baz')
def DoCheckSha256SignatureTest(self, expect_pass, expect_subprocess_call,
@@ -298,7 +298,7 @@
self.assertIsNone(checker.PayloadChecker._CheckSha256Signature(
sig_data, 'foo', expected_signed_hash, 'bar'))
else:
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckSha256Signature,
sig_data, 'foo', expected_signed_hash, 'bar')
finally:
@@ -358,31 +358,31 @@
def testCheckBlocksFitLength_TooManyBlocks(self):
"""Tests _CheckBlocksFitLength(); fails due to excess blocks."""
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
64, 5, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
60, 5, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
49, 5, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
48, 4, 16, 'foo')
def testCheckBlocksFitLength_TooFewBlocks(self):
"""Tests _CheckBlocksFitLength(); fails due to insufficient blocks."""
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
64, 3, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
60, 3, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
49, 3, 16, 'foo')
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
checker.PayloadChecker._CheckBlocksFitLength,
48, 2, 16, 'foo')
@@ -475,8 +475,7 @@
fail_old_rootfs_fs_size or fail_new_kernel_fs_size or
fail_new_rootfs_fs_size)
if should_fail:
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckManifest, report,
+ self.assertRaises(PayloadError, payload_checker._CheckManifest, report,
rootfs_part_size, kernel_part_size)
else:
self.assertIsNone(payload_checker._CheckManifest(report,
@@ -492,12 +491,10 @@
self.assertIsNone(payload_checker._CheckLength(
int(3.5 * block_size), 4, 'foo', 'bar'))
# Fails, too few blocks.
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckLength,
+ self.assertRaises(PayloadError, payload_checker._CheckLength,
int(3.5 * block_size), 3, 'foo', 'bar')
# Fails, too many blocks.
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckLength,
+ self.assertRaises(PayloadError, payload_checker._CheckLength,
int(3.5 * block_size), 5, 'foo', 'bar')
def testCheckExtents(self):
@@ -532,30 +529,26 @@
# Fails, extent missing a start block.
extents = self.NewExtentList((-1, 4), (8, 3), (1024, 16))
self.assertRaises(
- update_payload.PayloadError, payload_checker._CheckExtents,
- extents, (1024 + 16) * block_size, collections.defaultdict(int),
- 'foo')
+ PayloadError, payload_checker._CheckExtents, extents,
+ (1024 + 16) * block_size, collections.defaultdict(int), 'foo')
# Fails, extent missing block count.
extents = self.NewExtentList((0, -1), (8, 3), (1024, 16))
self.assertRaises(
- update_payload.PayloadError, payload_checker._CheckExtents,
- extents, (1024 + 16) * block_size, collections.defaultdict(int),
- 'foo')
+ PayloadError, payload_checker._CheckExtents, extents,
+ (1024 + 16) * block_size, collections.defaultdict(int), 'foo')
# Fails, extent has zero blocks.
extents = self.NewExtentList((0, 4), (8, 3), (1024, 0))
self.assertRaises(
- update_payload.PayloadError, payload_checker._CheckExtents,
- extents, (1024 + 16) * block_size, collections.defaultdict(int),
- 'foo')
+ PayloadError, payload_checker._CheckExtents, extents,
+ (1024 + 16) * block_size, collections.defaultdict(int), 'foo')
# Fails, extent exceeds partition boundaries.
extents = self.NewExtentList((0, 4), (8, 3), (1024, 16))
self.assertRaises(
- update_payload.PayloadError, payload_checker._CheckExtents,
- extents, (1024 + 15) * block_size, collections.defaultdict(int),
- 'foo')
+ PayloadError, payload_checker._CheckExtents, extents,
+ (1024 + 15) * block_size, collections.defaultdict(int), 'foo')
def testCheckReplaceOperation(self):
"""Tests _CheckReplaceOperation() where op.type == REPLACE."""
@@ -577,22 +570,19 @@
# Fail, src extents founds.
op.src_extents = ['bar']
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size, 'foo')
# Fail, missing data.
op.src_extents = []
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, None, (data_length + block_size - 1) / block_size, 'foo')
# Fail, length / block number mismatch.
op.src_extents = ['bar']
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size + 1, 'foo')
def testCheckReplaceBzOperation(self):
@@ -615,22 +605,19 @@
# Fail, src extents founds.
op.src_extents = ['bar']
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size + 5, 'foo')
# Fail, missing data.
op.src_extents = []
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, None, (data_length + block_size - 1) / block_size, 'foo')
# Fail, too few blocks to justify BZ.
op.src_extents = []
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckReplaceOperation,
+ PayloadError, payload_checker._CheckReplaceOperation,
op, data_length, (data_length + block_size - 1) / block_size, 'foo')
def testCheckMoveOperation_Pass(self):
@@ -657,8 +644,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, 1024, 134, 134, 'foo')
def testCheckMoveOperation_FailInsufficientSrcBlocks(self):
@@ -672,8 +658,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckMoveOperation_FailInsufficientDstBlocks(self):
@@ -687,8 +672,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 5)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckMoveOperation_FailExcessSrcBlocks(self):
@@ -702,16 +686,14 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 5)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
self.AddToMessage(op.src_extents,
self.NewExtentList((1, 4), (12, 2), (1024, 129)))
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckMoveOperation_FailExcessDstBlocks(self):
@@ -725,8 +707,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((16, 128), (512, 7)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckMoveOperation_FailStagnantBlocks(self):
@@ -740,8 +721,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((8, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckMoveOperation_FailZeroStartBlock(self):
@@ -755,8 +735,7 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((8, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
self.AddToMessage(op.src_extents,
@@ -764,29 +743,27 @@
self.AddToMessage(op.dst_extents,
self.NewExtentList((0, 128), (512, 6)))
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckMoveOperation,
+ PayloadError, payload_checker._CheckMoveOperation,
op, None, 134, 134, 'foo')
def testCheckAnyDiff(self):
"""Tests _CheckAnyDiffOperation()."""
payload_checker = checker.PayloadChecker(self.MockPayload())
+ op = update_metadata_pb2.InstallOperation()
# Pass.
self.assertIsNone(
- payload_checker._CheckAnyDiffOperation(10000, 3, 'foo'))
+ payload_checker._CheckAnyDiffOperation(op, 10000, 3, 'foo'))
# Fail, missing data blob.
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckAnyDiffOperation,
- None, 3, 'foo')
+ PayloadError, payload_checker._CheckAnyDiffOperation,
+ op, None, 3, 'foo')
# Fail, too big of a diff blob (unjustified).
self.assertRaises(
- update_payload.PayloadError,
- payload_checker._CheckAnyDiffOperation,
- 10000, 2, 'foo')
+ PayloadError, payload_checker._CheckAnyDiffOperation,
+ op, 10000, 2, 'foo')
def testCheckSourceCopyOperation_Pass(self):
"""Tests _CheckSourceCopyOperation(); pass case."""
@@ -797,15 +774,13 @@
def testCheckSourceCopyOperation_FailContainsData(self):
"""Tests _CheckSourceCopyOperation(); message contains data."""
payload_checker = checker.PayloadChecker(self.MockPayload())
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckSourceCopyOperation,
+ self.assertRaises(PayloadError, payload_checker._CheckSourceCopyOperation,
134, 0, 0, 'foo')
def testCheckSourceCopyOperation_FailBlockCountsMismatch(self):
"""Tests _CheckSourceCopyOperation(); src and dst block totals not equal."""
payload_checker = checker.PayloadChecker(self.MockPayload())
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckSourceCopyOperation,
+ self.assertRaises(PayloadError, payload_checker._CheckSourceCopyOperation,
None, 0, 1, 'foo')
def DoCheckOperationTest(self, op_type_name, is_last, allow_signature,
@@ -818,7 +793,7 @@
Args:
op_type_name: 'REPLACE', 'REPLACE_BZ', 'MOVE', 'BSDIFF', 'SOURCE_COPY',
- or 'SOURCE_BSDIFF'.
+ 'SOURCE_BSDIFF', BROTLI_BSDIFF or 'PUFFDIFF'.
is_last: Whether we're testing the last operation in a sequence.
allow_signature: Whether we're testing a signature-capable operation.
allow_unhashed: Whether we're allowing to not hash the data.
@@ -857,7 +832,8 @@
total_src_blocks = 0
if op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
- common.OpType.SOURCE_COPY, common.OpType.SOURCE_BSDIFF):
+ common.OpType.SOURCE_COPY, common.OpType.SOURCE_BSDIFF,
+ common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF):
if fail_src_extents:
self.AddToMessage(op.src_extents,
self.NewExtentList((1, 0)))
@@ -872,6 +848,9 @@
payload_checker.minor_version = 2 if fail_bad_minor_version else 1
elif op_type in (common.OpType.SOURCE_COPY, common.OpType.SOURCE_BSDIFF):
payload_checker.minor_version = 1 if fail_bad_minor_version else 2
+ elif op_type in (common.OpType.ZERO, common.OpType.DISCARD,
+ common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF):
+ payload_checker.minor_version = 3 if fail_bad_minor_version else 4
if op_type not in (common.OpType.MOVE, common.OpType.SOURCE_COPY):
if not fail_mismatched_data_offset_length:
@@ -889,6 +868,7 @@
op.data_sha256_hash = hashlib.sha256(fake_data).digest()
payload.ReadDataBlob(op.data_offset, op.data_length).AndReturn(
fake_data)
+
elif fail_data_hash:
# Create an invalid data blob hash.
op.data_sha256_hash = hashlib.sha256(
@@ -909,7 +889,9 @@
if total_src_blocks:
if fail_src_length:
op.src_length = total_src_blocks * block_size + 8
- else:
+ elif (op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
+ common.OpType.SOURCE_BSDIFF) and
+ payload_checker.minor_version <= 3):
op.src_length = total_src_blocks * block_size
elif fail_src_length:
# Add an orphaned src_length.
@@ -918,7 +900,9 @@
if total_dst_blocks:
if fail_dst_length:
op.dst_length = total_dst_blocks * block_size + 8
- else:
+ elif (op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
+ common.OpType.SOURCE_BSDIFF) and
+ payload_checker.minor_version <= 3):
op.dst_length = total_dst_blocks * block_size
self.mox.ReplayAll()
@@ -931,8 +915,7 @@
old_part_size, new_part_size, prev_data_offset, allow_signature,
blob_hash_counts)
if should_fail:
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckOperation, *args)
+ self.assertRaises(PayloadError, payload_checker._CheckOperation, *args)
else:
self.assertEqual(op.data_length if op.HasField('data_length') else 0,
payload_checker._CheckOperation(*args))
@@ -954,6 +937,7 @@
self.assertEqual(17, len(result))
def DoCheckOperationsTest(self, fail_nonexhaustive_full_update):
+ """Tests _CheckOperations()."""
# Generate a test payload. For this test, we only care about one
# (arbitrary) set of operations, so we'll only be generating kernel and
# test with them.
@@ -982,11 +966,10 @@
payload_checker.payload_type = checker._TYPE_FULL
report = checker._PayloadReport()
- args = (payload_checker.payload.manifest.install_operations, report,
- 'foo', 0, rootfs_part_size, rootfs_part_size, 0, False)
+ args = (payload_checker.payload.manifest.install_operations, report, 'foo',
+ 0, rootfs_part_size, rootfs_part_size, rootfs_part_size, 0, False)
if fail_nonexhaustive_full_update:
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckOperations, *args)
+ self.assertRaises(PayloadError, payload_checker._CheckOperations, *args)
else:
self.assertEqual(rootfs_data_length,
payload_checker._CheckOperations(*args))
@@ -994,6 +977,7 @@
def DoCheckSignaturesTest(self, fail_empty_sigs_blob, fail_missing_pseudo_op,
fail_mismatched_pseudo_op, fail_sig_missing_fields,
fail_unknown_sig_version, fail_incorrect_sig):
+ """Tests _CheckSignatures()."""
# Generate a test payload. For this test, we only care about the signature
# block and how it relates to the payload hash. Therefore, we're generating
# a random (otherwise useless) payload for this purpose.
@@ -1058,8 +1042,7 @@
fail_unknown_sig_version or fail_incorrect_sig)
args = (report, test_utils._PUBKEY_FILE_NAME)
if should_fail:
- self.assertRaises(update_payload.PayloadError,
- payload_checker._CheckSignatures, *args)
+ self.assertRaises(PayloadError, payload_checker._CheckSignatures, *args)
else:
self.assertIsNone(payload_checker._CheckSignatures(*args))
@@ -1088,7 +1071,7 @@
if should_succeed:
self.assertIsNone(payload_checker._CheckManifestMinorVersion(*args))
else:
- self.assertRaises(update_payload.PayloadError,
+ self.assertRaises(PayloadError,
payload_checker._CheckManifestMinorVersion, *args)
def DoRunTest(self, rootfs_part_size_provided, kernel_part_size_provided,
@@ -1096,6 +1079,7 @@
fail_mismatched_block_size, fail_excess_data,
fail_rootfs_part_size_exceeded,
fail_kernel_part_size_exceeded):
+ """Tests Run()."""
# Generate a test payload. For this test, we generate a full update that
# has sample kernel and rootfs operations. Since most testing is done with
# internal PayloadChecker methods that are tested elsewhere, here we only
@@ -1153,7 +1137,7 @@
'assert_type': 'delta' if fail_wrong_payload_type else 'full',
'block_size': use_block_size}}
if fail_invalid_block_size:
- self.assertRaises(update_payload.PayloadError, _GetPayloadChecker,
+ self.assertRaises(PayloadError, _GetPayloadChecker,
payload_gen.WriteToFileWithData, **kwargs)
else:
payload_checker = _GetPayloadChecker(payload_gen.WriteToFileWithData,
@@ -1167,8 +1151,7 @@
fail_rootfs_part_size_exceeded or
fail_kernel_part_size_exceeded)
if should_fail:
- self.assertRaises(update_payload.PayloadError, payload_checker.Run,
- **kwargs)
+ self.assertRaises(PayloadError, payload_checker.Run, **kwargs)
else:
self.assertIsNone(payload_checker.Run(**kwargs))
@@ -1275,7 +1258,8 @@
AddParametricTests('CheckOperation',
{'op_type_name': ('REPLACE', 'REPLACE_BZ', 'MOVE',
'BSDIFF', 'SOURCE_COPY',
- 'SOURCE_BSDIFF'),
+ 'SOURCE_BSDIFF', 'PUFFDIFF',
+ 'BROTLI_BSDIFF'),
'is_last': (True, False),
'allow_signature': (True, False),
'allow_unhashed': (True, False),
diff --git a/scripts/update_payload/common.py b/scripts/update_payload/common.py
index bab8a4f..231c504 100644
--- a/scripts/update_payload/common.py
+++ b/scripts/update_payload/common.py
@@ -6,8 +6,8 @@
from __future__ import print_function
-from error import PayloadError
-import update_metadata_pb2
+from update_payload import update_metadata_pb2
+from update_payload.error import PayloadError
#
@@ -35,7 +35,6 @@
class OpType(object):
"""Container for operation type constants."""
_CLASS = update_metadata_pb2.InstallOperation
- # pylint: disable=E1101
REPLACE = _CLASS.REPLACE
REPLACE_BZ = _CLASS.REPLACE_BZ
MOVE = _CLASS.MOVE
@@ -46,8 +45,9 @@
DISCARD = _CLASS.DISCARD
REPLACE_XZ = _CLASS.REPLACE_XZ
PUFFDIFF = _CLASS.PUFFDIFF
+ BROTLI_BSDIFF = _CLASS.BROTLI_BSDIFF
ALL = (REPLACE, REPLACE_BZ, MOVE, BSDIFF, SOURCE_COPY, SOURCE_BSDIFF, ZERO,
- DISCARD, REPLACE_XZ, PUFFDIFF)
+ DISCARD, REPLACE_XZ, PUFFDIFF, BROTLI_BSDIFF)
NAMES = {
REPLACE: 'REPLACE',
REPLACE_BZ: 'REPLACE_BZ',
@@ -59,6 +59,7 @@
DISCARD: 'DISCARD',
REPLACE_XZ: 'REPLACE_XZ',
PUFFDIFF: 'PUFFDIFF',
+ BROTLI_BSDIFF: 'BROTLI_BSDIFF',
}
def __init__(self):
diff --git a/scripts/update_payload/format_utils_unittest.py b/scripts/update_payload/format_utils_unittest.py
index 8c5ba8e..7153f9e 100755
--- a/scripts/update_payload/format_utils_unittest.py
+++ b/scripts/update_payload/format_utils_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
#
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,10 +8,11 @@
import unittest
-import format_utils
+from update_payload import format_utils
class NumToPercentTest(unittest.TestCase):
+ """ Tests number conversion to percentage format."""
def testHundredPercent(self):
self.assertEqual(format_utils.NumToPercent(1, 1), '100%')
@@ -43,6 +44,7 @@
class BytesToHumanReadableTest(unittest.TestCase):
+ """ Tests number conversion to human readable format."""
def testBaseTwo(self):
self.assertEqual(format_utils.BytesToHumanReadable(0x1000), '4 KiB')
self.assertEqual(format_utils.BytesToHumanReadable(0x400000), '4 MiB')
diff --git a/scripts/update_payload/histogram.py b/scripts/update_payload/histogram.py
index 9916329..f72db61 100644
--- a/scripts/update_payload/histogram.py
+++ b/scripts/update_payload/histogram.py
@@ -6,7 +6,7 @@
from collections import defaultdict
-import format_utils
+from update_payload import format_utils
class Histogram(object):
diff --git a/scripts/update_payload/histogram_unittest.py b/scripts/update_payload/histogram_unittest.py
index 421ff20..643bb32 100755
--- a/scripts/update_payload/histogram_unittest.py
+++ b/scripts/update_payload/histogram_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
#
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,11 +8,12 @@
import unittest
-import format_utils
-import histogram
+from update_payload import format_utils
+from update_payload import histogram
class HistogramTest(unittest.TestCase):
+ """ Tests histogram"""
@staticmethod
def AddHumanReadableSize(size):
diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py
index 184f805..8d9a20e 100644
--- a/scripts/update_payload/payload.py
+++ b/scripts/update_payload/payload.py
@@ -13,8 +13,8 @@
from update_payload import block_tracer
from update_payload import checker
from update_payload import common
-from update_payload.error import PayloadError
from update_payload import update_metadata_pb2
+from update_payload.error import PayloadError
#
@@ -193,8 +193,6 @@
if self.is_init:
raise PayloadError('payload object already initialized')
- # Initialize hash context.
- # pylint: disable=E1101
self.manifest_hasher = hashlib.sha256()
# Read the file header.
@@ -237,11 +235,9 @@
_DisplayIndentedValue('Build version', image_info.build_version)
if self.manifest.HasField('old_image_info'):
- # pylint: disable=E1101
_DescribeImageInfo('Old Image', self.manifest.old_image_info)
if self.manifest.HasField('new_image_info'):
- # pylint: disable=E1101
_DescribeImageInfo('New Image', self.manifest.new_image_info)
def _AssertInit(self):
@@ -299,7 +295,7 @@
def Apply(self, new_kernel_part, new_rootfs_part, old_kernel_part=None,
old_rootfs_part=None, bsdiff_in_place=True, bspatch_path=None,
- truncate_to_expected_size=True):
+ puffpatch_path=None, truncate_to_expected_size=True):
"""Applies the update payload.
Args:
@@ -309,6 +305,7 @@
old_rootfs_part: name of source rootfs partition file (optional)
bsdiff_in_place: whether to perform BSDIFF operations in-place (optional)
bspatch_path: path to the bspatch binary (optional)
+ puffpatch_path: path to the puffpatch binary (optional)
truncate_to_expected_size: whether to truncate the resulting partitions
to their expected sizes, as specified in the
payload (optional)
@@ -321,6 +318,7 @@
# Create a short-lived payload applier object and run it.
helper = applier.PayloadApplier(
self, bsdiff_in_place=bsdiff_in_place, bspatch_path=bspatch_path,
+ puffpatch_path=puffpatch_path,
truncate_to_expected_size=truncate_to_expected_size)
helper.Run(new_kernel_part, new_rootfs_part,
old_kernel_part=old_kernel_part,
diff --git a/scripts/update_payload/test_utils.py b/scripts/update_payload/test_utils.py
index 61a91f5..38712fb 100644
--- a/scripts/update_payload/test_utils.py
+++ b/scripts/update_payload/test_utils.py
@@ -12,9 +12,9 @@
import struct
import subprocess
-import common
-import payload
-import update_metadata_pb2
+from update_payload import common
+from update_payload import payload
+from update_payload import update_metadata_pb2
class TestError(Exception):
@@ -84,7 +84,6 @@
Raises:
TestError if something goes wrong.
"""
- # pylint: disable=E1101
data_sha256_hash = common.SIG_ASN1_HEADER + hashlib.sha256(data).digest()
sign_cmd = ['openssl', 'rsautl', '-sign', '-inkey', privkey_file_name]
try:
@@ -110,8 +109,6 @@
version: signature version (None means do not assign)
data: signature binary data (None means do not assign)
"""
- # Pylint fails to identify a member of the Signatures message.
- # pylint: disable=E1101
sig = self.sigs.signatures.add()
if version is not None:
sig.version = version
@@ -174,11 +171,9 @@
part_hash: the partition hash
"""
if is_kernel:
- # pylint: disable=E1101
part_info = (self.manifest.new_kernel_info if is_new
else self.manifest.old_kernel_info)
else:
- # pylint: disable=E1101
part_info = (self.manifest.new_rootfs_info if is_new
else self.manifest.old_rootfs_info)
_SetMsgField(part_info, 'size', part_size)
@@ -188,7 +183,6 @@
data_length=None, src_extents=None, src_length=None,
dst_extents=None, dst_length=None, data_sha256_hash=None):
"""Adds an InstallOperation entry."""
- # pylint: disable=E1101
operations = (self.manifest.kernel_install_operations if is_kernel
else self.manifest.install_operations)
@@ -293,7 +287,6 @@
data_offset = data_length = data_sha256_hash = None
if data_blob is not None:
if do_hash_data_blob:
- # pylint: disable=E1101
data_sha256_hash = hashlib.sha256(data_blob).digest()
data_length, data_offset = self.AddData(data_blob)
diff --git a/scripts/update_payload/update_metadata_pb2.py b/scripts/update_payload/update_metadata_pb2.py
index 8cf87e3..595f2f6 100644
--- a/scripts/update_payload/update_metadata_pb2.py
+++ b/scripts/update_payload/update_metadata_pb2.py
@@ -13,7 +13,7 @@
DESCRIPTOR = _descriptor.FileDescriptor(
name='update_metadata.proto',
package='chromeos_update_engine',
- serialized_pb='\n\x15update_metadata.proto\x12\x16\x63hromeos_update_engine\"1\n\x06\x45xtent\x12\x13\n\x0bstart_block\x18\x01 \x01(\x04\x12\x12\n\nnum_blocks\x18\x02 \x01(\x04\"z\n\nSignatures\x12@\n\nsignatures\x18\x01 \x03(\x0b\x32,.chromeos_update_engine.Signatures.Signature\x1a*\n\tSignature\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"+\n\rPartitionInfo\x12\x0c\n\x04size\x18\x01 \x01(\x04\x12\x0c\n\x04hash\x18\x02 \x01(\x0c\"w\n\tImageInfo\x12\r\n\x05\x62oard\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12\x15\n\rbuild_channel\x18\x05 \x01(\t\x12\x15\n\rbuild_version\x18\x06 \x01(\t\"\xd3\x03\n\x10InstallOperation\x12;\n\x04type\x18\x01 \x02(\x0e\x32-.chromeos_update_engine.InstallOperation.Type\x12\x13\n\x0b\x64\x61ta_offset\x18\x02 \x01(\r\x12\x13\n\x0b\x64\x61ta_length\x18\x03 \x01(\r\x12\x33\n\x0bsrc_extents\x18\x04 \x03(\x0b\x32\x1e.chromeos_update_engine.Extent\x12\x12\n\nsrc_length\x18\x05 \x01(\x04\x12\x33\n\x0b\x64st_extents\x18\x06 \x03(\x0b\x32\x1e.chromeos_update_engine.Extent\x12\x12\n\ndst_length\x18\x07 \x01(\x04\x12\x18\n\x10\x64\x61ta_sha256_hash\x18\x08 \x01(\x0c\x12\x17\n\x0fsrc_sha256_hash\x18\t \x01(\x0c\"\x92\x01\n\x04Type\x12\x0b\n\x07REPLACE\x10\x00\x12\x0e\n\nREPLACE_BZ\x10\x01\x12\x08\n\x04MOVE\x10\x02\x12\n\n\x06\x42SDIFF\x10\x03\x12\x0f\n\x0bSOURCE_COPY\x10\x04\x12\x11\n\rSOURCE_BSDIFF\x10\x05\x12\x08\n\x04ZERO\x10\x06\x12\x0b\n\x07\x44ISCARD\x10\x07\x12\x0e\n\nREPLACE_XZ\x10\x08\x12\x0c\n\x08PUFFDIFF\x10\t\"\xa6\x03\n\x0fPartitionUpdate\x12\x16\n\x0epartition_name\x18\x01 \x02(\t\x12\x17\n\x0frun_postinstall\x18\x02 \x01(\x08\x12\x18\n\x10postinstall_path\x18\x03 \x01(\t\x12\x17\n\x0f\x66ilesystem_type\x18\x04 \x01(\t\x12M\n\x17new_partition_signature\x18\x05 \x03(\x0b\x32,.chromeos_update_engine.Signatures.Signature\x12\x41\n\x12old_partition_info\x18\x06 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12\x41\n\x12new_partition_info\x18\x07 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12<\n\noperations\x18\x08 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12\x1c\n\x14postinstall_optional\x18\t \x01(\x08\"\xc4\x05\n\x14\x44\x65ltaArchiveManifest\x12\x44\n\x12install_operations\x18\x01 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12K\n\x19kernel_install_operations\x18\x02 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12\x18\n\nblock_size\x18\x03 \x01(\r:\x04\x34\x30\x39\x36\x12\x19\n\x11signatures_offset\x18\x04 \x01(\x04\x12\x17\n\x0fsignatures_size\x18\x05 \x01(\x04\x12>\n\x0fold_kernel_info\x18\x06 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fnew_kernel_info\x18\x07 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fold_rootfs_info\x18\x08 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fnew_rootfs_info\x18\t \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12\x39\n\x0eold_image_info\x18\n \x01(\x0b\x32!.chromeos_update_engine.ImageInfo\x12\x39\n\x0enew_image_info\x18\x0b \x01(\x0b\x32!.chromeos_update_engine.ImageInfo\x12\x18\n\rminor_version\x18\x0c \x01(\r:\x01\x30\x12;\n\npartitions\x18\r \x03(\x0b\x32\'.chromeos_update_engine.PartitionUpdateB\x02H\x03')
+ serialized_pb='\n\x15update_metadata.proto\x12\x16\x63hromeos_update_engine\"1\n\x06\x45xtent\x12\x13\n\x0bstart_block\x18\x01 \x01(\x04\x12\x12\n\nnum_blocks\x18\x02 \x01(\x04\"z\n\nSignatures\x12@\n\nsignatures\x18\x01 \x03(\x0b\x32,.chromeos_update_engine.Signatures.Signature\x1a*\n\tSignature\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"+\n\rPartitionInfo\x12\x0c\n\x04size\x18\x01 \x01(\x04\x12\x0c\n\x04hash\x18\x02 \x01(\x0c\"w\n\tImageInfo\x12\r\n\x05\x62oard\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12\x15\n\rbuild_channel\x18\x05 \x01(\t\x12\x15\n\rbuild_version\x18\x06 \x01(\t\"\xe6\x03\n\x10InstallOperation\x12;\n\x04type\x18\x01 \x02(\x0e\x32-.chromeos_update_engine.InstallOperation.Type\x12\x13\n\x0b\x64\x61ta_offset\x18\x02 \x01(\r\x12\x13\n\x0b\x64\x61ta_length\x18\x03 \x01(\r\x12\x33\n\x0bsrc_extents\x18\x04 \x03(\x0b\x32\x1e.chromeos_update_engine.Extent\x12\x12\n\nsrc_length\x18\x05 \x01(\x04\x12\x33\n\x0b\x64st_extents\x18\x06 \x03(\x0b\x32\x1e.chromeos_update_engine.Extent\x12\x12\n\ndst_length\x18\x07 \x01(\x04\x12\x18\n\x10\x64\x61ta_sha256_hash\x18\x08 \x01(\x0c\x12\x17\n\x0fsrc_sha256_hash\x18\t \x01(\x0c\"\xa5\x01\n\x04Type\x12\x0b\n\x07REPLACE\x10\x00\x12\x0e\n\nREPLACE_BZ\x10\x01\x12\x08\n\x04MOVE\x10\x02\x12\n\n\x06\x42SDIFF\x10\x03\x12\x0f\n\x0bSOURCE_COPY\x10\x04\x12\x11\n\rSOURCE_BSDIFF\x10\x05\x12\x08\n\x04ZERO\x10\x06\x12\x0b\n\x07\x44ISCARD\x10\x07\x12\x0e\n\nREPLACE_XZ\x10\x08\x12\x0c\n\x08PUFFDIFF\x10\t\x12\x11\n\rBROTLI_BSDIFF\x10\n\"\xa6\x03\n\x0fPartitionUpdate\x12\x16\n\x0epartition_name\x18\x01 \x02(\t\x12\x17\n\x0frun_postinstall\x18\x02 \x01(\x08\x12\x18\n\x10postinstall_path\x18\x03 \x01(\t\x12\x17\n\x0f\x66ilesystem_type\x18\x04 \x01(\t\x12M\n\x17new_partition_signature\x18\x05 \x03(\x0b\x32,.chromeos_update_engine.Signatures.Signature\x12\x41\n\x12old_partition_info\x18\x06 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12\x41\n\x12new_partition_info\x18\x07 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12<\n\noperations\x18\x08 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12\x1c\n\x14postinstall_optional\x18\t \x01(\x08\"\xc4\x05\n\x14\x44\x65ltaArchiveManifest\x12\x44\n\x12install_operations\x18\x01 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12K\n\x19kernel_install_operations\x18\x02 \x03(\x0b\x32(.chromeos_update_engine.InstallOperation\x12\x18\n\nblock_size\x18\x03 \x01(\r:\x04\x34\x30\x39\x36\x12\x19\n\x11signatures_offset\x18\x04 \x01(\x04\x12\x17\n\x0fsignatures_size\x18\x05 \x01(\x04\x12>\n\x0fold_kernel_info\x18\x06 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fnew_kernel_info\x18\x07 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fold_rootfs_info\x18\x08 \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12>\n\x0fnew_rootfs_info\x18\t \x01(\x0b\x32%.chromeos_update_engine.PartitionInfo\x12\x39\n\x0eold_image_info\x18\n \x01(\x0b\x32!.chromeos_update_engine.ImageInfo\x12\x39\n\x0enew_image_info\x18\x0b \x01(\x0b\x32!.chromeos_update_engine.ImageInfo\x12\x18\n\rminor_version\x18\x0c \x01(\r:\x01\x30\x12;\n\npartitions\x18\r \x03(\x0b\x32\'.chromeos_update_engine.PartitionUpdateB\x02H\x03')
@@ -63,11 +63,15 @@
name='PUFFDIFF', index=9, number=9,
options=None,
type=None),
+ _descriptor.EnumValueDescriptor(
+ name='BROTLI_BSDIFF', index=10, number=10,
+ options=None,
+ type=None),
],
containing_type=None,
options=None,
serialized_start=712,
- serialized_end=858,
+ serialized_end=877,
)
@@ -347,7 +351,7 @@
is_extendable=False,
extension_ranges=[],
serialized_start=391,
- serialized_end=858,
+ serialized_end=877,
)
@@ -430,8 +434,8 @@
options=None,
is_extendable=False,
extension_ranges=[],
- serialized_start=861,
- serialized_end=1283,
+ serialized_start=880,
+ serialized_end=1302,
)
@@ -542,8 +546,8 @@
options=None,
is_extendable=False,
extension_ranges=[],
- serialized_start=1286,
- serialized_end=1994,
+ serialized_start=1305,
+ serialized_end=2013,
)
_SIGNATURES_SIGNATURE.containing_type = _SIGNATURES;
diff --git a/sideload_main.cc b/sideload_main.cc
index 52f045e..ddb312e 100644
--- a/sideload_main.cc
+++ b/sideload_main.cc
@@ -25,7 +25,6 @@
#include <base/strings/stringprintf.h>
#include <brillo/asynchronous_signal_handler.h>
#include <brillo/flag_helper.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/base_message_loop.h>
#include <brillo/streams/file_stream.h>
#include <brillo/streams/stream.h>
diff --git a/update_attempter.cc b/update_attempter.cc
index cc9cb8f..9cef154 100644
--- a/update_attempter.cc
+++ b/update_attempter.cc
@@ -33,7 +33,6 @@
#include <base/strings/stringprintf.h>
#include <brillo/data_encoding.h>
#include <brillo/errors/error_codes.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/message_loop.h>
#include <policy/device_policy.h>
#include <policy/libpolicy.h>
@@ -120,20 +119,11 @@
return code;
}
-UpdateAttempter::UpdateAttempter(
- SystemState* system_state,
- CertificateChecker* cert_checker,
- org::chromium::NetworkProxyServiceInterfaceProxyInterface*
- network_proxy_service_proxy)
+UpdateAttempter::UpdateAttempter(SystemState* system_state,
+ CertificateChecker* cert_checker)
: processor_(new ActionProcessor()),
system_state_(system_state),
-#if USE_CHROME_NETWORK_PROXY
- cert_checker_(cert_checker),
- chrome_proxy_resolver_(network_proxy_service_proxy) {
-#else
- cert_checker_(cert_checker) {
-#endif // USE_CHROME_NETWORK_PROXY
-}
+ cert_checker_(cert_checker) {}
UpdateAttempter::~UpdateAttempter() {
// CertificateChecker might not be initialized in unittests.
@@ -329,12 +319,11 @@
bool use_p2p_for_downloading = false;
bool use_p2p_for_sharing = false;
- // Never use p2p for downloading in interactive checks unless the
- // developer has opted in for it via a marker file.
+ // Never use p2p for downloading in interactive checks unless the developer
+ // has opted in for it via a marker file.
//
- // (Why would a developer want to opt in? If he's working on the
- // update_engine or p2p codebases so he can actually test his
- // code.).
+ // (Why would a developer want to opt in? If they are working on the
+ // update_engine or p2p codebases so they can actually test their code.)
if (system_state_ != nullptr) {
if (!system_state_->p2p_manager()->IsP2PEnabled()) {
@@ -607,14 +596,12 @@
shared_ptr<OmahaResponseHandlerAction> response_handler_action(
new OmahaResponseHandlerAction(system_state_));
- shared_ptr<OmahaRequestAction> download_started_action(
- new OmahaRequestAction(system_state_,
- new OmahaEvent(
- OmahaEvent::kTypeUpdateDownloadStarted),
- brillo::make_unique_ptr(new LibcurlHttpFetcher(
- GetProxyResolver(),
- system_state_->hardware())),
- false));
+ shared_ptr<OmahaRequestAction> download_started_action(new OmahaRequestAction(
+ system_state_,
+ new OmahaEvent(OmahaEvent::kTypeUpdateDownloadStarted),
+ std::make_unique<LibcurlHttpFetcher>(GetProxyResolver(),
+ system_state_->hardware()),
+ false));
LibcurlHttpFetcher* download_fetcher =
new LibcurlHttpFetcher(GetProxyResolver(), system_state_->hardware());
@@ -626,25 +613,23 @@
system_state_->boot_control(),
system_state_->hardware(),
system_state_,
- download_fetcher)); // passes ownership
+ download_fetcher, // passes ownership
+ interactive));
shared_ptr<OmahaRequestAction> download_finished_action(
new OmahaRequestAction(
system_state_,
new OmahaEvent(OmahaEvent::kTypeUpdateDownloadFinished),
- brillo::make_unique_ptr(
- new LibcurlHttpFetcher(GetProxyResolver(),
- system_state_->hardware())),
+ std::make_unique<LibcurlHttpFetcher>(GetProxyResolver(),
+ system_state_->hardware()),
false));
shared_ptr<FilesystemVerifierAction> filesystem_verifier_action(
new FilesystemVerifierAction());
shared_ptr<OmahaRequestAction> update_complete_action(
- new OmahaRequestAction(
- system_state_,
- new OmahaEvent(OmahaEvent::kTypeUpdateComplete),
- brillo::make_unique_ptr(
- new LibcurlHttpFetcher(GetProxyResolver(),
- system_state_->hardware())),
- false));
+ new OmahaRequestAction(system_state_,
+ new OmahaEvent(OmahaEvent::kTypeUpdateComplete),
+ std::make_unique<LibcurlHttpFetcher>(
+ GetProxyResolver(), system_state_->hardware()),
+ false));
download_action->set_delegate(this);
response_handler_action_ = response_handler_action;
@@ -1084,6 +1069,15 @@
// Find out which action completed (successfully).
if (type == DownloadAction::StaticType()) {
SetStatusAndNotify(UpdateStatus::FINALIZING);
+ } else if (type == FilesystemVerifierAction::StaticType()) {
+ // Log the system properties before the postinst and after the file system
+ // is verified. It used to be done in the postinst itself. But postinst
+ // cannot do this anymore. On the other hand, these logs are frequently
+ // looked at and it is preferable not to scatter them in random location in
+ // the log and rather log it right before the postinst. The reason not do
+ // this in the |PostinstallRunnerAction| is to prevent dependency from
+ // libpayload_consumer to libupdate_engine.
+ LogImageProperties();
}
}
@@ -1325,9 +1319,8 @@
shared_ptr<OmahaRequestAction> error_event_action(
new OmahaRequestAction(system_state_,
error_event_.release(), // Pass ownership.
- brillo::make_unique_ptr(new LibcurlHttpFetcher(
- GetProxyResolver(),
- system_state_->hardware())),
+ std::make_unique<LibcurlHttpFetcher>(
+ GetProxyResolver(), system_state_->hardware()),
false));
actions_.push_back(shared_ptr<AbstractAction>(error_event_action));
processor_->EnqueueAction(error_event_action.get());
@@ -1371,9 +1364,8 @@
shared_ptr<OmahaRequestAction> ping_action(new OmahaRequestAction(
system_state_,
nullptr,
- brillo::make_unique_ptr(new LibcurlHttpFetcher(
- GetProxyResolver(),
- system_state_->hardware())),
+ std::make_unique<LibcurlHttpFetcher>(GetProxyResolver(),
+ system_state_->hardware()),
true));
actions_.push_back(shared_ptr<OmahaRequestAction>(ping_action));
processor_->set_delegate(nullptr);
diff --git a/update_attempter.h b/update_attempter.h
index d2fe3ec..76e93a2 100644
--- a/update_attempter.h
+++ b/update_attempter.h
@@ -46,12 +46,6 @@
#include "update_engine/update_manager/policy.h"
#include "update_engine/update_manager/update_manager.h"
-namespace org {
-namespace chromium {
-class NetworkProxyServiceInterfaceProxyInterface;
-} // namespace chromium
-} // namespace org
-
namespace policy {
class PolicyProvider;
}
@@ -69,10 +63,7 @@
using UpdateAttemptFlags = update_engine::UpdateAttemptFlags;
static const int kMaxDeltaUpdateFailures;
- UpdateAttempter(SystemState* system_state,
- CertificateChecker* cert_checker,
- org::chromium::NetworkProxyServiceInterfaceProxyInterface*
- network_proxy_service_proxy);
+ UpdateAttempter(SystemState* system_state, CertificateChecker* cert_checker);
~UpdateAttempter() override;
// Further initialization to be done post construction.
diff --git a/update_attempter_android.cc b/update_attempter_android.cc
index 99c941e..aacb06b 100644
--- a/update_attempter_android.cc
+++ b/update_attempter_android.cc
@@ -539,8 +539,9 @@
new DownloadAction(prefs_,
boot_control_,
hardware_,
- nullptr, // system_state, not used.
- download_fetcher)); // passes ownership
+ nullptr, // system_state, not used.
+ download_fetcher, // passes ownership
+ true /* is_interactive */));
shared_ptr<FilesystemVerifierAction> filesystem_verifier_action(
new FilesystemVerifierAction());
diff --git a/update_attempter_unittest.cc b/update_attempter_unittest.cc
index ff7700e..240e4ec 100644
--- a/update_attempter_unittest.cc
+++ b/update_attempter_unittest.cc
@@ -22,7 +22,6 @@
#include <base/files/file_util.h>
#include <base/message_loop/message_loop.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/base_message_loop.h>
#include <brillo/message_loops/message_loop.h>
#include <brillo/message_loops/message_loop_utils.h>
@@ -30,10 +29,6 @@
#include <policy/libpolicy.h>
#include <policy/mock_device_policy.h>
-#if USE_CHROME_NETWORK_PROXY
-#include "network_proxy/dbus-proxies.h"
-#include "network_proxy/dbus-proxy-mocks.h"
-#endif // USE_CHROME_NETWORK_PROXY
#include "update_engine/common/fake_clock.h"
#include "update_engine/common/fake_prefs.h"
#include "update_engine/common/mock_action.h"
@@ -53,22 +48,10 @@
#include "update_engine/payload_consumer/payload_constants.h"
#include "update_engine/payload_consumer/postinstall_runner_action.h"
-namespace org {
-namespace chromium {
-class NetworkProxyServiceInterfaceProxyMock;
-} // namespace chromium
-} // namespace org
-
using base::Time;
using base::TimeDelta;
using chromeos_update_manager::EvalStatus;
using chromeos_update_manager::UpdateCheckParams;
-using org::chromium::NetworkProxyServiceInterfaceProxyInterface;
-using org::chromium::NetworkProxyServiceInterfaceProxyMock;
-#if USE_LIBCROS
-using org::chromium::LibCrosServiceInterfaceProxyMock;
-using org::chromium::UpdateEngineLibcrosProxyResolvedInterfaceProxyMock;
-#endif // USE_LIBCROS
using std::string;
using std::unique_ptr;
using testing::_;
@@ -81,7 +64,7 @@
using testing::Return;
using testing::ReturnPointee;
using testing::SaveArg;
-using testing::SetArgumentPointee;
+using testing::SetArgPointee;
using update_engine::UpdateAttemptFlags;
using update_engine::UpdateEngineStatus;
using update_engine::UpdateStatus;
@@ -93,10 +76,8 @@
// methods.
class UpdateAttempterUnderTest : public UpdateAttempter {
public:
- UpdateAttempterUnderTest(
- SystemState* system_state,
- NetworkProxyServiceInterfaceProxyInterface* network_proxy_service_proxy)
- : UpdateAttempter(system_state, nullptr, network_proxy_service_proxy) {}
+ explicit UpdateAttempterUnderTest(SystemState* system_state)
+ : UpdateAttempter(system_state, nullptr) {}
// Wrap the update scheduling method, allowing us to opt out of scheduled
// updates for testing purposes.
@@ -196,13 +177,7 @@
brillo::BaseMessageLoop loop_{&base_loop_};
FakeSystemState fake_system_state_;
-#if USE_CHROME_NETWORK_PROXY
- NetworkProxyServiceInterfaceProxyMock network_proxy_service_proxy_mock_;
- UpdateAttempterUnderTest attempter_{&fake_system_state_,
- &network_proxy_service_proxy_mock_};
-#else
- UpdateAttempterUnderTest attempter_{&fake_system_state_, nullptr};
-#endif // USE_CHROME_NETWORK_PROXY
+ UpdateAttempterUnderTest attempter_{&fake_system_state_};
OpenSSLWrapper openssl_wrapper_;
CertificateChecker certificate_checker_;
@@ -224,7 +199,12 @@
TEST_F(UpdateAttempterTest, ActionCompletedDownloadTest) {
unique_ptr<MockHttpFetcher> fetcher(new MockHttpFetcher("", 0, nullptr));
fetcher->FailTransfer(503); // Sets the HTTP response code.
- DownloadAction action(prefs_, nullptr, nullptr, nullptr, fetcher.release());
+ DownloadAction action(prefs_,
+ nullptr,
+ nullptr,
+ nullptr,
+ fetcher.release(),
+ false /* is_interactive */);
EXPECT_CALL(*prefs_, GetInt64(kPrefsDeltaUpdateFailures, _)).Times(0);
attempter_.ActionCompleted(nullptr, &action, ErrorCode::kSuccess);
EXPECT_EQ(UpdateStatus::FINALIZING, attempter_.status());
@@ -388,13 +368,13 @@
EXPECT_TRUE(attempter_.omaha_request_params_->delta_okay());
EXPECT_CALL(*prefs_, GetInt64(kPrefsDeltaUpdateFailures, _))
.WillOnce(DoAll(
- SetArgumentPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures - 1),
+ SetArgPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures - 1),
Return(true)));
attempter_.DisableDeltaUpdateIfNeeded();
EXPECT_TRUE(attempter_.omaha_request_params_->delta_okay());
EXPECT_CALL(*prefs_, GetInt64(kPrefsDeltaUpdateFailures, _))
.WillOnce(DoAll(
- SetArgumentPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures),
+ SetArgPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures),
Return(true)));
attempter_.DisableDeltaUpdateIfNeeded();
EXPECT_FALSE(attempter_.omaha_request_params_->delta_okay());
@@ -406,10 +386,10 @@
TEST_F(UpdateAttempterTest, MarkDeltaUpdateFailureTest) {
EXPECT_CALL(*prefs_, GetInt64(kPrefsDeltaUpdateFailures, _))
.WillOnce(Return(false))
- .WillOnce(DoAll(SetArgumentPointee<1>(-1), Return(true)))
- .WillOnce(DoAll(SetArgumentPointee<1>(1), Return(true)))
+ .WillOnce(DoAll(SetArgPointee<1>(-1), Return(true)))
+ .WillOnce(DoAll(SetArgPointee<1>(1), Return(true)))
.WillOnce(DoAll(
- SetArgumentPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures),
+ SetArgPointee<1>(UpdateAttempter::kMaxDeltaUpdateFailures),
Return(true)));
EXPECT_CALL(*prefs_, SetInt64(Ne(kPrefsDeltaUpdateFailures), _))
.WillRepeatedly(Return(true));
@@ -476,10 +456,11 @@
// Expect that the device policy is loaded by the UpdateAttempter at some
// point by calling RefreshDevicePolicy.
- policy::MockDevicePolicy* device_policy = new policy::MockDevicePolicy();
- attempter_.policy_provider_.reset(new policy::PolicyProvider(device_policy));
+ auto device_policy = std::make_unique<policy::MockDevicePolicy>();
EXPECT_CALL(*device_policy, LoadPolicy())
.Times(testing::AtLeast(1)).WillRepeatedly(Return(true));
+ attempter_.policy_provider_.reset(
+ new policy::PolicyProvider(std::move(device_policy)));
{
InSequence s;
@@ -518,11 +499,23 @@
void UpdateAttempterTest::RollbackTestStart(
bool enterprise_rollback, bool valid_slot) {
// Create a device policy so that we can change settings.
- policy::MockDevicePolicy* device_policy = new policy::MockDevicePolicy();
- attempter_.policy_provider_.reset(new policy::PolicyProvider(device_policy));
-
+ auto device_policy = std::make_unique<policy::MockDevicePolicy>();
EXPECT_CALL(*device_policy, LoadPolicy()).WillRepeatedly(Return(true));
- fake_system_state_.set_device_policy(device_policy);
+ fake_system_state_.set_device_policy(device_policy.get());
+ if (enterprise_rollback) {
+ // We return an empty owner as this is an enterprise.
+ EXPECT_CALL(*device_policy, GetOwner(_)).WillRepeatedly(
+ DoAll(SetArgPointee<0>(string("")),
+ Return(true)));
+ } else {
+ // We return a fake owner as this is an owned consumer device.
+ EXPECT_CALL(*device_policy, GetOwner(_)).WillRepeatedly(
+ DoAll(SetArgPointee<0>(string("fake.mail@fake.com")),
+ Return(true)));
+ }
+
+ attempter_.policy_provider_.reset(
+ new policy::PolicyProvider(std::move(device_policy)));
if (valid_slot) {
BootControlInterface::Slot rollback_slot = 1;
@@ -540,18 +533,6 @@
is_rollback_allowed = true;
}
- if (enterprise_rollback) {
- // We return an empty owner as this is an enterprise.
- EXPECT_CALL(*device_policy, GetOwner(_)).WillRepeatedly(
- DoAll(SetArgumentPointee<0>(string("")),
- Return(true)));
- } else {
- // We return a fake owner as this is an owned consumer device.
- EXPECT_CALL(*device_policy, GetOwner(_)).WillRepeatedly(
- DoAll(SetArgumentPointee<0>(string("fake.mail@fake.com")),
- Return(true)));
- }
-
if (is_rollback_allowed) {
InSequence s;
for (size_t i = 0; i < arraysize(kRollbackActionTypes); ++i) {
@@ -822,17 +803,18 @@
void UpdateAttempterTest::ReadScatterFactorFromPolicyTestStart() {
int64_t scatter_factor_in_seconds = 36000;
- policy::MockDevicePolicy* device_policy = new policy::MockDevicePolicy();
- attempter_.policy_provider_.reset(new policy::PolicyProvider(device_policy));
-
+ auto device_policy = std::make_unique<policy::MockDevicePolicy>();
EXPECT_CALL(*device_policy, LoadPolicy()).WillRepeatedly(Return(true));
- fake_system_state_.set_device_policy(device_policy);
+ fake_system_state_.set_device_policy(device_policy.get());
EXPECT_CALL(*device_policy, GetScatterFactorInSeconds(_))
.WillRepeatedly(DoAll(
- SetArgumentPointee<0>(scatter_factor_in_seconds),
+ SetArgPointee<0>(scatter_factor_in_seconds),
Return(true)));
+ attempter_.policy_provider_.reset(
+ new policy::PolicyProvider(std::move(device_policy)));
+
attempter_.Update("", "", "", "", false, false);
EXPECT_EQ(scatter_factor_in_seconds, attempter_.scatter_factor_.InSeconds());
@@ -860,17 +842,18 @@
int64_t scatter_factor_in_seconds = 10;
- policy::MockDevicePolicy* device_policy = new policy::MockDevicePolicy();
- attempter_.policy_provider_.reset(new policy::PolicyProvider(device_policy));
-
+ auto device_policy = std::make_unique<policy::MockDevicePolicy>();
EXPECT_CALL(*device_policy, LoadPolicy()).WillRepeatedly(Return(true));
- fake_system_state_.set_device_policy(device_policy);
+ fake_system_state_.set_device_policy(device_policy.get());
EXPECT_CALL(*device_policy, GetScatterFactorInSeconds(_))
.WillRepeatedly(DoAll(
- SetArgumentPointee<0>(scatter_factor_in_seconds),
+ SetArgPointee<0>(scatter_factor_in_seconds),
Return(true)));
+ attempter_.policy_provider_.reset(
+ new policy::PolicyProvider(std::move(device_policy)));
+
attempter_.Update("", "", "", "", false, false);
EXPECT_EQ(scatter_factor_in_seconds, attempter_.scatter_factor_.InSeconds());
@@ -919,17 +902,18 @@
// otherwise.
int64_t scatter_factor_in_seconds = 50;
- policy::MockDevicePolicy* device_policy = new policy::MockDevicePolicy();
- attempter_.policy_provider_.reset(new policy::PolicyProvider(device_policy));
-
+ auto device_policy = std::make_unique<policy::MockDevicePolicy>();
EXPECT_CALL(*device_policy, LoadPolicy()).WillRepeatedly(Return(true));
- fake_system_state_.set_device_policy(device_policy);
+ fake_system_state_.set_device_policy(device_policy.get());
EXPECT_CALL(*device_policy, GetScatterFactorInSeconds(_))
.WillRepeatedly(DoAll(
- SetArgumentPointee<0>(scatter_factor_in_seconds),
+ SetArgPointee<0>(scatter_factor_in_seconds),
Return(true)));
+ attempter_.policy_provider_.reset(
+ new policy::PolicyProvider(std::move(device_policy)));
+
// Trigger an interactive check so we can test that scattering is disabled.
attempter_.Update("", "", "", "", false, true);
EXPECT_EQ(scatter_factor_in_seconds, attempter_.scatter_factor_.InSeconds());
diff --git a/update_engine.conf b/update_engine.conf
index 449e669..e3f246f 100644
--- a/update_engine.conf
+++ b/update_engine.conf
@@ -1,2 +1,2 @@
PAYLOAD_MAJOR_VERSION=2
-PAYLOAD_MINOR_VERSION=3
+PAYLOAD_MINOR_VERSION=4
diff --git a/update_engine.gyp b/update_engine.gyp
index ba4cfff..f312a1d 100644
--- a/update_engine.gyp
+++ b/update_engine.gyp
@@ -85,7 +85,7 @@
},
},
'sources': [
- 'update_metadata.proto'
+ 'update_metadata.proto',
],
'includes': ['../../../platform2/common-mk/protoc.gypi'],
},
@@ -109,7 +109,7 @@
'action_name': 'update_engine-dbus-libcros-client-action',
'variables': {
'mock_output_file': 'include/libcros/dbus-proxy-mocks.h',
- 'proxy_output_file': 'include/libcros/dbus-proxies.h'
+ 'proxy_output_file': 'include/libcros/dbus-proxies.h',
},
'sources': [
'dbus_bindings/org.chromium.LibCrosService.dbus-xml',
@@ -117,21 +117,6 @@
'includes': ['../../../platform2/common-mk/generate-dbus-proxies.gypi'],
}],
},
- {
- 'target_name': 'update_engine-dbus-chrome_network_proxy-client',
- 'type': 'none',
- 'actions': [{
- 'action_name': 'update_engine-dbus-chrome_network_proxy-client-action',
- 'variables': {
- 'mock_output_file': 'include/network_proxy/dbus-proxy-mocks.h',
- 'proxy_output_file': 'include/network_proxy/dbus-proxies.h'
- },
- 'sources': [
- 'dbus_bindings/org.chromium.NetworkProxyService.dbus-xml',
- ],
- 'includes': ['../../../platform2/common-mk/generate-dbus-proxies.gypi'],
- }],
- },
# The payload application component and common dependencies.
{
'target_name': 'libpayload_consumer',
@@ -145,6 +130,7 @@
'exported_deps': [
'libcrypto',
'xz-embedded',
+ 'libpuffpatch',
],
'deps': ['<@(exported_deps)'],
},
@@ -185,8 +171,10 @@
'common/terminator.cc',
'common/utils.cc',
'payload_consumer/bzip_extent_writer.cc',
+ 'payload_consumer/cached_file_descriptor.cc',
'payload_consumer/delta_performer.cc',
'payload_consumer/download_action.cc',
+ 'payload_consumer/extent_reader.cc',
'payload_consumer/extent_writer.cc',
'payload_consumer/file_descriptor.cc',
'payload_consumer/file_descriptor_utils.cc',
@@ -233,6 +221,7 @@
'libshill-client',
'libssl',
'libupdate_engine-client',
+ 'vboot_host',
],
'deps': ['<@(exported_deps)'],
},
@@ -254,7 +243,6 @@
'-lpolicy-<(libbase_ver)',
'-lrootdev',
'-lrt',
- '-lvboot_host',
],
},
'sources': [
@@ -300,9 +288,6 @@
],
'conditions': [
['USE_chrome_network_proxy == 1', {
- 'dependencies': [
- 'update_engine-dbus-chrome_network_proxy-client',
- ],
'sources': [
'chrome_browser_proxy_resolver.cc',
],
@@ -355,7 +340,7 @@
'common/error_code_utils.cc',
'omaha_utils.cc',
'update_engine_client.cc',
- ],
+ ],
},
# server-side code. This is used for delta_generator and unittests but not
# for any client code.
@@ -369,6 +354,7 @@
'variables': {
'exported_deps': [
'ext2fs',
+ 'libpuffdiff',
],
'deps': ['<@(exported_deps)'],
},
@@ -396,6 +382,7 @@
'payload_generator/block_mapping.cc',
'payload_generator/bzip.cc',
'payload_generator/cycle_breaker.cc',
+ 'payload_generator/deflate_utils.cc',
'payload_generator/delta_diff_generator.cc',
'payload_generator/delta_diff_utils.cc',
'payload_generator/ext2_filesystem.cc',
@@ -410,6 +397,7 @@
'payload_generator/payload_generation_config.cc',
'payload_generator/payload_signer.cc',
'payload_generator/raw_filesystem.cc',
+ 'payload_generator/squashfs_filesystem.cc',
'payload_generator/tarjan.cc',
'payload_generator/topological_sort.cc',
'payload_generator/xz_chromeos.cc',
@@ -483,7 +471,6 @@
{
'target_name': 'update_engine_unittests',
'type': 'executable',
- 'includes': ['../../../platform2/common-mk/common_test.gypi'],
'variables': {
'deps': [
'libbrillo-test-<(libbase_ver)',
@@ -531,9 +518,11 @@
'omaha_utils_unittest.cc',
'p2p_manager_unittest.cc',
'payload_consumer/bzip_extent_writer_unittest.cc',
+ 'payload_consumer/cached_file_descriptor_unittest.cc',
'payload_consumer/delta_performer_integration_test.cc',
'payload_consumer/delta_performer_unittest.cc',
'payload_consumer/download_action_unittest.cc',
+ 'payload_consumer/extent_reader_unittest.cc',
'payload_consumer/extent_writer_unittest.cc',
'payload_consumer/fake_file_descriptor.cc',
'payload_consumer/file_descriptor_utils_unittest.cc',
@@ -545,6 +534,7 @@
'payload_generator/blob_file_writer_unittest.cc',
'payload_generator/block_mapping_unittest.cc',
'payload_generator/cycle_breaker_unittest.cc',
+ 'payload_generator/deflate_utils_unittest.cc',
'payload_generator/delta_diff_utils_unittest.cc',
'payload_generator/ext2_filesystem_unittest.cc',
'payload_generator/extent_ranges_unittest.cc',
@@ -557,11 +547,13 @@
'payload_generator/payload_file_unittest.cc',
'payload_generator/payload_generation_config_unittest.cc',
'payload_generator/payload_signer_unittest.cc',
+ 'payload_generator/squashfs_filesystem_unittest.cc',
'payload_generator/tarjan_unittest.cc',
'payload_generator/topological_sort_unittest.cc',
'payload_generator/zip_unittest.cc',
'payload_state_unittest.cc',
'proxy_resolver_unittest.cc',
+ 'testrunner.cc',
'update_attempter_unittest.cc',
'update_manager/boxed_value_unittest.cc',
'update_manager/chromeos_policy_unittest.cc',
@@ -577,23 +569,6 @@
'update_manager/umtest_utils.cc',
'update_manager/update_manager_unittest.cc',
'update_manager/variable_unittest.cc',
- # Main entry point for runnning tests.
- 'testrunner.cc',
- ],
- 'conditions': [
- ['USE_chrome_network_proxy == 1', {
- 'dependencies': [
- 'update_engine-dbus-chrome_network_proxy-client',
- ],
- 'sources': [
- 'chrome_browser_proxy_resolver_unittest.cc',
- ],
- }],
- ['USE_chrome_kiosk_app == 1', {
- 'dependencies': [
- 'update_engine-dbus-libcros-client',
- ],
- }],
],
},
],
diff --git a/update_engine_client.cc b/update_engine_client.cc
index dbfc66b..bb19632 100644
--- a/update_engine_client.cc
+++ b/update_engine_client.cc
@@ -26,6 +26,7 @@
#include <base/command_line.h>
#include <base/logging.h>
#include <base/macros.h>
+#include <base/threading/platform_thread.h>
#include <brillo/daemons/daemon.h>
#include <brillo/flag_helper.h>
@@ -52,6 +53,11 @@
// initialization.
const int kContinueRunning = -1;
+// The ShowStatus request will be retried `kShowStatusRetryCount` times at
+// `kShowStatusRetryInterval` second intervals on failure.
+const int kShowStatusRetryCount = 30;
+const int kShowStatusRetryIntervalInSeconds = 2;
+
class UpdateEngineClient : public brillo::Daemon {
public:
UpdateEngineClient(int argc, char** argv) : argc_(argc), argv_(argv) {
@@ -151,9 +157,18 @@
string new_version;
int64_t new_size = 0;
- if (!client_->GetStatus(&last_checked_time, &progress, ¤t_op,
- &new_version, &new_size)) {
- return false;
+ int retry_count = kShowStatusRetryCount;
+ while (retry_count > 0) {
+ if (client_->GetStatus(&last_checked_time, &progress, ¤t_op,
+ &new_version, &new_size)) {
+ break;
+ }
+ if (--retry_count == 0) {
+ return false;
+ }
+ LOG(WARNING) << "Will try " << retry_count << " more times!";
+ base::PlatformThread::Sleep(
+ base::TimeDelta::FromSeconds(kShowStatusRetryIntervalInSeconds));
}
printf("LAST_CHECKED_TIME=%" PRIi64
diff --git a/update_manager/evaluation_context.cc b/update_manager/evaluation_context.cc
index 98238f2..b6c7b91 100644
--- a/update_manager/evaluation_context.cc
+++ b/update_manager/evaluation_context.cc
@@ -24,7 +24,6 @@
#include <base/bind.h>
#include <base/json/json_writer.h>
#include <base/location.h>
-#include <base/memory/ptr_util.h>
#include <base/strings/string_util.h>
#include <base/values.h>
@@ -229,7 +228,7 @@
}
string EvaluationContext::DumpContext() const {
- auto variables = base::MakeUnique<base::DictionaryValue>();
+ auto variables = std::make_unique<base::DictionaryValue>();
for (auto& it : value_cache_) {
variables->SetString(it.first->GetName(), it.second.ToString());
}
diff --git a/update_manager/real_device_policy_provider_unittest.cc b/update_manager/real_device_policy_provider_unittest.cc
index 71c95bb..167cbd9 100644
--- a/update_manager/real_device_policy_provider_unittest.cc
+++ b/update_manager/real_device_policy_provider_unittest.cc
@@ -18,7 +18,7 @@
#include <memory>
-#include <brillo/make_unique_ptr.h>
+#include <base/memory/ptr_util.h>
#include <brillo/message_loops/fake_message_loop.h>
#include <brillo/message_loops/message_loop.h>
#include <brillo/message_loops/message_loop_utils.h>
@@ -63,8 +63,7 @@
auto session_manager_proxy_mock =
new org::chromium::SessionManagerInterfaceProxyMock();
provider_.reset(new RealDevicePolicyProvider(
- brillo::make_unique_ptr(session_manager_proxy_mock),
- &mock_policy_provider_));
+ base::WrapUnique(session_manager_proxy_mock), &mock_policy_provider_));
#else
provider_.reset(new RealDevicePolicyProvider(&mock_policy_provider_));
#endif // USE_DBUS
diff --git a/update_manager/real_shill_provider_unittest.cc b/update_manager/real_shill_provider_unittest.cc
index dc22e54..6506923 100644
--- a/update_manager/real_shill_provider_unittest.cc
+++ b/update_manager/real_shill_provider_unittest.cc
@@ -18,8 +18,8 @@
#include <memory>
#include <utility>
+#include <base/memory/ptr_util.h>
#include <base/time/time.h>
-#include <brillo/make_unique_ptr.h>
#include <brillo/message_loops/fake_message_loop.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
@@ -263,9 +263,9 @@
EXPECT_CALL(*service_proxy_mock, GetProperties(_, _, _))
.WillOnce(DoAll(SetArgPointee<0>(reply_dict), Return(true)));
- fake_shill_proxy_->SetServiceForPath(
- dbus::ObjectPath(service_path),
- brillo::make_unique_ptr(service_proxy_mock));
+ fake_shill_proxy_->SetServiceForPath(dbus::ObjectPath(service_path),
+ base::WrapUnique(service_proxy_mock));
+
return service_proxy_mock;
}
diff --git a/update_manager/real_system_provider_unittest.cc b/update_manager/real_system_provider_unittest.cc
index 33838e5..103a35f 100644
--- a/update_manager/real_system_provider_unittest.cc
+++ b/update_manager/real_system_provider_unittest.cc
@@ -19,7 +19,6 @@
#include <memory>
#include <base/time/time.h>
-#include <brillo/make_unique_ptr.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
diff --git a/update_manager/state_factory.cc b/update_manager/state_factory.cc
index a79b676..208ed51 100644
--- a/update_manager/state_factory.cc
+++ b/update_manager/state_factory.cc
@@ -19,7 +19,6 @@
#include <memory>
#include <base/logging.h>
-#include <brillo/make_unique_ptr.h>
#if USE_DBUS
#include <session_manager/dbus-proxies.h>
#endif // USE_DBUS
@@ -57,8 +56,7 @@
chromeos_update_engine::DBusConnection::Get()->GetDBus();
unique_ptr<RealDevicePolicyProvider> device_policy_provider(
new RealDevicePolicyProvider(
- brillo::make_unique_ptr(
- new org::chromium::SessionManagerInterfaceProxy(bus)),
+ std::make_unique<org::chromium::SessionManagerInterfaceProxy>(bus),
policy_provider));
#else
unique_ptr<RealDevicePolicyProvider> device_policy_provider(
diff --git a/update_metadata.proto b/update_metadata.proto
index be45f88..b5d6c59 100644
--- a/update_metadata.proto
+++ b/update_metadata.proto
@@ -77,10 +77,15 @@
// - REPLACE_XZ: Replace the dst_extents with the contents of the attached
// xz file after decompression. The xz file should only use crc32 or no crc at
// all to be compatible with xz-embedded.
+// - PUFFDIFF: Read the data in src_extents in the old partition, perform
+// puffpatch with the attached data and write the new data to dst_extents in
+// the new partition.
//
// The operations allowed in the payload (supported by the client) depend on the
// major and minor version. See InstallOperation.Type bellow for details.
+syntax = "proto2";
+
package chromeos_update_engine;
option optimize_for = LITE_RUNTIME;
@@ -159,12 +164,13 @@
// On minor version 3 or newer and on major version 2 or newer, these
// operations are supported:
- ZERO = 6; // Write zeros in the destination.
- DISCARD = 7; // Discard the destination blocks, reading as undefined.
REPLACE_XZ = 8; // Replace destination extents w/ attached xz data.
// On minor version 4 or newer, these operations are supported:
+ ZERO = 6; // Write zeros in the destination.
+ DISCARD = 7; // Discard the destination blocks, reading as undefined.
PUFFDIFF = 9; // The data is in puffdiff format.
+ BROTLI_BSDIFF = 10; // Like SOURCE_BSDIFF, but compressed with brotli.
}
required Type type = 1;
// The offset into the delta file (after the protobuf)
@@ -176,14 +182,15 @@
// Ordered list of extents that are read from (if any) and written to.
repeated Extent src_extents = 4;
// Byte length of src, equal to the number of blocks in src_extents *
- // block_size. It is used for BSDIFF, because we need to pass that
- // external program the number of bytes to read from the blocks we pass it.
- // This is not used in any other operation.
+ // block_size. It is used for BSDIFF and SOURCE_BSDIFF, because we need to
+ // pass that external program the number of bytes to read from the blocks we
+ // pass it. This is not used in any other operation.
optional uint64 src_length = 5;
repeated Extent dst_extents = 6;
// Byte length of dst, equal to the number of blocks in dst_extents *
- // block_size. Used for BSDIFF, but not in any other operation.
+ // block_size. Used for BSDIFF and SOURCE_BSDIFF, but not in any other
+ // operation.
optional uint64 dst_length = 7;
// Optional SHA 256 hash of the blob associated with this operation.