blob: 234a5878880e0bf6d71f17d4334c3a2eb8d0303e [file] [log] [blame]
Amin Hassanid7da8f42017-08-23 14:29:40 -07001//
2// Copyright (C) 2017 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/squashfs_filesystem.h"
18
19#include <fcntl.h>
20
21#include <algorithm>
22#include <string>
Amin Hassani3cd4df12017-08-25 11:21:53 -070023#include <utility>
Amin Hassanid7da8f42017-08-23 14:29:40 -070024
25#include <base/files/file_util.h>
Amin Hassani77c25fc2019-01-29 10:24:19 -080026#include <base/files/scoped_temp_dir.h>
Amin Hassanid7da8f42017-08-23 14:29:40 -070027#include <base/logging.h>
28#include <base/strings/string_number_conversions.h>
29#include <base/strings/string_split.h>
30#include <brillo/streams/file_stream.h>
31
32#include "update_engine/common/subprocess.h"
33#include "update_engine/common/utils.h"
Amin Hassani3cd4df12017-08-25 11:21:53 -070034#include "update_engine/payload_generator/deflate_utils.h"
Amin Hassanid7da8f42017-08-23 14:29:40 -070035#include "update_engine/payload_generator/delta_diff_generator.h"
36#include "update_engine/payload_generator/extent_ranges.h"
37#include "update_engine/payload_generator/extent_utils.h"
38#include "update_engine/update_metadata.pb.h"
39
Amin Hassani77c25fc2019-01-29 10:24:19 -080040using base::FilePath;
41using base::ScopedTempDir;
Amin Hassanid7da8f42017-08-23 14:29:40 -070042using std::string;
43using std::unique_ptr;
44using std::vector;
45
46namespace chromeos_update_engine {
47
48namespace {
49
Amin Hassanid7da8f42017-08-23 14:29:40 -070050// The size of the squashfs super block.
51constexpr size_t kSquashfsSuperBlockSize = 96;
52constexpr uint64_t kSquashfsCompressedBit = 1 << 24;
Amin Hassani3cd4df12017-08-25 11:21:53 -070053constexpr uint32_t kSquashfsZlibCompression = 1;
Amin Hassanid7da8f42017-08-23 14:29:40 -070054
Amin Hassani77c25fc2019-01-29 10:24:19 -080055constexpr char kUpdateEngineConf[] = "etc/update_engine.conf";
56
Amin Hassanid7da8f42017-08-23 14:29:40 -070057bool ReadSquashfsHeader(const brillo::Blob blob,
58 SquashfsFilesystem::SquashfsHeader* header) {
59 if (blob.size() < kSquashfsSuperBlockSize) {
60 return false;
61 }
62
63 memcpy(&header->magic, blob.data(), 4);
64 memcpy(&header->block_size, blob.data() + 12, 4);
65 memcpy(&header->compression_type, blob.data() + 20, 2);
66 memcpy(&header->major_version, blob.data() + 28, 2);
67 return true;
68}
69
70bool CheckHeader(const SquashfsFilesystem::SquashfsHeader& header) {
71 return header.magic == 0x73717368 && header.major_version == 4;
72}
73
74bool GetFileMapContent(const string& sqfs_path, string* map) {
75 // Create a tmp file
76 string map_file;
77 TEST_AND_RETURN_FALSE(
78 utils::MakeTempFile("squashfs_file_map.XXXXXX", &map_file, nullptr));
79 ScopedPathUnlinker map_unlinker(map_file);
80
81 // Run unsquashfs to get the system file map.
82 // unsquashfs -m <map-file> <squashfs-file>
83 vector<string> cmd = {"unsquashfs", "-m", map_file, sqfs_path};
Amin Hassani3a4caa12019-11-06 11:12:28 -080084 string stdout, stderr;
Amin Hassanid7da8f42017-08-23 14:29:40 -070085 int exit_code;
Amin Hassani3a4caa12019-11-06 11:12:28 -080086 if (!Subprocess::SynchronousExec(cmd, &exit_code, &stdout, &stderr) ||
Amin Hassanid7da8f42017-08-23 14:29:40 -070087 exit_code != 0) {
Amin Hassani3a4caa12019-11-06 11:12:28 -080088 LOG(ERROR) << "Failed to run `unsquashfs -m` with stdout content: "
89 << stdout << " and stderr content: " << stderr;
Amin Hassanid7da8f42017-08-23 14:29:40 -070090 return false;
91 }
92 TEST_AND_RETURN_FALSE(utils::ReadFile(map_file, map));
93 return true;
94}
95
Amin Hassani77c25fc2019-01-29 10:24:19 -080096bool GetUpdateEngineConfig(const std::string& sqfs_path, string* config) {
97 ScopedTempDir unsquash_dir;
98 if (!unsquash_dir.CreateUniqueTempDir()) {
99 PLOG(ERROR) << "Failed to create a temporary directory.";
100 return false;
101 }
102
103 // Run unsquashfs to extract update_engine.conf
104 // -f: To force overriding if the target directory exists.
105 // -d: The directory to unsquash the files.
106 vector<string> cmd = {"unsquashfs",
107 "-f",
108 "-d",
109 unsquash_dir.GetPath().value(),
110 sqfs_path,
111 kUpdateEngineConf};
Amin Hassani3a4caa12019-11-06 11:12:28 -0800112 string stdout, stderr;
Amin Hassani77c25fc2019-01-29 10:24:19 -0800113 int exit_code;
Amin Hassani3a4caa12019-11-06 11:12:28 -0800114 if (!Subprocess::SynchronousExec(cmd, &exit_code, &stdout, &stderr) ||
Amin Hassani77c25fc2019-01-29 10:24:19 -0800115 exit_code != 0) {
Amin Hassani3a4caa12019-11-06 11:12:28 -0800116 PLOG(ERROR) << "Failed to unsquashfs etc/update_engine.conf with stdout: "
117 << stdout << " and stderr: " << stderr;
Amin Hassani77c25fc2019-01-29 10:24:19 -0800118 return false;
119 }
120
121 auto config_path = unsquash_dir.GetPath().Append(kUpdateEngineConf);
122 string config_content;
123 if (!utils::ReadFile(config_path.value(), &config_content)) {
124 PLOG(ERROR) << "Failed to read " << config_path.value();
125 return false;
126 }
127
128 if (config_content.empty()) {
129 LOG(ERROR) << "update_engine config file was empty!!";
130 return false;
131 }
132
133 *config = std::move(config_content);
134 return true;
135}
136
Amin Hassanid7da8f42017-08-23 14:29:40 -0700137} // namespace
138
139bool SquashfsFilesystem::Init(const string& map,
Amin Hassani3cd4df12017-08-25 11:21:53 -0700140 const string& sqfs_path,
Amin Hassanid7da8f42017-08-23 14:29:40 -0700141 size_t size,
Amin Hassani3cd4df12017-08-25 11:21:53 -0700142 const SquashfsHeader& header,
143 bool extract_deflates) {
Amin Hassanid7da8f42017-08-23 14:29:40 -0700144 size_ = size;
Amin Hassani3cd4df12017-08-25 11:21:53 -0700145
146 bool is_zlib = header.compression_type == kSquashfsZlibCompression;
147 if (!is_zlib) {
148 LOG(WARNING) << "Filesystem is not Gzipped. Not filling deflates!";
149 }
150 vector<puffin::ByteExtent> zlib_blks;
151
Amin Hassanid7da8f42017-08-23 14:29:40 -0700152 // Reading files map. For the format of the file map look at the comments for
153 // |CreateFromFileMap()|.
154 auto lines = base::SplitStringPiece(map,
155 "\n",
156 base::WhitespaceHandling::KEEP_WHITESPACE,
157 base::SplitResult::SPLIT_WANT_NONEMPTY);
158 for (const auto& line : lines) {
159 auto splits =
160 base::SplitStringPiece(line,
161 " \t",
162 base::WhitespaceHandling::TRIM_WHITESPACE,
163 base::SplitResult::SPLIT_WANT_NONEMPTY);
164 // Only filename is invalid.
165 TEST_AND_RETURN_FALSE(splits.size() > 1);
166 uint64_t start;
167 TEST_AND_RETURN_FALSE(base::StringToUint64(splits[1], &start));
168 uint64_t cur_offset = start;
Amin Hassani1a200c12020-02-26 14:47:23 -0800169 bool is_compressed = false;
Amin Hassanid7da8f42017-08-23 14:29:40 -0700170 for (size_t i = 2; i < splits.size(); ++i) {
171 uint64_t blk_size;
172 TEST_AND_RETURN_FALSE(base::StringToUint64(splits[i], &blk_size));
173 // TODO(ahassani): For puffin push it into a proper list if uncompressed.
174 auto new_blk_size = blk_size & ~kSquashfsCompressedBit;
175 TEST_AND_RETURN_FALSE(new_blk_size <= header.block_size);
Amin Hassani3cd4df12017-08-25 11:21:53 -0700176 if (new_blk_size > 0 && !(blk_size & kSquashfsCompressedBit)) {
Amin Hassani1a200c12020-02-26 14:47:23 -0800177 // It is a compressed block.
Amin Hassani3cd4df12017-08-25 11:21:53 -0700178 if (is_zlib && extract_deflates) {
179 zlib_blks.emplace_back(cur_offset, new_blk_size);
180 }
Amin Hassani1a200c12020-02-26 14:47:23 -0800181 is_compressed = true;
Amin Hassani3cd4df12017-08-25 11:21:53 -0700182 }
Amin Hassanid7da8f42017-08-23 14:29:40 -0700183 cur_offset += new_blk_size;
184 }
185
186 // If size is zero do not add the file.
187 if (cur_offset - start > 0) {
188 File file;
189 file.name = splits[0].as_string();
190 file.extents = {ExtentForBytes(kBlockSize, start, cur_offset - start)};
Amin Hassani1a200c12020-02-26 14:47:23 -0800191 file.is_compressed = is_compressed;
Amin Hassanid7da8f42017-08-23 14:29:40 -0700192 files_.emplace_back(file);
193 }
194 }
195
196 // Sort all files by their offset in the squashfs.
197 std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
198 return a.extents[0].start_block() < b.extents[0].start_block();
199 });
200 // If there is any overlap between two consecutive extents, remove them. Here
201 // we are assuming all files have exactly one extent. If this assumption
202 // changes then this implementation needs to change too.
203 for (auto first = files_.begin(), second = first + 1;
204 first != files_.end() && second != files_.end();
205 second = first + 1) {
206 auto first_begin = first->extents[0].start_block();
207 auto first_end = first_begin + first->extents[0].num_blocks();
208 auto second_begin = second->extents[0].start_block();
209 auto second_end = second_begin + second->extents[0].num_blocks();
210 // Remove the first file if the size is zero.
211 if (first_end == first_begin) {
212 first = files_.erase(first);
213 } else if (first_end > second_begin) { // We found a collision.
214 if (second_end <= first_end) {
215 // Second file is inside the first file, remove the second file.
216 second = files_.erase(second);
217 } else if (first_begin == second_begin) {
218 // First file is inside the second file, remove the first file.
219 first = files_.erase(first);
220 } else {
221 // Remove overlapping extents from the first file.
222 first->extents[0].set_num_blocks(second_begin - first_begin);
223 ++first;
224 }
225 } else {
226 ++first;
227 }
228 }
229
230 // Find all the metadata including superblock and add them to the list of
231 // files.
232 ExtentRanges file_extents;
233 for (const auto& file : files_) {
234 file_extents.AddExtents(file.extents);
235 }
Sen Jiang0a582fb2018-06-26 19:27:21 -0700236 vector<Extent> full = {ExtentForBytes(kBlockSize, 0, size_)};
Amin Hassanid7da8f42017-08-23 14:29:40 -0700237 auto metadata_extents = FilterExtentRanges(full, file_extents);
238 // For now there should be at most two extents. One for superblock and one for
239 // metadata at the end. Just create appropriate files with <metadata-i> name.
240 // We can add all these extents as one metadata too, but that violates the
241 // contiguous write optimization.
242 for (size_t i = 0; i < metadata_extents.size(); i++) {
243 File file;
244 file.name = "<metadata-" + std::to_string(i) + ">";
245 file.extents = {metadata_extents[i]};
246 files_.emplace_back(file);
247 }
248
249 // Do one last sort before returning.
250 std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) {
251 return a.extents[0].start_block() < b.extents[0].start_block();
252 });
Amin Hassani3cd4df12017-08-25 11:21:53 -0700253
254 if (is_zlib && extract_deflates) {
255 // If it is infact gzipped, then the sqfs_path should be valid to read its
256 // content.
257 TEST_AND_RETURN_FALSE(!sqfs_path.empty());
258 if (zlib_blks.empty()) {
259 return true;
260 }
261
262 // Sort zlib blocks.
263 std::sort(zlib_blks.begin(),
264 zlib_blks.end(),
265 [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) {
266 return a.offset < b.offset;
267 });
268
Amin Hassani5d185052019-04-23 07:28:30 -0700269 // Sometimes a squashfs can have a two files that are hard linked. In this
270 // case both files will have the same starting offset in the image and hence
271 // the same zlib blocks. So we need to remove these duplicates to eliminate
272 // further potential probems. As a matter of fact the next statement will
273 // fail if there are duplicates (there will be overlap between two blocks).
274 auto last = std::unique(zlib_blks.begin(), zlib_blks.end());
275 zlib_blks.erase(last, zlib_blks.end());
276
Amin Hassani3cd4df12017-08-25 11:21:53 -0700277 // Sanity check. Make sure zlib blocks are not overlapping.
278 auto result = std::adjacent_find(
279 zlib_blks.begin(),
280 zlib_blks.end(),
281 [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) {
282 return (a.offset + a.length) > b.offset;
283 });
284 TEST_AND_RETURN_FALSE(result == zlib_blks.end());
285
286 vector<puffin::BitExtent> deflates;
287 TEST_AND_RETURN_FALSE(
288 puffin::LocateDeflatesInZlibBlocks(sqfs_path, zlib_blks, &deflates));
289
290 // Add deflates for each file.
291 for (auto& file : files_) {
292 file.deflates = deflate_utils::FindDeflates(file.extents, deflates);
293 }
294 }
Amin Hassanid7da8f42017-08-23 14:29:40 -0700295 return true;
296}
297
298unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile(
Amin Hassani77c25fc2019-01-29 10:24:19 -0800299 const string& sqfs_path, bool extract_deflates, bool load_settings) {
Amin Hassanid7da8f42017-08-23 14:29:40 -0700300 if (sqfs_path.empty())
301 return nullptr;
302
303 brillo::StreamPtr sqfs_file =
Amin Hassani77c25fc2019-01-29 10:24:19 -0800304 brillo::FileStream::Open(FilePath(sqfs_path),
Amin Hassanid7da8f42017-08-23 14:29:40 -0700305 brillo::Stream::AccessMode::READ,
306 brillo::FileStream::Disposition::OPEN_EXISTING,
307 nullptr);
308 if (!sqfs_file) {
309 LOG(ERROR) << "Unable to open " << sqfs_path << " for reading.";
310 return nullptr;
311 }
312
313 SquashfsHeader header;
314 brillo::Blob blob(kSquashfsSuperBlockSize);
315 if (!sqfs_file->ReadAllBlocking(blob.data(), blob.size(), nullptr)) {
316 LOG(ERROR) << "Unable to read from file: " << sqfs_path;
317 return nullptr;
318 }
319 if (!ReadSquashfsHeader(blob, &header) || !CheckHeader(header)) {
320 // This is not necessary an error.
321 return nullptr;
322 }
323
324 // Read the map file.
325 string filemap;
326 if (!GetFileMapContent(sqfs_path, &filemap)) {
327 LOG(ERROR) << "Failed to produce squashfs map file: " << sqfs_path;
328 return nullptr;
329 }
330
331 unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
Amin Hassani3cd4df12017-08-25 11:21:53 -0700332 if (!sqfs->Init(
333 filemap, sqfs_path, sqfs_file->GetSize(), header, extract_deflates)) {
Amin Hassanid7da8f42017-08-23 14:29:40 -0700334 LOG(ERROR) << "Failed to initialized the Squashfs file system";
335 return nullptr;
336 }
Amin Hassani3cd4df12017-08-25 11:21:53 -0700337
Amin Hassani77c25fc2019-01-29 10:24:19 -0800338 if (load_settings) {
339 if (!GetUpdateEngineConfig(sqfs_path, &sqfs->update_engine_config_)) {
340 return nullptr;
341 }
342 }
343
Amin Hassanid7da8f42017-08-23 14:29:40 -0700344 return sqfs;
345}
346
347unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFileMap(
348 const string& filemap, size_t size, const SquashfsHeader& header) {
349 if (!CheckHeader(header)) {
350 LOG(ERROR) << "Invalid Squashfs super block!";
351 return nullptr;
352 }
353
354 unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem());
Amin Hassani3cd4df12017-08-25 11:21:53 -0700355 if (!sqfs->Init(filemap, "", size, header, false)) {
Amin Hassanid7da8f42017-08-23 14:29:40 -0700356 LOG(ERROR) << "Failed to initialize the Squashfs file system using filemap";
357 return nullptr;
358 }
359 // TODO(ahassani): Add a function that initializes the puffin related extents.
360 return sqfs;
361}
362
363size_t SquashfsFilesystem::GetBlockSize() const {
364 return kBlockSize;
365}
366
367size_t SquashfsFilesystem::GetBlockCount() const {
368 return size_ / kBlockSize;
369}
370
371bool SquashfsFilesystem::GetFiles(vector<File>* files) const {
372 files->insert(files->end(), files_.begin(), files_.end());
373 return true;
374}
375
376bool SquashfsFilesystem::LoadSettings(brillo::KeyValueStore* store) const {
Amin Hassani77c25fc2019-01-29 10:24:19 -0800377 if (!store->LoadFromString(update_engine_config_)) {
378 LOG(ERROR) << "Failed to load the settings with config: "
379 << update_engine_config_;
380 return false;
381 }
382 return true;
Amin Hassanid7da8f42017-08-23 14:29:40 -0700383}
384
385bool SquashfsFilesystem::IsSquashfsImage(const brillo::Blob& blob) {
386 SquashfsHeader header;
387 return ReadSquashfsHeader(blob, &header) && CheckHeader(header);
388}
Amin Hassanid7da8f42017-08-23 14:29:40 -0700389} // namespace chromeos_update_engine