blob: 56f946293ddec0b08d6125e8ab12927c7d670493 [file] [log] [blame]
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +01001// Copyright 2022, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Low-level entry and exit points of pvmfw.
16
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +010017use crate::config;
Pierre-Clément Tosi41748ed2023-03-31 18:20:40 +010018use crate::crypto;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000019use crate::fdt;
Alice Wang93ee98a2023-06-08 08:20:39 +000020use crate::memory;
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +010021use core::arch::asm;
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +010022use core::mem::{drop, size_of};
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000023use core::num::NonZeroUsize;
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +010024use core::ops::Range;
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010025use core::slice;
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070026use hyp::{get_hypervisor, HypervisorCap};
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +010027use log::debug;
28use log::error;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000029use log::info;
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +010030use log::warn;
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +010031use log::LevelFilter;
Alice Wang4be4dd02023-06-07 07:50:40 +000032use vmbase::util::RangeExt as _;
Alice Wangeacb7382023-06-05 12:53:54 +000033use vmbase::{
Pierre-Clément Tosic332fae2023-06-22 11:37:12 +000034 configure_global_allocator_size, console, heap,
Alice Wang4c70d142023-06-06 11:52:33 +000035 layout::{self, crosvm},
36 logger, main,
Pierre-Clément Tosif3681e82023-06-22 11:38:22 +000037 memory::{min_dcache_line_size, MemoryTracker, MEMORY, SIZE_128KB, SIZE_4KB},
Alice Wangeacb7382023-06-05 12:53:54 +000038 power::reboot,
Pierre-Clément Tosi3e3d7332023-06-22 10:44:29 +000039 rand,
Alice Wangeacb7382023-06-05 12:53:54 +000040};
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010041use zeroize::Zeroize;
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010042
43#[derive(Debug, Clone)]
Andrew Walbran19690632022-12-07 16:41:30 +000044pub enum RebootReason {
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +010045 /// A malformed BCC was received.
46 InvalidBcc,
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +010047 /// An invalid configuration was appended to pvmfw.
48 InvalidConfig,
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010049 /// An unexpected internal error happened.
50 InternalError,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000051 /// The provided FDT was invalid.
52 InvalidFdt,
53 /// The provided payload was invalid.
54 InvalidPayload,
55 /// The provided ramdisk was invalid.
56 InvalidRamdisk,
Alice Wang28cbcf12022-12-01 07:58:28 +000057 /// Failed to verify the payload.
58 PayloadVerificationError,
Pierre-Clément Tosi4f4f5eb2022-12-08 14:31:42 +000059 /// DICE layering process failed.
60 SecretDerivationError,
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010061}
62
63main!(start);
Pierre-Clément Tosif3681e82023-06-22 11:38:22 +000064configure_global_allocator_size!(SIZE_128KB);
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010065
66/// Entry point for pVM firmware.
67pub fn start(fdt_address: u64, payload_start: u64, payload_size: u64, _arg3: u64) {
68 // Limitations in this function:
69 // - can't access non-pvmfw memory (only statically-mapped memory)
70 // - can't access MMIO (therefore, no logging)
71
Pierre-Clément Tosif03089c2023-05-15 17:33:39 +000072 // SAFETY - This function should and will only be called once, here.
73 unsafe { heap::init() };
74
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010075 match main_wrapper(fdt_address as usize, payload_start as usize, payload_size as usize) {
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +010076 Ok((entry, bcc)) => jump_to_payload(fdt_address, entry.try_into().unwrap(), bcc),
Pierre-Clément Tosid836b5b2022-12-05 10:49:38 +000077 Err(_) => reboot(), // TODO(b/220071963) propagate the reason back to the host.
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +010078 }
79
80 // if we reach this point and return, vmbase::entry::rust_entry() will call power::shutdown().
81}
82
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000083struct MemorySlices<'a> {
84 fdt: &'a mut libfdt::Fdt,
85 kernel: &'a [u8],
86 ramdisk: Option<&'a [u8]>,
87}
88
89impl<'a> MemorySlices<'a> {
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010090 fn new(fdt: usize, kernel: usize, kernel_size: usize) -> Result<Self, RebootReason> {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000091 // SAFETY - SIZE_2MB is non-zero.
Alice Wange9ae5c42023-06-12 11:14:13 +000092 const FDT_SIZE: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(crosvm::FDT_MAX_SIZE) };
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000093 // TODO - Only map the FDT as read-only, until we modify it right before jump_to_payload()
94 // e.g. by generating a DTBO for a template DT in main() and, on return, re-map DT as RW,
95 // overwrite with the template DT and apply the DTBO.
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010096 let range = MEMORY.lock().as_mut().unwrap().alloc_mut(fdt, FDT_SIZE).map_err(|e| {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000097 error!("Failed to allocate the FDT range: {e}");
98 RebootReason::InternalError
99 })?;
100
101 // SAFETY - The tracker validated the range to be in main memory, mapped, and not overlap.
102 let fdt = unsafe { slice::from_raw_parts_mut(range.start as *mut u8, range.len()) };
103 let fdt = libfdt::Fdt::from_mut_slice(fdt).map_err(|e| {
104 error!("Failed to spawn the FDT wrapper: {e}");
105 RebootReason::InvalidFdt
106 })?;
107
Jiyong Park6a8789a2023-03-21 14:50:59 +0900108 let info = fdt::sanitize_device_tree(fdt)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000109 debug!("Fdt passed validation!");
110
Jiyong Park6a8789a2023-03-21 14:50:59 +0900111 let memory_range = info.memory_range;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000112 debug!("Resizing MemoryTracker to range {memory_range:#x?}");
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100113 MEMORY.lock().as_mut().unwrap().shrink(&memory_range).map_err(|e| {
114 error!("Failed to use memory range value from DT: {memory_range:#x?}: {e}");
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000115 RebootReason::InvalidFdt
116 })?;
117
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000118 if get_hypervisor().has_cap(HypervisorCap::DYNAMIC_MEM_SHARE) {
Alice Wangb6d2c642023-06-13 13:07:06 +0000119 let granule = get_hypervisor().memory_protection_granule().map_err(|e| {
120 error!("Failed to get memory protection granule: {e}");
121 RebootReason::InternalError
122 })?;
123 MEMORY.lock().as_mut().unwrap().init_dynamic_shared_pool(granule).map_err(|e| {
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000124 error!("Failed to initialize dynamically shared pool: {e}");
125 RebootReason::InternalError
126 })?;
127 } else {
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700128 let range = info.swiotlb_info.fixed_range().ok_or_else(|| {
129 error!("Pre-shared pool range not specified in swiotlb node");
130 RebootReason::InvalidFdt
131 })?;
132
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100133 MEMORY.lock().as_mut().unwrap().init_static_shared_pool(range).map_err(|e| {
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700134 error!("Failed to initialize pre-shared pool {e}");
135 RebootReason::InvalidFdt
136 })?;
137 }
138
Jiyong Park6a8789a2023-03-21 14:50:59 +0900139 let kernel_range = if let Some(r) = info.kernel_range {
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100140 MEMORY.lock().as_mut().unwrap().alloc_range(&r).map_err(|e| {
Pierre-Clément Tosic3811b82022-11-29 11:24:16 +0000141 error!("Failed to obtain the kernel range with DT range: {e}");
142 RebootReason::InternalError
143 })?
144 } else if cfg!(feature = "legacy") {
145 warn!("Failed to find the kernel range in the DT; falling back to legacy ABI");
146
147 let kernel_size = NonZeroUsize::new(kernel_size).ok_or_else(|| {
148 error!("Invalid kernel size: {kernel_size:#x}");
149 RebootReason::InvalidPayload
150 })?;
151
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100152 MEMORY.lock().as_mut().unwrap().alloc(kernel, kernel_size).map_err(|e| {
Pierre-Clément Tosic3811b82022-11-29 11:24:16 +0000153 error!("Failed to obtain the kernel range with legacy range: {e}");
154 RebootReason::InternalError
155 })?
156 } else {
157 error!("Failed to locate the kernel from the DT");
158 return Err(RebootReason::InvalidPayload);
159 };
160
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000161 // SAFETY - The tracker validated the range to be in main memory, mapped, and not overlap.
162 let kernel =
Pierre-Clément Tosic3811b82022-11-29 11:24:16 +0000163 unsafe { slice::from_raw_parts(kernel_range.start as *const u8, kernel_range.len()) };
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000164
Jiyong Park6a8789a2023-03-21 14:50:59 +0900165 let ramdisk = if let Some(r) = info.initrd_range {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000166 debug!("Located ramdisk at {r:?}");
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100167 let r = MEMORY.lock().as_mut().unwrap().alloc_range(&r).map_err(|e| {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000168 error!("Failed to obtain the initrd range: {e}");
169 RebootReason::InvalidRamdisk
170 })?;
171
172 // SAFETY - The region was validated by memory to be in main memory, mapped, and
173 // not overlap.
174 Some(unsafe { slice::from_raw_parts(r.start as *const u8, r.len()) })
175 } else {
176 info!("Couldn't locate the ramdisk from the device tree");
177 None
178 };
179
180 Ok(Self { fdt, kernel, ramdisk })
181 }
182}
183
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +0100184/// Sets up the environment for main() and wraps its result for start().
185///
186/// Provide the abstractions necessary for start() to abort the pVM boot and for main() to run with
187/// the assumption that its environment has been properly configured.
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100188fn main_wrapper(
189 fdt: usize,
190 payload: usize,
191 payload_size: usize,
192) -> Result<(usize, Range<usize>), RebootReason> {
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +0100193 // Limitations in this function:
194 // - only access MMIO once (and while) it has been mapped and configured
195 // - only perform logging once the logger has been initialized
196 // - only access non-pvmfw memory once (and while) it has been mapped
Pierre-Clément Tosifc531152022-10-20 12:22:23 +0100197
Pierre-Clément Tosidbd72862022-10-21 14:31:02 +0100198 logger::init(LevelFilter::Info).map_err(|_| RebootReason::InternalError)?;
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +0100199
Pierre-Clément Tosi072969b2022-10-19 17:32:24 +0100200 // Use debug!() to avoid printing to the UART if we failed to configure it as only local
201 // builds that have tweaked the logger::init() call will actually attempt to log the message.
202
Alice Wang90e6f162023-04-17 13:49:45 +0000203 get_hypervisor().mmio_guard_init().map_err(|e| {
Pierre-Clément Tosi072969b2022-10-19 17:32:24 +0100204 debug!("{e}");
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +0100205 RebootReason::InternalError
206 })?;
207
Alice Wang90e6f162023-04-17 13:49:45 +0000208 get_hypervisor().mmio_guard_map(console::BASE_ADDRESS).map_err(|e| {
Pierre-Clément Tosi072969b2022-10-19 17:32:24 +0100209 debug!("Failed to configure the UART: {e}");
210 RebootReason::InternalError
211 })?;
212
Pierre-Clément Tosi41748ed2023-03-31 18:20:40 +0100213 crypto::init();
214
Alice Wang807fa592023-06-02 09:54:43 +0000215 let page_table = memory::init_page_table().map_err(|e| {
Pierre-Clément Tosia8a4a202022-11-03 14:16:46 +0000216 error!("Failed to set up the dynamic page tables: {e}");
217 RebootReason::InternalError
218 })?;
Alan Stokesc3829f12023-06-02 15:02:23 +0100219
220 // SAFETY - We only get the appended payload from here, once. The region was statically mapped,
221 // then remapped by `init_page_table()`.
222 let appended_data = unsafe { get_appended_data_slice() };
223
224 let mut appended = AppendedPayload::new(appended_data).ok_or_else(|| {
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100225 error!("No valid configuration found");
226 RebootReason::InvalidConfig
Pierre-Clément Tosia8a4a202022-11-03 14:16:46 +0000227 })?;
228
Pierre-Clément Tosiefe780c2023-02-21 21:36:30 +0000229 let (bcc_slice, debug_policy) = appended.get_entries();
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100230
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100231 // Up to this point, we were using the built-in static (from .rodata) page tables.
Alice Wang4c70d142023-06-06 11:52:33 +0000232 MEMORY.lock().replace(MemoryTracker::new(
233 page_table,
Alice Wang63f4c9e2023-06-12 09:36:43 +0000234 crosvm::MEM_START..layout::MAX_VIRT_ADDR,
Alice Wang89d29592023-06-12 09:41:29 +0000235 crosvm::MMIO_RANGE,
Alice Wang5bb79502023-06-12 09:25:07 +0000236 Some(memory::appended_payload_range()),
Alice Wang4c70d142023-06-06 11:52:33 +0000237 ));
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100238
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100239 let slices = MemorySlices::new(fdt, payload, payload_size)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000240
Pierre-Clément Tosia59103d2023-02-02 14:46:55 +0000241 rand::init().map_err(|e| {
242 error!("Failed to initialize rand: {e}");
243 RebootReason::InternalError
244 })?;
245
Pierre-Clément Tosi072969b2022-10-19 17:32:24 +0100246 // This wrapper allows main() to be blissfully ignorant of platform details.
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100247 let next_bcc = crate::main(slices.fdt, slices.kernel, slices.ramdisk, bcc_slice, debug_policy)?;
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100248
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100249 // Writable-dirty regions will be flushed when MemoryTracker is dropped.
250 bcc_slice.zeroize();
Pierre-Clément Tosi072969b2022-10-19 17:32:24 +0100251
Andrew Walbran19690632022-12-07 16:41:30 +0000252 info!("Expecting a bug making MMIO_GUARD_UNMAP return NOT_SUPPORTED on success");
Jakob Vukalovic85a00d72023-04-20 09:51:10 +0100253 MEMORY.lock().as_mut().unwrap().mmio_unmap_all().map_err(|e| {
Andrew Walbran19690632022-12-07 16:41:30 +0000254 error!("Failed to unshare MMIO ranges: {e}");
255 RebootReason::InternalError
256 })?;
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000257 // Call unshare_all_memory here (instead of relying on the dtor) while UART is still mapped.
258 MEMORY.lock().as_mut().unwrap().unshare_all_memory();
Alice Wang90e6f162023-04-17 13:49:45 +0000259 get_hypervisor().mmio_guard_unmap(console::BASE_ADDRESS).map_err(|e| {
Pierre-Clément Tosia99bfa62022-10-06 13:30:52 +0100260 error!("Failed to unshare the UART: {e}");
261 RebootReason::InternalError
262 })?;
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100263
264 // Drop MemoryTracker and deactivate page table.
265 drop(MEMORY.lock().take());
Pierre-Clément Tosia99bfa62022-10-06 13:30:52 +0100266
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100267 Ok((slices.kernel.as_ptr() as usize, next_bcc))
Pierre-Clément Tosi5bbfca52022-10-21 12:14:35 +0100268}
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100269
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100270fn jump_to_payload(fdt_address: u64, payload_start: u64, bcc: Range<usize>) -> ! {
271 const ASM_STP_ALIGN: usize = size_of::<u64>() * 2;
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000272 const SCTLR_EL1_RES1: u64 = (0b11 << 28) | (0b101 << 20) | (0b1 << 11);
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100273 // Stage 1 instruction access cacheability is unaffected.
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000274 const SCTLR_EL1_I: u64 = 0b1 << 12;
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100275 // SETEND instruction disabled at EL0 in aarch32 mode.
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000276 const SCTLR_EL1_SED: u64 = 0b1 << 8;
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100277 // Various IT instructions are disabled at EL0 in aarch32 mode.
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000278 const SCTLR_EL1_ITD: u64 = 0b1 << 7;
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100279
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000280 const SCTLR_EL1_VAL: u64 = SCTLR_EL1_RES1 | SCTLR_EL1_ITD | SCTLR_EL1_SED | SCTLR_EL1_I;
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100281
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100282 let scratch = layout::scratch_range();
283
284 assert_ne!(scratch.len(), 0, "scratch memory is empty.");
285 assert_eq!(scratch.start % ASM_STP_ALIGN, 0, "scratch memory is misaligned.");
286 assert_eq!(scratch.end % ASM_STP_ALIGN, 0, "scratch memory is misaligned.");
287
288 assert!(bcc.is_within(&scratch));
289 assert_eq!(bcc.start % ASM_STP_ALIGN, 0, "Misaligned guest BCC.");
290 assert_eq!(bcc.end % ASM_STP_ALIGN, 0, "Misaligned guest BCC.");
291
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000292 let stack = memory::stack_range();
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100293
294 assert_ne!(stack.len(), 0, "stack region is empty.");
295 assert_eq!(stack.start % ASM_STP_ALIGN, 0, "Misaligned stack region.");
296 assert_eq!(stack.end % ASM_STP_ALIGN, 0, "Misaligned stack region.");
297
298 // Zero all memory that could hold secrets and that can't be safely written to from Rust.
Pierre-Clément Tosi6c0d48b2022-11-07 11:00:32 +0000299 // Disable the exception vector, caches and page table and then jump to the payload at the
300 // given address, passing it the given FDT pointer.
301 //
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100302 // SAFETY - We're exiting pvmfw by passing the register values we need to a noreturn asm!().
303 unsafe {
304 asm!(
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100305 "cmp {scratch}, {bcc}",
306 "b.hs 1f",
307
308 // Zero .data & .bss until BCC.
309 "0: stp xzr, xzr, [{scratch}], 16",
310 "cmp {scratch}, {bcc}",
311 "b.lo 0b",
312
313 "1:",
314 // Skip BCC.
315 "mov {scratch}, {bcc_end}",
316 "cmp {scratch}, {scratch_end}",
317 "b.hs 1f",
318
319 // Keep zeroing .data & .bss.
320 "0: stp xzr, xzr, [{scratch}], 16",
321 "cmp {scratch}, {scratch_end}",
322 "b.lo 0b",
323
324 "1:",
325 // Flush d-cache over .data & .bss (including BCC).
326 "0: dc cvau, {cache_line}",
327 "add {cache_line}, {cache_line}, {dcache_line_size}",
328 "cmp {cache_line}, {scratch_end}",
329 "b.lo 0b",
330
331 "mov {cache_line}, {stack}",
332 // Zero stack region.
333 "0: stp xzr, xzr, [{stack}], 16",
334 "cmp {stack}, {stack_end}",
335 "b.lo 0b",
336
337 // Flush d-cache over stack region.
338 "0: dc cvau, {cache_line}",
339 "add {cache_line}, {cache_line}, {dcache_line_size}",
340 "cmp {cache_line}, {stack_end}",
341 "b.lo 0b",
342
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100343 "msr sctlr_el1, {sctlr_el1_val}",
344 "isb",
345 "mov x1, xzr",
346 "mov x2, xzr",
347 "mov x3, xzr",
348 "mov x4, xzr",
349 "mov x5, xzr",
350 "mov x6, xzr",
351 "mov x7, xzr",
352 "mov x8, xzr",
353 "mov x9, xzr",
354 "mov x10, xzr",
355 "mov x11, xzr",
356 "mov x12, xzr",
357 "mov x13, xzr",
358 "mov x14, xzr",
359 "mov x15, xzr",
360 "mov x16, xzr",
361 "mov x17, xzr",
362 "mov x18, xzr",
363 "mov x19, xzr",
364 "mov x20, xzr",
365 "mov x21, xzr",
366 "mov x22, xzr",
367 "mov x23, xzr",
368 "mov x24, xzr",
369 "mov x25, xzr",
370 "mov x26, xzr",
371 "mov x27, xzr",
372 "mov x28, xzr",
373 "mov x29, xzr",
374 "msr ttbr0_el1, xzr",
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100375 // Ensure that CMOs have completed before entering payload.
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100376 "dsb nsh",
377 "br x30",
378 sctlr_el1_val = in(reg) SCTLR_EL1_VAL,
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100379 bcc = in(reg) u64::try_from(bcc.start).unwrap(),
380 bcc_end = in(reg) u64::try_from(bcc.end).unwrap(),
381 cache_line = in(reg) u64::try_from(scratch.start).unwrap(),
382 scratch = in(reg) u64::try_from(scratch.start).unwrap(),
383 scratch_end = in(reg) u64::try_from(scratch.end).unwrap(),
384 stack = in(reg) u64::try_from(stack.start).unwrap(),
385 stack_end = in(reg) u64::try_from(stack.end).unwrap(),
Alice Wang3fa9b802023-06-06 07:52:31 +0000386 dcache_line_size = in(reg) u64::try_from(min_dcache_line_size()).unwrap(),
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100387 in("x0") fdt_address,
388 in("x30") payload_start,
Pierre-Clément Tosi97f52492023-04-04 15:52:17 +0100389 options(noreturn),
Pierre-Clément Tosi645e90e2022-10-21 13:27:19 +0100390 );
391 };
392}
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100393
Alan Stokesc3829f12023-06-02 15:02:23 +0100394/// # Safety
395///
396/// This must only be called once, since we are returning a mutable reference.
397/// The appended data region must be mapped.
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100398unsafe fn get_appended_data_slice() -> &'static mut [u8] {
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000399 let range = memory::appended_payload_range();
Alan Stokesa0e42962023-04-14 17:59:50 +0100400 // SAFETY: This region is mapped and the linker script prevents it from overlapping with other
401 // objects.
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100402 unsafe { slice::from_raw_parts_mut(range.start as *mut u8, range.len()) }
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100403}
404
Pierre-Clément Tosi7aca7ff2022-12-12 14:04:30 +0000405enum AppendedConfigType {
406 Valid,
407 Invalid,
408 NotFound,
409}
410
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100411enum AppendedPayload<'a> {
412 /// Configuration data.
413 Config(config::Config<'a>),
414 /// Deprecated raw BCC, as used in Android T.
415 LegacyBcc(&'a mut [u8]),
416}
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100417
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100418impl<'a> AppendedPayload<'a> {
Alan Stokesc3829f12023-06-02 15:02:23 +0100419 fn new(data: &'a mut [u8]) -> Option<Self> {
420 match Self::guess_config_type(data) {
Alan Stokesa0e42962023-04-14 17:59:50 +0100421 AppendedConfigType::Valid => {
Alan Stokesc3829f12023-06-02 15:02:23 +0100422 let config = config::Config::new(data);
Alan Stokesa0e42962023-04-14 17:59:50 +0100423 Some(Self::Config(config.unwrap()))
424 }
Pierre-Clément Tosi7aca7ff2022-12-12 14:04:30 +0000425 AppendedConfigType::NotFound if cfg!(feature = "legacy") => {
Alice Wangeacb7382023-06-05 12:53:54 +0000426 const BCC_SIZE: usize = SIZE_4KB;
Pierre-Clément Tosi7aca7ff2022-12-12 14:04:30 +0000427 warn!("Assuming the appended data at {:?} to be a raw BCC", data.as_ptr());
428 Some(Self::LegacyBcc(&mut data[..BCC_SIZE]))
429 }
430 _ => None,
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100431 }
432 }
433
Alan Stokesc3829f12023-06-02 15:02:23 +0100434 fn guess_config_type(data: &mut [u8]) -> AppendedConfigType {
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100435 // This function is necessary to prevent the borrow checker from getting confused
436 // about the ownership of data in new(); see https://users.rust-lang.org/t/78467.
437 let addr = data.as_ptr();
Alan Stokesa0e42962023-04-14 17:59:50 +0100438
Alan Stokesc3829f12023-06-02 15:02:23 +0100439 match config::Config::new(data) {
Pierre-Clément Tosi7aca7ff2022-12-12 14:04:30 +0000440 Err(config::Error::InvalidMagic) => {
441 warn!("No configuration data found at {addr:?}");
442 AppendedConfigType::NotFound
443 }
444 Err(e) => {
445 error!("Invalid configuration data at {addr:?}: {e}");
446 AppendedConfigType::Invalid
447 }
448 Ok(_) => AppendedConfigType::Valid,
449 }
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100450 }
451
Pierre-Clément Tosiefe780c2023-02-21 21:36:30 +0000452 fn get_entries(&mut self) -> (&mut [u8], Option<&mut [u8]>) {
Pierre-Clément Tosi20b60962022-10-17 13:35:27 +0100453 match self {
Pierre-Clément Tosiefe780c2023-02-21 21:36:30 +0000454 Self::Config(ref mut cfg) => cfg.get_entries(),
455 Self::LegacyBcc(ref mut bcc) => (bcc, None),
Pierre-Clément Tosi8edf72e2022-12-06 16:02:57 +0000456 }
Pierre-Clément Tosie8726e42022-10-17 13:35:27 +0100457 }
458}