blob: 1467611a02fcf43d49362a98e372d97ae5c3d82f [file] [log] [blame]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +00001// Copyright 2022, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Low-level allocation and tracking of main memory.
16
Andrew Walbran848decf2022-12-15 14:39:38 +000017#![deny(unsafe_op_in_unsafe_fn)]
18
Alice Wangeacb7382023-06-05 12:53:54 +000019use crate::helpers::{self, RangeExt, PVMFW_PAGE_SIZE};
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000020use aarch64_paging::idmap::IdMap;
Jakob Vukalovicb99905d2023-04-20 15:46:02 +010021use aarch64_paging::paging::{Attributes, Descriptor, MemoryRegion as VaRange};
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000022use aarch64_paging::MapError;
Andrew Walbran848decf2022-12-15 14:39:38 +000023use alloc::alloc::handle_alloc_error;
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070024use alloc::boxed::Box;
Alice Wangf47b2342023-06-02 11:51:57 +000025use buddy_system_allocator::LockedFrameAllocator;
Andrew Walbran848decf2022-12-15 14:39:38 +000026use core::alloc::Layout;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000027use core::cmp::max;
28use core::cmp::min;
29use core::fmt;
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010030use core::iter::once;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000031use core::num::NonZeroUsize;
32use core::ops::Range;
Andrew Walbran848decf2022-12-15 14:39:38 +000033use core::ptr::NonNull;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000034use core::result;
Alice Wang90e6f162023-04-17 13:49:45 +000035use hyp::get_hypervisor;
Pierre-Clément Tosi90238c52023-04-27 17:59:10 +000036use log::trace;
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +010037use log::{debug, error};
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070038use once_cell::race::OnceBox;
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010039use spin::mutex::SpinMutex;
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000040use tinyvec::ArrayVec;
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000041use vmbase::{
42 dsb, isb, layout,
Alice Wangeacb7382023-06-05 12:53:54 +000043 memory::{
44 page_4kb_of, set_dbm_enabled, MemorySharer, PageTable, MMIO_LAZY_MAP_FLAG, SIZE_2MB,
45 SIZE_4KB, SIZE_4MB,
46 },
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000047 tlbi,
Alice Wangeacb7382023-06-05 12:53:54 +000048 util::align_up,
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000049};
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000050
Jiyong Park0ee65392023-03-27 20:52:45 +090051/// Base of the system's contiguous "main" memory.
52pub const BASE_ADDR: usize = 0x8000_0000;
53/// First address that can't be translated by a level 1 TTBR0_EL1.
54pub const MAX_ADDR: usize = 1 << 40;
55
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000056const PT_ROOT_LEVEL: usize = 1;
57const PT_ASID: usize = 1;
58
Andrew Walbran0d8b54d2022-12-08 16:32:33 +000059pub type MemoryRange = Range<usize>;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000060
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010061pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
62unsafe impl Send for MemoryTracker {}
63
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010064#[derive(Clone, Copy, Debug, Default, PartialEq)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000065enum MemoryType {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000066 #[default]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000067 ReadOnly,
68 ReadWrite,
69}
70
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000071#[derive(Clone, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000072struct MemoryRegion {
73 range: MemoryRange,
74 mem_type: MemoryType,
75}
76
77impl MemoryRegion {
78 /// True if the instance overlaps with the passed range.
79 pub fn overlaps(&self, range: &MemoryRange) -> bool {
Andrew Walbran19690632022-12-07 16:41:30 +000080 overlaps(&self.range, range)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000081 }
82
83 /// True if the instance is fully contained within the passed range.
84 pub fn is_within(&self, range: &MemoryRange) -> bool {
Srivatsa Vaddagiric25d68e2023-04-19 22:56:33 -070085 self.as_ref().is_within(range)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000086 }
87}
88
89impl AsRef<MemoryRange> for MemoryRegion {
90 fn as_ref(&self) -> &MemoryRange {
91 &self.range
92 }
93}
94
Andrew Walbran19690632022-12-07 16:41:30 +000095/// Returns true if one range overlaps with the other at all.
96fn overlaps<T: Copy + Ord>(a: &Range<T>, b: &Range<T>) -> bool {
97 max(a.start, b.start) < min(a.end, b.end)
98}
99
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000100/// Tracks non-overlapping slices of main memory.
101pub struct MemoryTracker {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000102 total: MemoryRange,
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000103 page_table: PageTable,
Andrew Walbran19690632022-12-07 16:41:30 +0000104 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
105 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000106}
107
108/// Errors for MemoryTracker operations.
109#[derive(Debug, Clone)]
110pub enum MemoryTrackerError {
111 /// Tried to modify the memory base address.
112 DifferentBaseAddress,
113 /// Tried to shrink to a larger memory size.
114 SizeTooLarge,
115 /// Tracked regions would not fit in memory size.
116 SizeTooSmall,
117 /// Reached limit number of tracked regions.
118 Full,
119 /// Region is out of the tracked memory address space.
120 OutOfRange,
121 /// New region overlaps with tracked regions.
122 Overlaps,
123 /// Region couldn't be mapped.
124 FailedToMap,
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100125 /// Region couldn't be unmapped.
126 FailedToUnmap,
Alice Wang90e6f162023-04-17 13:49:45 +0000127 /// Error from the interaction with the hypervisor.
128 Hypervisor(hyp::Error),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000129 /// Failure to set `SHARED_MEMORY`.
130 SharedMemorySetFailure,
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700131 /// Failure to set `SHARED_POOL`.
132 SharedPoolSetFailure,
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100133 /// Invalid page table entry.
134 InvalidPte,
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100135 /// Failed to flush memory region.
136 FlushRegionFailed,
137 /// Failed to set PTE dirty state.
138 SetPteDirtyFailed,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000139}
140
141impl fmt::Display for MemoryTrackerError {
142 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
143 match self {
144 Self::DifferentBaseAddress => write!(f, "Received different base address"),
145 Self::SizeTooLarge => write!(f, "Tried to shrink to a larger memory size"),
146 Self::SizeTooSmall => write!(f, "Tracked regions would not fit in memory size"),
147 Self::Full => write!(f, "Reached limit number of tracked regions"),
148 Self::OutOfRange => write!(f, "Region is out of the tracked memory address space"),
149 Self::Overlaps => write!(f, "New region overlaps with tracked regions"),
150 Self::FailedToMap => write!(f, "Failed to map the new region"),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100151 Self::FailedToUnmap => write!(f, "Failed to unmap the new region"),
Alice Wang90e6f162023-04-17 13:49:45 +0000152 Self::Hypervisor(e) => e.fmt(f),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000153 Self::SharedMemorySetFailure => write!(f, "Failed to set SHARED_MEMORY"),
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700154 Self::SharedPoolSetFailure => write!(f, "Failed to set SHARED_POOL"),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100155 Self::InvalidPte => write!(f, "Page table entry is not valid"),
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100156 Self::FlushRegionFailed => write!(f, "Failed to flush memory region"),
157 Self::SetPteDirtyFailed => write!(f, "Failed to set PTE dirty state"),
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000158 }
159 }
160}
161
Alice Wang90e6f162023-04-17 13:49:45 +0000162impl From<hyp::Error> for MemoryTrackerError {
163 fn from(e: hyp::Error) -> Self {
164 Self::Hypervisor(e)
Andrew Walbran19690632022-12-07 16:41:30 +0000165 }
166}
167
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000168type Result<T> = result::Result<T, MemoryTrackerError>;
169
Andrew Walbran87933f32023-05-09 15:29:06 +0000170static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000171static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
172
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000173impl MemoryTracker {
174 const CAPACITY: usize = 5;
Andrew Walbran19690632022-12-07 16:41:30 +0000175 const MMIO_CAPACITY: usize = 5;
Pierre-Clément Tosi164a6f52023-04-18 19:29:11 +0100176 const PVMFW_RANGE: MemoryRange = (BASE_ADDR - SIZE_4MB)..BASE_ADDR;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000177
178 /// Create a new instance from an active page table, covering the maximum RAM size.
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000179 pub fn new(mut page_table: PageTable) -> Self {
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100180 // Activate dirty state management first, otherwise we may get permission faults immediately
181 // after activating the new page table. This has no effect before the new page table is
182 // activated because none of the entries in the initial idmap have the DBM flag.
Alice Wang4dd20932023-05-26 13:47:16 +0000183 set_dbm_enabled(true);
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100184
185 debug!("Activating dynamic page table...");
186 // SAFETY - page_table duplicates the static mappings for everything that the Rust code is
187 // aware of so activating it shouldn't have any visible effect.
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000188 unsafe { page_table.activate() }
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100189 debug!("... Success!");
190
Andrew Walbran19690632022-12-07 16:41:30 +0000191 Self {
Jiyong Park0ee65392023-03-27 20:52:45 +0900192 total: BASE_ADDR..MAX_ADDR,
Andrew Walbran19690632022-12-07 16:41:30 +0000193 page_table,
194 regions: ArrayVec::new(),
195 mmio_regions: ArrayVec::new(),
196 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000197 }
198
199 /// Resize the total RAM size.
200 ///
201 /// This function fails if it contains regions that are not included within the new size.
202 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
203 if range.start != self.total.start {
204 return Err(MemoryTrackerError::DifferentBaseAddress);
205 }
206 if self.total.end < range.end {
207 return Err(MemoryTrackerError::SizeTooLarge);
208 }
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000209 if !self.regions.iter().all(|r| r.is_within(range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000210 return Err(MemoryTrackerError::SizeTooSmall);
211 }
212
213 self.total = range.clone();
214 Ok(())
215 }
216
217 /// Allocate the address range for a const slice; returns None if failed.
218 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000219 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
220 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000221 self.page_table.map_rodata(range).map_err(|e| {
222 error!("Error during range allocation: {e}");
223 MemoryTrackerError::FailedToMap
224 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000225 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000226 }
227
228 /// Allocate the address range for a mutable slice; returns None if failed.
229 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000230 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
231 self.check(&region)?;
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000232 self.page_table.map_data_dbm(range).map_err(|e| {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000233 error!("Error during mutable range allocation: {e}");
234 MemoryTrackerError::FailedToMap
235 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000236 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000237 }
238
239 /// Allocate the address range for a const slice; returns None if failed.
240 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
241 self.alloc_range(&(base..(base + size.get())))
242 }
243
244 /// Allocate the address range for a mutable slice; returns None if failed.
245 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
246 self.alloc_range_mut(&(base..(base + size.get())))
247 }
248
Andrew Walbran19690632022-12-07 16:41:30 +0000249 /// Checks that the given range of addresses is within the MMIO region, and then maps it
250 /// appropriately.
251 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
252 // MMIO space is below the main memory region.
Pierre-Clément Tosi164a6f52023-04-18 19:29:11 +0100253 if range.end > self.total.start || overlaps(&Self::PVMFW_RANGE, &range) {
Andrew Walbran19690632022-12-07 16:41:30 +0000254 return Err(MemoryTrackerError::OutOfRange);
255 }
256 if self.mmio_regions.iter().any(|r| overlaps(r, &range)) {
257 return Err(MemoryTrackerError::Overlaps);
258 }
259 if self.mmio_regions.len() == self.mmio_regions.capacity() {
260 return Err(MemoryTrackerError::Full);
261 }
262
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100263 self.page_table.map_device_lazy(&range).map_err(|e| {
Andrew Walbran19690632022-12-07 16:41:30 +0000264 error!("Error during MMIO device mapping: {e}");
265 MemoryTrackerError::FailedToMap
266 })?;
267
Andrew Walbran19690632022-12-07 16:41:30 +0000268 if self.mmio_regions.try_push(range).is_some() {
269 return Err(MemoryTrackerError::Full);
270 }
271
272 Ok(())
273 }
274
Andrew Walbranda65ab12022-12-07 15:10:13 +0000275 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
276 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
277 /// add it.
278 fn check(&self, region: &MemoryRegion) -> Result<()> {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000279 if !region.is_within(&self.total) {
280 return Err(MemoryTrackerError::OutOfRange);
281 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000282 if self.regions.iter().any(|r| r.overlaps(&region.range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000283 return Err(MemoryTrackerError::Overlaps);
284 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000285 if self.regions.len() == self.regions.capacity() {
286 return Err(MemoryTrackerError::Full);
287 }
288 Ok(())
289 }
290
291 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000292 if self.regions.try_push(region).is_some() {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000293 return Err(MemoryTrackerError::Full);
294 }
295
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000296 Ok(self.regions.last().unwrap().as_ref().clone())
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000297 }
Andrew Walbran19690632022-12-07 16:41:30 +0000298
299 /// Unmaps all tracked MMIO regions from the MMIO guard.
300 ///
301 /// Note that they are not unmapped from the page table.
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100302 pub fn mmio_unmap_all(&mut self) -> Result<()> {
303 for range in &self.mmio_regions {
304 self.page_table
305 .modify_range(range, &mmio_guard_unmap_page)
306 .map_err(|_| MemoryTrackerError::FailedToUnmap)?;
Andrew Walbran19690632022-12-07 16:41:30 +0000307 }
Andrew Walbran19690632022-12-07 16:41:30 +0000308 Ok(())
309 }
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700310
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000311 /// Initialize the shared heap to dynamically share memory from the global allocator.
312 pub fn init_dynamic_shared_pool(&mut self) -> Result<()> {
Alice Wangf47b2342023-06-02 11:51:57 +0000313 const INIT_CAP: usize = 10;
314
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000315 let granule = get_hypervisor().memory_protection_granule()?;
Alice Wangf47b2342023-06-02 11:51:57 +0000316 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule, INIT_CAP));
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000317 if previous.is_some() {
318 return Err(MemoryTrackerError::SharedMemorySetFailure);
319 }
320
321 SHARED_POOL
Andrew Walbran87933f32023-05-09 15:29:06 +0000322 .set(Box::new(LockedFrameAllocator::new()))
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000323 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
324
325 Ok(())
326 }
327
328 /// Initialize the shared heap from a static region of memory.
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700329 ///
330 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
331 /// to share its memory with host. Instead they allow host to designate part
332 /// of guest memory as "shared" ahead of guest starting its execution. The
333 /// shared memory region is indicated in swiotlb node. On such platforms use
334 /// a separate heap to allocate buffers that can be shared with host.
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000335 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700336 let size = NonZeroUsize::new(range.len()).unwrap();
337 let range = self.alloc_mut(range.start, size)?;
Andrew Walbran87933f32023-05-09 15:29:06 +0000338 let shared_pool = LockedFrameAllocator::<32>::new();
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700339
Andrew Walbran87933f32023-05-09 15:29:06 +0000340 shared_pool.lock().insert(range);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700341
342 SHARED_POOL
343 .set(Box::new(shared_pool))
344 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
345
346 Ok(())
347 }
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000348
349 /// Unshares any memory that may have been shared.
350 pub fn unshare_all_memory(&mut self) {
351 drop(SHARED_MEMORY.lock().take());
352 }
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100353
354 /// Handles translation fault for blocks flagged for lazy MMIO mapping by enabling the page
355 /// table entry and MMIO guard mapping the block. Breaks apart a block entry if required.
356 pub fn handle_mmio_fault(&mut self, addr: usize) -> Result<()> {
357 let page_range = page_4kb_of(addr)..page_4kb_of(addr) + PVMFW_PAGE_SIZE;
358 self.page_table
359 .modify_range(&page_range, &verify_lazy_mapped_block)
360 .map_err(|_| MemoryTrackerError::InvalidPte)?;
361 get_hypervisor().mmio_guard_map(page_range.start)?;
362 // Maps a single device page, breaking up block mappings if necessary.
363 self.page_table.map_device(&page_range).map_err(|_| MemoryTrackerError::FailedToMap)
364 }
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100365
366 /// Flush all memory regions marked as writable-dirty.
367 fn flush_dirty_pages(&mut self) -> Result<()> {
368 // Collect memory ranges for which dirty state is tracked.
369 let writable_regions =
370 self.regions.iter().filter(|r| r.mem_type == MemoryType::ReadWrite).map(|r| &r.range);
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000371 let payload_range = appended_payload_range();
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100372 // Execute a barrier instruction to ensure all hardware updates to the page table have been
373 // observed before reading PTE flags to determine dirty state.
374 dsb!("ish");
375 // Now flush writable-dirty pages in those regions.
376 for range in writable_regions.chain(once(&payload_range)) {
377 self.page_table
378 .modify_range(range, &flush_dirty_range)
379 .map_err(|_| MemoryTrackerError::FlushRegionFailed)?;
380 }
381 Ok(())
382 }
383
384 /// Handles permission fault for read-only blocks by setting writable-dirty state.
385 /// In general, this should be called from the exception handler when hardware dirty
386 /// state management is disabled or unavailable.
387 pub fn handle_permission_fault(&mut self, addr: usize) -> Result<()> {
388 self.page_table
389 .modify_range(&(addr..addr + 1), &mark_dirty_block)
390 .map_err(|_| MemoryTrackerError::SetPteDirtyFailed)
391 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000392}
393
394impl Drop for MemoryTracker {
395 fn drop(&mut self) {
Alice Wang4dd20932023-05-26 13:47:16 +0000396 set_dbm_enabled(false);
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100397 self.flush_dirty_pages().unwrap();
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100398 self.unshare_all_memory();
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000399 }
400}
Andrew Walbran19690632022-12-07 16:41:30 +0000401
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000402/// Allocates a memory range of at least the given size and alignment that is shared with the host.
403/// Returns a pointer to the buffer.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000404pub fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
405 assert_ne!(layout.size(), 0);
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000406 let Some(buffer) = try_shared_alloc(layout) else {
Andrew Walbran848decf2022-12-15 14:39:38 +0000407 handle_alloc_error(layout);
408 };
409
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000410 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000411 Ok(buffer)
412}
413
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000414fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
415 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
416
Andrew Walbran87933f32023-05-09 15:29:06 +0000417 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
418 Some(NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000419 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
420 shared_memory.refill(&mut shared_pool, layout);
Andrew Walbran87933f32023-05-09 15:29:06 +0000421 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000422 } else {
423 None
424 }
425}
426
Andrew Walbran848decf2022-12-15 14:39:38 +0000427/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
428///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000429/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
Andrew Walbran848decf2022-12-15 14:39:38 +0000430///
431/// # Safety
432///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000433/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
Andrew Walbran848decf2022-12-15 14:39:38 +0000434/// deallocated.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000435pub unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
Andrew Walbran87933f32023-05-09 15:29:06 +0000436 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700437
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000438 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000439 Ok(())
440}
441
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100442/// Checks whether a PTE at given level is a page or block descriptor.
443#[inline]
444fn is_leaf_pte(flags: &Attributes, level: usize) -> bool {
445 const LEAF_PTE_LEVEL: usize = 3;
446 if flags.contains(Attributes::TABLE_OR_PAGE) {
447 level == LEAF_PTE_LEVEL
448 } else {
449 level < LEAF_PTE_LEVEL
450 }
451}
452
453/// Checks whether block flags indicate it should be MMIO guard mapped.
454fn verify_lazy_mapped_block(
455 _range: &VaRange,
456 desc: &mut Descriptor,
457 level: usize,
458) -> result::Result<(), ()> {
459 let flags = desc.flags().expect("Unsupported PTE flags set");
460 if !is_leaf_pte(&flags, level) {
461 return Ok(()); // Skip table PTEs as they aren't tagged with MMIO_LAZY_MAP_FLAG.
462 }
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000463 if flags.contains(MMIO_LAZY_MAP_FLAG) && !flags.contains(Attributes::VALID) {
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100464 Ok(())
465 } else {
466 Err(())
467 }
468}
469
470/// MMIO guard unmaps page
471fn mmio_guard_unmap_page(
472 va_range: &VaRange,
473 desc: &mut Descriptor,
474 level: usize,
475) -> result::Result<(), ()> {
476 let flags = desc.flags().expect("Unsupported PTE flags set");
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100477 if !is_leaf_pte(&flags, level) {
478 return Ok(());
479 }
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100480 // This function will be called on an address range that corresponds to a device. Only if a
481 // page has been accessed (written to or read from), will it contain the VALID flag and be MMIO
482 // guard mapped. Therefore, we can skip unmapping invalid pages, they were never MMIO guard
483 // mapped anyway.
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100484 if flags.contains(Attributes::VALID) {
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100485 assert!(
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000486 flags.contains(MMIO_LAZY_MAP_FLAG),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100487 "Attempting MMIO guard unmap for non-device pages"
488 );
489 assert_eq!(
490 va_range.len(),
491 PVMFW_PAGE_SIZE,
492 "Failed to break down block mapping before MMIO guard mapping"
493 );
494 let page_base = va_range.start().0;
495 assert_eq!(page_base % PVMFW_PAGE_SIZE, 0);
496 // Since mmio_guard_map takes IPAs, if pvmfw moves non-ID address mapping, page_base
497 // should be converted to IPA. However, since 0x0 is a valid MMIO address, we don't use
498 // virt_to_phys here, and just pass page_base instead.
499 get_hypervisor().mmio_guard_unmap(page_base).map_err(|e| {
500 error!("Error MMIO guard unmapping: {e}");
501 })?;
502 }
503 Ok(())
504}
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100505
506/// Flushes a memory range the descriptor refers to, if the descriptor is in writable-dirty state.
507fn flush_dirty_range(
508 va_range: &VaRange,
509 desc: &mut Descriptor,
510 level: usize,
511) -> result::Result<(), ()> {
512 // Only flush ranges corresponding to dirty leaf PTEs.
513 let flags = desc.flags().ok_or(())?;
514 if !is_leaf_pte(&flags, level) {
515 return Ok(());
516 }
517 if !flags.contains(Attributes::READ_ONLY) {
518 helpers::flush_region(va_range.start().0, va_range.len());
519 }
520 Ok(())
521}
522
523/// Clears read-only flag on a PTE, making it writable-dirty. Used when dirty state is managed
524/// in software to handle permission faults on read-only descriptors.
525fn mark_dirty_block(
526 va_range: &VaRange,
527 desc: &mut Descriptor,
528 level: usize,
529) -> result::Result<(), ()> {
530 let flags = desc.flags().ok_or(())?;
531 if !is_leaf_pte(&flags, level) {
532 return Ok(());
533 }
534 if flags.contains(Attributes::DBM) {
535 assert!(flags.contains(Attributes::READ_ONLY), "unexpected PTE writable state");
536 desc.modify_flags(Attributes::empty(), Attributes::READ_ONLY);
537 // Updating the read-only bit of a PTE requires TLB invalidation.
538 // A TLB maintenance instruction is only guaranteed to be complete after a DSB instruction.
539 // An ISB instruction is required to ensure the effects of completed TLB maintenance
540 // instructions are visible to instructions fetched afterwards.
541 // See ARM ARM E2.3.10, and G5.9.
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000542 tlbi!("vale1", PT_ASID, va_range.start().0);
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100543 dsb!("ish");
544 isb!();
545 Ok(())
546 } else {
547 Err(())
548 }
549}
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000550
551/// Returns memory range reserved for the appended payload.
552pub fn appended_payload_range() -> Range<usize> {
Alice Wangeacb7382023-06-05 12:53:54 +0000553 let start = align_up(layout::binary_end(), SIZE_4KB).unwrap();
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000554 // pvmfw is contained in a 2MiB region so the payload can't be larger than the 2MiB alignment.
Alice Wangeacb7382023-06-05 12:53:54 +0000555 let end = align_up(start, SIZE_2MB).unwrap();
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000556 start..end
557}
558
559/// Region allocated for the stack.
560pub fn stack_range() -> Range<usize> {
561 const STACK_PAGES: usize = 8;
562
563 layout::stack_range(STACK_PAGES * PVMFW_PAGE_SIZE)
564}
565
566pub fn init_page_table() -> result::Result<PageTable, MapError> {
567 let mut page_table: PageTable = IdMap::new(PT_ASID, PT_ROOT_LEVEL).into();
568
569 // Stack and scratch ranges are explicitly zeroed and flushed before jumping to payload,
570 // so dirty state management can be omitted.
571 page_table.map_data(&layout::scratch_range())?;
572 page_table.map_data(&stack_range())?;
573 page_table.map_code(&layout::text_range())?;
574 page_table.map_rodata(&layout::rodata_range())?;
575 page_table.map_data_dbm(&appended_payload_range())?;
Alice Wang807fa592023-06-02 09:54:43 +0000576 if let Err(e) = page_table.map_device(&layout::console_uart_range()) {
577 error!("Failed to remap the UART as a dynamic page table entry: {e}");
578 return Err(e);
579 }
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000580 Ok(page_table)
581}