blob: dfa29e425357617cff7333c8eded2be1e3d15689 [file] [log] [blame]
Alice Wangf47b2342023-06-02 11:51:57 +00001// Copyright 2023, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Shared memory management.
16
Alice Wang93ee98a2023-06-08 08:20:39 +000017use super::dbm::{flush_dirty_range, mark_dirty_block, set_dbm_enabled};
18use super::error::MemoryTrackerError;
19use super::page_table::{is_leaf_pte, PageTable, MMIO_LAZY_MAP_FLAG};
20use super::util::{page_4kb_of, virt_to_phys};
21use crate::dsb;
Alice Wanga9fe1fb2023-07-04 09:10:35 +000022use crate::exceptions::HandleExceptionError;
Alice Wang93ee98a2023-06-08 08:20:39 +000023use crate::util::RangeExt as _;
Alice Wanga3931aa2023-07-05 12:52:09 +000024use aarch64_paging::paging::{Attributes, Descriptor, MemoryRegion as VaRange, VirtualAddress};
Alice Wangf47b2342023-06-02 11:51:57 +000025use alloc::alloc::{alloc_zeroed, dealloc, handle_alloc_error};
Alice Wang93ee98a2023-06-08 08:20:39 +000026use alloc::boxed::Box;
Alice Wangf47b2342023-06-02 11:51:57 +000027use alloc::vec::Vec;
Alice Wang93ee98a2023-06-08 08:20:39 +000028use buddy_system_allocator::{FrameAllocator, LockedFrameAllocator};
Alice Wangf47b2342023-06-02 11:51:57 +000029use core::alloc::Layout;
Alice Wangdf6bacc2023-07-17 14:30:57 +000030use core::cmp::max;
Pierre-Clément Tosi8937cb82023-07-06 15:07:38 +000031use core::mem::size_of;
Alice Wang93ee98a2023-06-08 08:20:39 +000032use core::num::NonZeroUsize;
33use core::ops::Range;
Alice Wangf47b2342023-06-02 11:51:57 +000034use core::ptr::NonNull;
Alice Wangb73a81b2023-06-07 13:05:09 +000035use core::result;
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +000036use hyp::{get_mem_sharer, get_mmio_guard, MMIO_GUARD_GRANULE_SIZE};
Alice Wang93ee98a2023-06-08 08:20:39 +000037use log::{debug, error, trace};
38use once_cell::race::OnceBox;
39use spin::mutex::SpinMutex;
40use tinyvec::ArrayVec;
41
42/// A global static variable representing the system memory tracker, protected by a spin mutex.
43pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
44
45static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
46static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
47
48/// Memory range.
49pub type MemoryRange = Range<usize>;
Alice Wanga3931aa2023-07-05 12:52:09 +000050
51fn get_va_range(range: &MemoryRange) -> VaRange {
52 VaRange::new(range.start, range.end)
53}
54
Alice Wang93ee98a2023-06-08 08:20:39 +000055type Result<T> = result::Result<T, MemoryTrackerError>;
56
57#[derive(Clone, Copy, Debug, Default, PartialEq)]
58enum MemoryType {
59 #[default]
60 ReadOnly,
61 ReadWrite,
62}
63
64#[derive(Clone, Debug, Default)]
65struct MemoryRegion {
66 range: MemoryRange,
67 mem_type: MemoryType,
68}
69
70/// Tracks non-overlapping slices of main memory.
71pub struct MemoryTracker {
72 total: MemoryRange,
73 page_table: PageTable,
74 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
75 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
76 mmio_range: MemoryRange,
Alice Wang5bb79502023-06-12 09:25:07 +000077 payload_range: Option<MemoryRange>,
Alice Wang93ee98a2023-06-08 08:20:39 +000078}
79
Alice Wang93ee98a2023-06-08 08:20:39 +000080impl MemoryTracker {
81 const CAPACITY: usize = 5;
82 const MMIO_CAPACITY: usize = 5;
83
84 /// Creates a new instance from an active page table, covering the maximum RAM size.
85 pub fn new(
86 mut page_table: PageTable,
87 total: MemoryRange,
88 mmio_range: MemoryRange,
Alice Wanga3931aa2023-07-05 12:52:09 +000089 payload_range: Option<Range<VirtualAddress>>,
Alice Wang93ee98a2023-06-08 08:20:39 +000090 ) -> Self {
91 assert!(
92 !total.overlaps(&mmio_range),
93 "MMIO space should not overlap with the main memory region."
94 );
95
96 // Activate dirty state management first, otherwise we may get permission faults immediately
97 // after activating the new page table. This has no effect before the new page table is
98 // activated because none of the entries in the initial idmap have the DBM flag.
99 set_dbm_enabled(true);
100
101 debug!("Activating dynamic page table...");
Andrew Walbranc06e7342023-07-05 14:00:51 +0000102 // SAFETY: page_table duplicates the static mappings for everything that the Rust code is
Alice Wang93ee98a2023-06-08 08:20:39 +0000103 // aware of so activating it shouldn't have any visible effect.
104 unsafe { page_table.activate() }
105 debug!("... Success!");
106
107 Self {
108 total,
109 page_table,
110 regions: ArrayVec::new(),
111 mmio_regions: ArrayVec::new(),
112 mmio_range,
Alice Wanga3931aa2023-07-05 12:52:09 +0000113 payload_range: payload_range.map(|r| r.start.0..r.end.0),
Alice Wang93ee98a2023-06-08 08:20:39 +0000114 }
115 }
116
117 /// Resize the total RAM size.
118 ///
119 /// This function fails if it contains regions that are not included within the new size.
120 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
121 if range.start != self.total.start {
122 return Err(MemoryTrackerError::DifferentBaseAddress);
123 }
124 if self.total.end < range.end {
125 return Err(MemoryTrackerError::SizeTooLarge);
126 }
127 if !self.regions.iter().all(|r| r.range.is_within(range)) {
128 return Err(MemoryTrackerError::SizeTooSmall);
129 }
130
131 self.total = range.clone();
132 Ok(())
133 }
134
135 /// Allocate the address range for a const slice; returns None if failed.
136 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
137 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
138 self.check(&region)?;
Alice Wanga3931aa2023-07-05 12:52:09 +0000139 self.page_table.map_rodata(&get_va_range(range)).map_err(|e| {
Alice Wang93ee98a2023-06-08 08:20:39 +0000140 error!("Error during range allocation: {e}");
141 MemoryTrackerError::FailedToMap
142 })?;
143 self.add(region)
144 }
145
146 /// Allocate the address range for a mutable slice; returns None if failed.
147 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
148 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
149 self.check(&region)?;
Alice Wanga3931aa2023-07-05 12:52:09 +0000150 self.page_table.map_data_dbm(&get_va_range(range)).map_err(|e| {
Alice Wang93ee98a2023-06-08 08:20:39 +0000151 error!("Error during mutable range allocation: {e}");
152 MemoryTrackerError::FailedToMap
153 })?;
154 self.add(region)
155 }
156
157 /// Allocate the address range for a const slice; returns None if failed.
158 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
159 self.alloc_range(&(base..(base + size.get())))
160 }
161
162 /// Allocate the address range for a mutable slice; returns None if failed.
163 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
164 self.alloc_range_mut(&(base..(base + size.get())))
165 }
166
167 /// Checks that the given range of addresses is within the MMIO region, and then maps it
168 /// appropriately.
169 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
170 if !range.is_within(&self.mmio_range) {
171 return Err(MemoryTrackerError::OutOfRange);
172 }
173 if self.mmio_regions.iter().any(|r| range.overlaps(r)) {
174 return Err(MemoryTrackerError::Overlaps);
175 }
176 if self.mmio_regions.len() == self.mmio_regions.capacity() {
177 return Err(MemoryTrackerError::Full);
178 }
179
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000180 if get_mmio_guard().is_some() {
Pierre-Clément Tosi32279ef2023-06-29 10:46:59 +0000181 self.page_table.map_device_lazy(&get_va_range(&range)).map_err(|e| {
182 error!("Error during lazy MMIO device mapping: {e}");
183 MemoryTrackerError::FailedToMap
184 })?;
185 } else {
186 self.page_table.map_device(&get_va_range(&range)).map_err(|e| {
187 error!("Error during MMIO device mapping: {e}");
188 MemoryTrackerError::FailedToMap
189 })?;
190 }
Alice Wang93ee98a2023-06-08 08:20:39 +0000191
192 if self.mmio_regions.try_push(range).is_some() {
193 return Err(MemoryTrackerError::Full);
194 }
195
196 Ok(())
197 }
198
199 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
200 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
201 /// add it.
202 fn check(&self, region: &MemoryRegion) -> Result<()> {
203 if !region.range.is_within(&self.total) {
204 return Err(MemoryTrackerError::OutOfRange);
205 }
206 if self.regions.iter().any(|r| region.range.overlaps(&r.range)) {
207 return Err(MemoryTrackerError::Overlaps);
208 }
209 if self.regions.len() == self.regions.capacity() {
210 return Err(MemoryTrackerError::Full);
211 }
212 Ok(())
213 }
214
215 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
216 if self.regions.try_push(region).is_some() {
217 return Err(MemoryTrackerError::Full);
218 }
219
220 Ok(self.regions.last().unwrap().range.clone())
221 }
222
223 /// Unmaps all tracked MMIO regions from the MMIO guard.
224 ///
225 /// Note that they are not unmapped from the page table.
226 pub fn mmio_unmap_all(&mut self) -> Result<()> {
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000227 if get_mmio_guard().is_some() {
Pierre-Clément Tosi32279ef2023-06-29 10:46:59 +0000228 for range in &self.mmio_regions {
229 self.page_table
230 .modify_range(&get_va_range(range), &mmio_guard_unmap_page)
231 .map_err(|_| MemoryTrackerError::FailedToUnmap)?;
232 }
Alice Wang93ee98a2023-06-08 08:20:39 +0000233 }
234 Ok(())
235 }
236
237 /// Initialize the shared heap to dynamically share memory from the global allocator.
Alice Wangb6d2c642023-06-13 13:07:06 +0000238 pub fn init_dynamic_shared_pool(&mut self, granule: usize) -> Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000239 const INIT_CAP: usize = 10;
240
Alice Wang93ee98a2023-06-08 08:20:39 +0000241 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule, INIT_CAP));
242 if previous.is_some() {
243 return Err(MemoryTrackerError::SharedMemorySetFailure);
244 }
245
246 SHARED_POOL
247 .set(Box::new(LockedFrameAllocator::new()))
248 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
249
250 Ok(())
251 }
252
253 /// Initialize the shared heap from a static region of memory.
254 ///
255 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
256 /// to share its memory with host. Instead they allow host to designate part
257 /// of guest memory as "shared" ahead of guest starting its execution. The
258 /// shared memory region is indicated in swiotlb node. On such platforms use
259 /// a separate heap to allocate buffers that can be shared with host.
260 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
261 let size = NonZeroUsize::new(range.len()).unwrap();
262 let range = self.alloc_mut(range.start, size)?;
263 let shared_pool = LockedFrameAllocator::<32>::new();
264
265 shared_pool.lock().insert(range);
266
267 SHARED_POOL
268 .set(Box::new(shared_pool))
269 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
270
271 Ok(())
272 }
273
Pierre-Clément Tosi8937cb82023-07-06 15:07:38 +0000274 /// Initialize the shared heap to use heap memory directly.
275 ///
276 /// When running on "non-protected" hypervisors which permit host direct accesses to guest
277 /// memory, there is no need to perform any memory sharing and/or allocate buffers from a
278 /// dedicated region so this function instructs the shared pool to use the global allocator.
279 pub fn init_heap_shared_pool(&mut self) -> Result<()> {
280 // As MemorySharer only calls MEM_SHARE methods if the hypervisor supports them, internally
281 // using init_dynamic_shared_pool() on a non-protected platform will make use of the heap
282 // without any actual "dynamic memory sharing" taking place and, as such, the granule may
283 // be set to the one of the global_allocator i.e. a byte.
284 self.init_dynamic_shared_pool(size_of::<u8>())
285 }
286
Alice Wang93ee98a2023-06-08 08:20:39 +0000287 /// Unshares any memory that may have been shared.
288 pub fn unshare_all_memory(&mut self) {
289 drop(SHARED_MEMORY.lock().take());
290 }
291
292 /// Handles translation fault for blocks flagged for lazy MMIO mapping by enabling the page
293 /// table entry and MMIO guard mapping the block. Breaks apart a block entry if required.
Alice Wanga9fe1fb2023-07-04 09:10:35 +0000294 fn handle_mmio_fault(&mut self, addr: VirtualAddress) -> Result<()> {
Alice Wang88736462023-07-05 12:14:15 +0000295 let page_start = VirtualAddress(page_4kb_of(addr.0));
Alice Wanga3931aa2023-07-05 12:52:09 +0000296 let page_range: VaRange = (page_start..page_start + MMIO_GUARD_GRANULE_SIZE).into();
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000297 let mmio_guard = get_mmio_guard().unwrap();
Alice Wang93ee98a2023-06-08 08:20:39 +0000298 self.page_table
299 .modify_range(&page_range, &verify_lazy_mapped_block)
300 .map_err(|_| MemoryTrackerError::InvalidPte)?;
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000301 mmio_guard.map(page_start.0)?;
Alice Wang93ee98a2023-06-08 08:20:39 +0000302 // Maps a single device page, breaking up block mappings if necessary.
303 self.page_table.map_device(&page_range).map_err(|_| MemoryTrackerError::FailedToMap)
304 }
305
306 /// Flush all memory regions marked as writable-dirty.
307 fn flush_dirty_pages(&mut self) -> Result<()> {
308 // Collect memory ranges for which dirty state is tracked.
309 let writable_regions =
310 self.regions.iter().filter(|r| r.mem_type == MemoryType::ReadWrite).map(|r| &r.range);
311 // Execute a barrier instruction to ensure all hardware updates to the page table have been
312 // observed before reading PTE flags to determine dirty state.
313 dsb!("ish");
314 // Now flush writable-dirty pages in those regions.
Alice Wang5bb79502023-06-12 09:25:07 +0000315 for range in writable_regions.chain(self.payload_range.as_ref().into_iter()) {
Alice Wang93ee98a2023-06-08 08:20:39 +0000316 self.page_table
Alice Wanga3931aa2023-07-05 12:52:09 +0000317 .modify_range(&get_va_range(range), &flush_dirty_range)
Alice Wang93ee98a2023-06-08 08:20:39 +0000318 .map_err(|_| MemoryTrackerError::FlushRegionFailed)?;
319 }
320 Ok(())
321 }
322
323 /// Handles permission fault for read-only blocks by setting writable-dirty state.
324 /// In general, this should be called from the exception handler when hardware dirty
325 /// state management is disabled or unavailable.
Alice Wanga9fe1fb2023-07-04 09:10:35 +0000326 fn handle_permission_fault(&mut self, addr: VirtualAddress) -> Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000327 self.page_table
Alice Wanga3931aa2023-07-05 12:52:09 +0000328 .modify_range(&(addr..addr + 1).into(), &mark_dirty_block)
Alice Wang93ee98a2023-06-08 08:20:39 +0000329 .map_err(|_| MemoryTrackerError::SetPteDirtyFailed)
330 }
331}
332
333impl Drop for MemoryTracker {
334 fn drop(&mut self) {
335 set_dbm_enabled(false);
336 self.flush_dirty_pages().unwrap();
337 self.unshare_all_memory();
338 }
339}
340
341/// Allocates a memory range of at least the given size and alignment that is shared with the host.
342/// Returns a pointer to the buffer.
Alice Wang6c4cda02023-07-18 08:18:07 +0000343pub(crate) fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
Alice Wang7cbe29a2023-07-27 11:45:58 +0000344 assert_ne!(layout.size(), 0);
Alice Wang93ee98a2023-06-08 08:20:39 +0000345 let Some(buffer) = try_shared_alloc(layout) else {
346 handle_alloc_error(layout);
347 };
348
349 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
350 Ok(buffer)
351}
352
353fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
354 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
355
356 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
357 Some(NonNull::new(buffer as _).unwrap())
358 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
Alice Wang2a6b2172023-07-18 10:38:16 +0000359 // Adjusts the layout size to the max of the next power of two and the alignment,
360 // as this is the actual size of the memory allocated in `alloc_aligned()`.
361 let size = max(layout.size().next_power_of_two(), layout.align());
362 let refill_layout = Layout::from_size_align(size, layout.align()).unwrap();
363 shared_memory.refill(&mut shared_pool, refill_layout);
Alice Wang93ee98a2023-06-08 08:20:39 +0000364 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
365 } else {
366 None
367 }
368}
369
370/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
371///
372/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
373///
374/// # Safety
375///
376/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
377/// deallocated.
Alice Wang6c4cda02023-07-18 08:18:07 +0000378pub(crate) unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000379 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
380
381 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
382 Ok(())
383}
Alice Wangf47b2342023-06-02 11:51:57 +0000384
385/// Allocates memory on the heap and shares it with the host.
386///
387/// Unshares all pages when dropped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000388struct MemorySharer {
Alice Wangf47b2342023-06-02 11:51:57 +0000389 granule: usize,
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000390 frames: Vec<(usize, Layout)>,
Alice Wangf47b2342023-06-02 11:51:57 +0000391}
392
393impl MemorySharer {
394 /// Constructs a new `MemorySharer` instance with the specified granule size and capacity.
395 /// `granule` must be a power of 2.
Alice Wang93ee98a2023-06-08 08:20:39 +0000396 fn new(granule: usize, capacity: usize) -> Self {
Alice Wangf47b2342023-06-02 11:51:57 +0000397 assert!(granule.is_power_of_two());
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000398 Self { granule, frames: Vec::with_capacity(capacity) }
Alice Wangf47b2342023-06-02 11:51:57 +0000399 }
400
Alice Wang93ee98a2023-06-08 08:20:39 +0000401 /// Gets from the global allocator a granule-aligned region that suits `hint` and share it.
402 fn refill(&mut self, pool: &mut FrameAllocator<32>, hint: Layout) {
Alice Wangf47b2342023-06-02 11:51:57 +0000403 let layout = hint.align_to(self.granule).unwrap().pad_to_align();
404 assert_ne!(layout.size(), 0);
Andrew Walbranc06e7342023-07-05 14:00:51 +0000405 // SAFETY: layout has non-zero size.
Alice Wangf47b2342023-06-02 11:51:57 +0000406 let Some(shared) = NonNull::new(unsafe { alloc_zeroed(layout) }) else {
407 handle_alloc_error(layout);
408 };
409
410 let base = shared.as_ptr() as usize;
411 let end = base.checked_add(layout.size()).unwrap();
Alice Wangf47b2342023-06-02 11:51:57 +0000412
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000413 if let Some(mem_sharer) = get_mem_sharer() {
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000414 trace!("Sharing memory region {:#x?}", base..end);
415 for vaddr in (base..end).step_by(self.granule) {
416 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000417 mem_sharer.share(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000418 }
419 }
420
421 self.frames.push((base, layout));
Alice Wangf47b2342023-06-02 11:51:57 +0000422 pool.add_frame(base, end);
423 }
424}
425
426impl Drop for MemorySharer {
427 fn drop(&mut self) {
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000428 while let Some((base, layout)) = self.frames.pop() {
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000429 if let Some(mem_sharer) = get_mem_sharer() {
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000430 let end = base.checked_add(layout.size()).unwrap();
431 trace!("Unsharing memory region {:#x?}", base..end);
432 for vaddr in (base..end).step_by(self.granule) {
433 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000434 mem_sharer.unshare(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
Pierre-Clément Tosid2f7ad12023-06-29 11:48:29 +0000435 }
Alice Wangf47b2342023-06-02 11:51:57 +0000436 }
437
Andrew Walbranc06e7342023-07-05 14:00:51 +0000438 // SAFETY: The region was obtained from alloc_zeroed() with the recorded layout.
Alice Wangf47b2342023-06-02 11:51:57 +0000439 unsafe { dealloc(base as *mut _, layout) };
440 }
441 }
442}
Alice Wangb73a81b2023-06-07 13:05:09 +0000443
444/// Checks whether block flags indicate it should be MMIO guard mapped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000445fn verify_lazy_mapped_block(
Alice Wangb73a81b2023-06-07 13:05:09 +0000446 _range: &VaRange,
447 desc: &mut Descriptor,
448 level: usize,
449) -> result::Result<(), ()> {
450 let flags = desc.flags().expect("Unsupported PTE flags set");
451 if !is_leaf_pte(&flags, level) {
452 return Ok(()); // Skip table PTEs as they aren't tagged with MMIO_LAZY_MAP_FLAG.
453 }
454 if flags.contains(MMIO_LAZY_MAP_FLAG) && !flags.contains(Attributes::VALID) {
455 Ok(())
456 } else {
457 Err(())
458 }
459}
460
461/// MMIO guard unmaps page
Alice Wang93ee98a2023-06-08 08:20:39 +0000462fn mmio_guard_unmap_page(
Alice Wangb73a81b2023-06-07 13:05:09 +0000463 va_range: &VaRange,
464 desc: &mut Descriptor,
465 level: usize,
466) -> result::Result<(), ()> {
467 let flags = desc.flags().expect("Unsupported PTE flags set");
468 if !is_leaf_pte(&flags, level) {
469 return Ok(());
470 }
471 // This function will be called on an address range that corresponds to a device. Only if a
472 // page has been accessed (written to or read from), will it contain the VALID flag and be MMIO
473 // guard mapped. Therefore, we can skip unmapping invalid pages, they were never MMIO guard
474 // mapped anyway.
475 if flags.contains(Attributes::VALID) {
476 assert!(
477 flags.contains(MMIO_LAZY_MAP_FLAG),
478 "Attempting MMIO guard unmap for non-device pages"
479 );
480 assert_eq!(
481 va_range.len(),
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000482 MMIO_GUARD_GRANULE_SIZE,
Alice Wangb73a81b2023-06-07 13:05:09 +0000483 "Failed to break down block mapping before MMIO guard mapping"
484 );
485 let page_base = va_range.start().0;
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000486 assert_eq!(page_base % MMIO_GUARD_GRANULE_SIZE, 0);
Alice Wangb73a81b2023-06-07 13:05:09 +0000487 // Since mmio_guard_map takes IPAs, if pvmfw moves non-ID address mapping, page_base
488 // should be converted to IPA. However, since 0x0 is a valid MMIO address, we don't use
489 // virt_to_phys here, and just pass page_base instead.
Pierre-Clément Tosid643cfe2023-06-29 09:30:51 +0000490 get_mmio_guard().unwrap().unmap(page_base).map_err(|e| {
Alice Wangb73a81b2023-06-07 13:05:09 +0000491 error!("Error MMIO guard unmapping: {e}");
492 })?;
493 }
494 Ok(())
495}
Alice Wanga9fe1fb2023-07-04 09:10:35 +0000496
497/// Handles a translation fault with the given fault address register (FAR).
498#[inline]
499pub fn handle_translation_fault(far: VirtualAddress) -> result::Result<(), HandleExceptionError> {
500 let mut guard = MEMORY.try_lock().ok_or(HandleExceptionError::PageTableUnavailable)?;
501 let memory = guard.as_mut().ok_or(HandleExceptionError::PageTableNotInitialized)?;
502 Ok(memory.handle_mmio_fault(far)?)
503}
504
505/// Handles a permission fault with the given fault address register (FAR).
506#[inline]
507pub fn handle_permission_fault(far: VirtualAddress) -> result::Result<(), HandleExceptionError> {
508 let mut guard = MEMORY.try_lock().ok_or(HandleExceptionError::PageTableUnavailable)?;
509 let memory = guard.as_mut().ok_or(HandleExceptionError::PageTableNotInitialized)?;
510 Ok(memory.handle_permission_fault(far)?)
511}