blob: ffbb1ca6a9d29e6f35c7249b504e8730ebd9cd3a [file] [log] [blame]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +00001// Copyright 2022, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Low-level allocation and tracking of main memory.
16
Andrew Walbran848decf2022-12-15 14:39:38 +000017#![deny(unsafe_op_in_unsafe_fn)]
18
Alice Wang4be4dd02023-06-07 07:50:40 +000019use crate::helpers::PVMFW_PAGE_SIZE;
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000020use aarch64_paging::idmap::IdMap;
Jakob Vukalovicb99905d2023-04-20 15:46:02 +010021use aarch64_paging::paging::{Attributes, Descriptor, MemoryRegion as VaRange};
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000022use aarch64_paging::MapError;
Andrew Walbran848decf2022-12-15 14:39:38 +000023use alloc::alloc::handle_alloc_error;
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070024use alloc::boxed::Box;
Alice Wangf47b2342023-06-02 11:51:57 +000025use buddy_system_allocator::LockedFrameAllocator;
Andrew Walbran848decf2022-12-15 14:39:38 +000026use core::alloc::Layout;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000027use core::fmt;
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010028use core::iter::once;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000029use core::num::NonZeroUsize;
30use core::ops::Range;
Andrew Walbran848decf2022-12-15 14:39:38 +000031use core::ptr::NonNull;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000032use core::result;
Alice Wang90e6f162023-04-17 13:49:45 +000033use hyp::get_hypervisor;
Pierre-Clément Tosi90238c52023-04-27 17:59:10 +000034use log::trace;
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +010035use log::{debug, error};
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070036use once_cell::race::OnceBox;
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010037use spin::mutex::SpinMutex;
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000038use tinyvec::ArrayVec;
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000039use vmbase::{
Alice Wang4c70d142023-06-06 11:52:33 +000040 dsb, isb, layout,
Alice Wangeacb7382023-06-05 12:53:54 +000041 memory::{
Alice Wang3fa9b802023-06-06 07:52:31 +000042 flush_dirty_range, is_leaf_pte, page_4kb_of, set_dbm_enabled, MemorySharer, PageTable,
Alice Wang4c70d142023-06-06 11:52:33 +000043 MMIO_LAZY_MAP_FLAG, SIZE_2MB, SIZE_4KB,
Alice Wangeacb7382023-06-05 12:53:54 +000044 },
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000045 tlbi,
Alice Wang4be4dd02023-06-07 07:50:40 +000046 util::{align_up, RangeExt as _},
Pierre-Clément Tosi3d4c5c32023-05-31 16:57:06 +000047};
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000048
Jiyong Park0ee65392023-03-27 20:52:45 +090049/// First address that can't be translated by a level 1 TTBR0_EL1.
50pub const MAX_ADDR: usize = 1 << 40;
51
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000052const PT_ROOT_LEVEL: usize = 1;
53const PT_ASID: usize = 1;
54
Andrew Walbran0d8b54d2022-12-08 16:32:33 +000055pub type MemoryRange = Range<usize>;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000056
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010057pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
58unsafe impl Send for MemoryTracker {}
59
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +010060#[derive(Clone, Copy, Debug, Default, PartialEq)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000061enum MemoryType {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000062 #[default]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000063 ReadOnly,
64 ReadWrite,
65}
66
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000067#[derive(Clone, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000068struct MemoryRegion {
69 range: MemoryRange,
70 mem_type: MemoryType,
71}
72
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000073/// Tracks non-overlapping slices of main memory.
74pub struct MemoryTracker {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000075 total: MemoryRange,
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +000076 page_table: PageTable,
Andrew Walbran19690632022-12-07 16:41:30 +000077 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
78 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
Alice Wang4c70d142023-06-06 11:52:33 +000079 mmio_range: MemoryRange,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000080}
81
82/// Errors for MemoryTracker operations.
83#[derive(Debug, Clone)]
84pub enum MemoryTrackerError {
85 /// Tried to modify the memory base address.
86 DifferentBaseAddress,
87 /// Tried to shrink to a larger memory size.
88 SizeTooLarge,
89 /// Tracked regions would not fit in memory size.
90 SizeTooSmall,
91 /// Reached limit number of tracked regions.
92 Full,
93 /// Region is out of the tracked memory address space.
94 OutOfRange,
95 /// New region overlaps with tracked regions.
96 Overlaps,
97 /// Region couldn't be mapped.
98 FailedToMap,
Jakob Vukalovicb99905d2023-04-20 15:46:02 +010099 /// Region couldn't be unmapped.
100 FailedToUnmap,
Alice Wang90e6f162023-04-17 13:49:45 +0000101 /// Error from the interaction with the hypervisor.
102 Hypervisor(hyp::Error),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000103 /// Failure to set `SHARED_MEMORY`.
104 SharedMemorySetFailure,
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700105 /// Failure to set `SHARED_POOL`.
106 SharedPoolSetFailure,
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100107 /// Invalid page table entry.
108 InvalidPte,
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100109 /// Failed to flush memory region.
110 FlushRegionFailed,
111 /// Failed to set PTE dirty state.
112 SetPteDirtyFailed,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000113}
114
115impl fmt::Display for MemoryTrackerError {
116 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
117 match self {
118 Self::DifferentBaseAddress => write!(f, "Received different base address"),
119 Self::SizeTooLarge => write!(f, "Tried to shrink to a larger memory size"),
120 Self::SizeTooSmall => write!(f, "Tracked regions would not fit in memory size"),
121 Self::Full => write!(f, "Reached limit number of tracked regions"),
122 Self::OutOfRange => write!(f, "Region is out of the tracked memory address space"),
123 Self::Overlaps => write!(f, "New region overlaps with tracked regions"),
124 Self::FailedToMap => write!(f, "Failed to map the new region"),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100125 Self::FailedToUnmap => write!(f, "Failed to unmap the new region"),
Alice Wang90e6f162023-04-17 13:49:45 +0000126 Self::Hypervisor(e) => e.fmt(f),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000127 Self::SharedMemorySetFailure => write!(f, "Failed to set SHARED_MEMORY"),
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700128 Self::SharedPoolSetFailure => write!(f, "Failed to set SHARED_POOL"),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100129 Self::InvalidPte => write!(f, "Page table entry is not valid"),
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100130 Self::FlushRegionFailed => write!(f, "Failed to flush memory region"),
131 Self::SetPteDirtyFailed => write!(f, "Failed to set PTE dirty state"),
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000132 }
133 }
134}
135
Alice Wang90e6f162023-04-17 13:49:45 +0000136impl From<hyp::Error> for MemoryTrackerError {
137 fn from(e: hyp::Error) -> Self {
138 Self::Hypervisor(e)
Andrew Walbran19690632022-12-07 16:41:30 +0000139 }
140}
141
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000142type Result<T> = result::Result<T, MemoryTrackerError>;
143
Andrew Walbran87933f32023-05-09 15:29:06 +0000144static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000145static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
146
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000147impl MemoryTracker {
148 const CAPACITY: usize = 5;
Andrew Walbran19690632022-12-07 16:41:30 +0000149 const MMIO_CAPACITY: usize = 5;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000150
151 /// Create a new instance from an active page table, covering the maximum RAM size.
Alice Wang4c70d142023-06-06 11:52:33 +0000152 pub fn new(mut page_table: PageTable, total: MemoryRange, mmio_range: MemoryRange) -> Self {
153 assert!(
154 !total.overlaps(&mmio_range),
155 "MMIO space should not overlap with the main memory region."
156 );
157
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100158 // Activate dirty state management first, otherwise we may get permission faults immediately
159 // after activating the new page table. This has no effect before the new page table is
160 // activated because none of the entries in the initial idmap have the DBM flag.
Alice Wang4dd20932023-05-26 13:47:16 +0000161 set_dbm_enabled(true);
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100162
163 debug!("Activating dynamic page table...");
164 // SAFETY - page_table duplicates the static mappings for everything that the Rust code is
165 // aware of so activating it shouldn't have any visible effect.
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000166 unsafe { page_table.activate() }
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100167 debug!("... Success!");
168
Andrew Walbran19690632022-12-07 16:41:30 +0000169 Self {
Alice Wang4c70d142023-06-06 11:52:33 +0000170 total,
Andrew Walbran19690632022-12-07 16:41:30 +0000171 page_table,
172 regions: ArrayVec::new(),
173 mmio_regions: ArrayVec::new(),
Alice Wang4c70d142023-06-06 11:52:33 +0000174 mmio_range,
Andrew Walbran19690632022-12-07 16:41:30 +0000175 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000176 }
177
178 /// Resize the total RAM size.
179 ///
180 /// This function fails if it contains regions that are not included within the new size.
181 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
182 if range.start != self.total.start {
183 return Err(MemoryTrackerError::DifferentBaseAddress);
184 }
185 if self.total.end < range.end {
186 return Err(MemoryTrackerError::SizeTooLarge);
187 }
Alice Wang81e8f142023-06-06 12:47:14 +0000188 if !self.regions.iter().all(|r| r.range.is_within(range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000189 return Err(MemoryTrackerError::SizeTooSmall);
190 }
191
192 self.total = range.clone();
193 Ok(())
194 }
195
196 /// Allocate the address range for a const slice; returns None if failed.
197 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000198 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
199 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000200 self.page_table.map_rodata(range).map_err(|e| {
201 error!("Error during range allocation: {e}");
202 MemoryTrackerError::FailedToMap
203 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000204 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000205 }
206
207 /// Allocate the address range for a mutable slice; returns None if failed.
208 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000209 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
210 self.check(&region)?;
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000211 self.page_table.map_data_dbm(range).map_err(|e| {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000212 error!("Error during mutable range allocation: {e}");
213 MemoryTrackerError::FailedToMap
214 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000215 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000216 }
217
218 /// Allocate the address range for a const slice; returns None if failed.
219 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
220 self.alloc_range(&(base..(base + size.get())))
221 }
222
223 /// Allocate the address range for a mutable slice; returns None if failed.
224 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
225 self.alloc_range_mut(&(base..(base + size.get())))
226 }
227
Andrew Walbran19690632022-12-07 16:41:30 +0000228 /// Checks that the given range of addresses is within the MMIO region, and then maps it
229 /// appropriately.
230 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
Alice Wang4c70d142023-06-06 11:52:33 +0000231 if !range.is_within(&self.mmio_range) {
Andrew Walbran19690632022-12-07 16:41:30 +0000232 return Err(MemoryTrackerError::OutOfRange);
233 }
Alice Wang81e8f142023-06-06 12:47:14 +0000234 if self.mmio_regions.iter().any(|r| range.overlaps(r)) {
Andrew Walbran19690632022-12-07 16:41:30 +0000235 return Err(MemoryTrackerError::Overlaps);
236 }
237 if self.mmio_regions.len() == self.mmio_regions.capacity() {
238 return Err(MemoryTrackerError::Full);
239 }
240
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100241 self.page_table.map_device_lazy(&range).map_err(|e| {
Andrew Walbran19690632022-12-07 16:41:30 +0000242 error!("Error during MMIO device mapping: {e}");
243 MemoryTrackerError::FailedToMap
244 })?;
245
Andrew Walbran19690632022-12-07 16:41:30 +0000246 if self.mmio_regions.try_push(range).is_some() {
247 return Err(MemoryTrackerError::Full);
248 }
249
250 Ok(())
251 }
252
Andrew Walbranda65ab12022-12-07 15:10:13 +0000253 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
254 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
255 /// add it.
256 fn check(&self, region: &MemoryRegion) -> Result<()> {
Alice Wang81e8f142023-06-06 12:47:14 +0000257 if !region.range.is_within(&self.total) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000258 return Err(MemoryTrackerError::OutOfRange);
259 }
Alice Wang81e8f142023-06-06 12:47:14 +0000260 if self.regions.iter().any(|r| region.range.overlaps(&r.range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000261 return Err(MemoryTrackerError::Overlaps);
262 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000263 if self.regions.len() == self.regions.capacity() {
264 return Err(MemoryTrackerError::Full);
265 }
266 Ok(())
267 }
268
269 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000270 if self.regions.try_push(region).is_some() {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000271 return Err(MemoryTrackerError::Full);
272 }
273
Alice Wang81e8f142023-06-06 12:47:14 +0000274 Ok(self.regions.last().unwrap().range.clone())
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000275 }
Andrew Walbran19690632022-12-07 16:41:30 +0000276
277 /// Unmaps all tracked MMIO regions from the MMIO guard.
278 ///
279 /// Note that they are not unmapped from the page table.
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100280 pub fn mmio_unmap_all(&mut self) -> Result<()> {
281 for range in &self.mmio_regions {
282 self.page_table
283 .modify_range(range, &mmio_guard_unmap_page)
284 .map_err(|_| MemoryTrackerError::FailedToUnmap)?;
Andrew Walbran19690632022-12-07 16:41:30 +0000285 }
Andrew Walbran19690632022-12-07 16:41:30 +0000286 Ok(())
287 }
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700288
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000289 /// Initialize the shared heap to dynamically share memory from the global allocator.
290 pub fn init_dynamic_shared_pool(&mut self) -> Result<()> {
Alice Wangf47b2342023-06-02 11:51:57 +0000291 const INIT_CAP: usize = 10;
292
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000293 let granule = get_hypervisor().memory_protection_granule()?;
Alice Wangf47b2342023-06-02 11:51:57 +0000294 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule, INIT_CAP));
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000295 if previous.is_some() {
296 return Err(MemoryTrackerError::SharedMemorySetFailure);
297 }
298
299 SHARED_POOL
Andrew Walbran87933f32023-05-09 15:29:06 +0000300 .set(Box::new(LockedFrameAllocator::new()))
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000301 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
302
303 Ok(())
304 }
305
306 /// Initialize the shared heap from a static region of memory.
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700307 ///
308 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
309 /// to share its memory with host. Instead they allow host to designate part
310 /// of guest memory as "shared" ahead of guest starting its execution. The
311 /// shared memory region is indicated in swiotlb node. On such platforms use
312 /// a separate heap to allocate buffers that can be shared with host.
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000313 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700314 let size = NonZeroUsize::new(range.len()).unwrap();
315 let range = self.alloc_mut(range.start, size)?;
Andrew Walbran87933f32023-05-09 15:29:06 +0000316 let shared_pool = LockedFrameAllocator::<32>::new();
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700317
Andrew Walbran87933f32023-05-09 15:29:06 +0000318 shared_pool.lock().insert(range);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700319
320 SHARED_POOL
321 .set(Box::new(shared_pool))
322 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
323
324 Ok(())
325 }
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000326
327 /// Unshares any memory that may have been shared.
328 pub fn unshare_all_memory(&mut self) {
329 drop(SHARED_MEMORY.lock().take());
330 }
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100331
332 /// Handles translation fault for blocks flagged for lazy MMIO mapping by enabling the page
333 /// table entry and MMIO guard mapping the block. Breaks apart a block entry if required.
334 pub fn handle_mmio_fault(&mut self, addr: usize) -> Result<()> {
335 let page_range = page_4kb_of(addr)..page_4kb_of(addr) + PVMFW_PAGE_SIZE;
336 self.page_table
337 .modify_range(&page_range, &verify_lazy_mapped_block)
338 .map_err(|_| MemoryTrackerError::InvalidPte)?;
339 get_hypervisor().mmio_guard_map(page_range.start)?;
340 // Maps a single device page, breaking up block mappings if necessary.
341 self.page_table.map_device(&page_range).map_err(|_| MemoryTrackerError::FailedToMap)
342 }
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100343
344 /// Flush all memory regions marked as writable-dirty.
345 fn flush_dirty_pages(&mut self) -> Result<()> {
346 // Collect memory ranges for which dirty state is tracked.
347 let writable_regions =
348 self.regions.iter().filter(|r| r.mem_type == MemoryType::ReadWrite).map(|r| &r.range);
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000349 let payload_range = appended_payload_range();
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100350 // Execute a barrier instruction to ensure all hardware updates to the page table have been
351 // observed before reading PTE flags to determine dirty state.
352 dsb!("ish");
353 // Now flush writable-dirty pages in those regions.
354 for range in writable_regions.chain(once(&payload_range)) {
355 self.page_table
356 .modify_range(range, &flush_dirty_range)
357 .map_err(|_| MemoryTrackerError::FlushRegionFailed)?;
358 }
359 Ok(())
360 }
361
362 /// Handles permission fault for read-only blocks by setting writable-dirty state.
363 /// In general, this should be called from the exception handler when hardware dirty
364 /// state management is disabled or unavailable.
365 pub fn handle_permission_fault(&mut self, addr: usize) -> Result<()> {
366 self.page_table
367 .modify_range(&(addr..addr + 1), &mark_dirty_block)
368 .map_err(|_| MemoryTrackerError::SetPteDirtyFailed)
369 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000370}
371
372impl Drop for MemoryTracker {
373 fn drop(&mut self) {
Alice Wang4dd20932023-05-26 13:47:16 +0000374 set_dbm_enabled(false);
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100375 self.flush_dirty_pages().unwrap();
Jakob Vukalovic4c1edbe2023-04-17 19:10:57 +0100376 self.unshare_all_memory();
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000377 }
378}
Andrew Walbran19690632022-12-07 16:41:30 +0000379
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000380/// Allocates a memory range of at least the given size and alignment that is shared with the host.
381/// Returns a pointer to the buffer.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000382pub fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
383 assert_ne!(layout.size(), 0);
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000384 let Some(buffer) = try_shared_alloc(layout) else {
Andrew Walbran848decf2022-12-15 14:39:38 +0000385 handle_alloc_error(layout);
386 };
387
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000388 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000389 Ok(buffer)
390}
391
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000392fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
393 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
394
Andrew Walbran87933f32023-05-09 15:29:06 +0000395 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
396 Some(NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000397 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
398 shared_memory.refill(&mut shared_pool, layout);
Andrew Walbran87933f32023-05-09 15:29:06 +0000399 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000400 } else {
401 None
402 }
403}
404
Andrew Walbran848decf2022-12-15 14:39:38 +0000405/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
406///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000407/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
Andrew Walbran848decf2022-12-15 14:39:38 +0000408///
409/// # Safety
410///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000411/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
Andrew Walbran848decf2022-12-15 14:39:38 +0000412/// deallocated.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000413pub unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
Andrew Walbran87933f32023-05-09 15:29:06 +0000414 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700415
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000416 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000417 Ok(())
418}
419
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100420/// Checks whether block flags indicate it should be MMIO guard mapped.
421fn verify_lazy_mapped_block(
422 _range: &VaRange,
423 desc: &mut Descriptor,
424 level: usize,
425) -> result::Result<(), ()> {
426 let flags = desc.flags().expect("Unsupported PTE flags set");
427 if !is_leaf_pte(&flags, level) {
428 return Ok(()); // Skip table PTEs as they aren't tagged with MMIO_LAZY_MAP_FLAG.
429 }
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000430 if flags.contains(MMIO_LAZY_MAP_FLAG) && !flags.contains(Attributes::VALID) {
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100431 Ok(())
432 } else {
433 Err(())
434 }
435}
436
437/// MMIO guard unmaps page
438fn mmio_guard_unmap_page(
439 va_range: &VaRange,
440 desc: &mut Descriptor,
441 level: usize,
442) -> result::Result<(), ()> {
443 let flags = desc.flags().expect("Unsupported PTE flags set");
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100444 if !is_leaf_pte(&flags, level) {
445 return Ok(());
446 }
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100447 // This function will be called on an address range that corresponds to a device. Only if a
448 // page has been accessed (written to or read from), will it contain the VALID flag and be MMIO
449 // guard mapped. Therefore, we can skip unmapping invalid pages, they were never MMIO guard
450 // mapped anyway.
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100451 if flags.contains(Attributes::VALID) {
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100452 assert!(
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000453 flags.contains(MMIO_LAZY_MAP_FLAG),
Jakob Vukalovicb99905d2023-04-20 15:46:02 +0100454 "Attempting MMIO guard unmap for non-device pages"
455 );
456 assert_eq!(
457 va_range.len(),
458 PVMFW_PAGE_SIZE,
459 "Failed to break down block mapping before MMIO guard mapping"
460 );
461 let page_base = va_range.start().0;
462 assert_eq!(page_base % PVMFW_PAGE_SIZE, 0);
463 // Since mmio_guard_map takes IPAs, if pvmfw moves non-ID address mapping, page_base
464 // should be converted to IPA. However, since 0x0 is a valid MMIO address, we don't use
465 // virt_to_phys here, and just pass page_base instead.
466 get_hypervisor().mmio_guard_unmap(page_base).map_err(|e| {
467 error!("Error MMIO guard unmapping: {e}");
468 })?;
469 }
470 Ok(())
471}
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100472
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100473/// Clears read-only flag on a PTE, making it writable-dirty. Used when dirty state is managed
474/// in software to handle permission faults on read-only descriptors.
475fn mark_dirty_block(
476 va_range: &VaRange,
477 desc: &mut Descriptor,
478 level: usize,
479) -> result::Result<(), ()> {
480 let flags = desc.flags().ok_or(())?;
481 if !is_leaf_pte(&flags, level) {
482 return Ok(());
483 }
484 if flags.contains(Attributes::DBM) {
485 assert!(flags.contains(Attributes::READ_ONLY), "unexpected PTE writable state");
486 desc.modify_flags(Attributes::empty(), Attributes::READ_ONLY);
487 // Updating the read-only bit of a PTE requires TLB invalidation.
488 // A TLB maintenance instruction is only guaranteed to be complete after a DSB instruction.
489 // An ISB instruction is required to ensure the effects of completed TLB maintenance
490 // instructions are visible to instructions fetched afterwards.
491 // See ARM ARM E2.3.10, and G5.9.
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000492 tlbi!("vale1", PT_ASID, va_range.start().0);
Jakob Vukalovic44b1ce32023-04-17 19:10:10 +0100493 dsb!("ish");
494 isb!();
495 Ok(())
496 } else {
497 Err(())
498 }
499}
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000500
501/// Returns memory range reserved for the appended payload.
502pub fn appended_payload_range() -> Range<usize> {
Alice Wangeacb7382023-06-05 12:53:54 +0000503 let start = align_up(layout::binary_end(), SIZE_4KB).unwrap();
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000504 // pvmfw is contained in a 2MiB region so the payload can't be larger than the 2MiB alignment.
Alice Wangeacb7382023-06-05 12:53:54 +0000505 let end = align_up(start, SIZE_2MB).unwrap();
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000506 start..end
507}
508
509/// Region allocated for the stack.
510pub fn stack_range() -> Range<usize> {
511 const STACK_PAGES: usize = 8;
512
513 layout::stack_range(STACK_PAGES * PVMFW_PAGE_SIZE)
514}
515
516pub fn init_page_table() -> result::Result<PageTable, MapError> {
517 let mut page_table: PageTable = IdMap::new(PT_ASID, PT_ROOT_LEVEL).into();
518
519 // Stack and scratch ranges are explicitly zeroed and flushed before jumping to payload,
520 // so dirty state management can be omitted.
521 page_table.map_data(&layout::scratch_range())?;
522 page_table.map_data(&stack_range())?;
523 page_table.map_code(&layout::text_range())?;
524 page_table.map_rodata(&layout::rodata_range())?;
525 page_table.map_data_dbm(&appended_payload_range())?;
Alice Wang807fa592023-06-02 09:54:43 +0000526 if let Err(e) = page_table.map_device(&layout::console_uart_range()) {
527 error!("Failed to remap the UART as a dynamic page table entry: {e}");
528 return Err(e);
529 }
Pierre-Clément Tosiad1fc752023-05-31 16:56:56 +0000530 Ok(page_table)
531}