blob: 3faee5b0d337f0f897b066044eb73eaf6dba5cc0 [file] [log] [blame]
Alice Wangf47b2342023-06-02 11:51:57 +00001// Copyright 2023, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Shared memory management.
16
Alice Wang93ee98a2023-06-08 08:20:39 +000017use super::dbm::{flush_dirty_range, mark_dirty_block, set_dbm_enabled};
18use super::error::MemoryTrackerError;
19use super::page_table::{is_leaf_pte, PageTable, MMIO_LAZY_MAP_FLAG};
20use super::util::{page_4kb_of, virt_to_phys};
21use crate::dsb;
22use crate::util::RangeExt as _;
Alice Wanga3931aa2023-07-05 12:52:09 +000023use aarch64_paging::paging::{Attributes, Descriptor, MemoryRegion as VaRange, VirtualAddress};
Alice Wangf47b2342023-06-02 11:51:57 +000024use alloc::alloc::{alloc_zeroed, dealloc, handle_alloc_error};
Alice Wang93ee98a2023-06-08 08:20:39 +000025use alloc::boxed::Box;
Alice Wangf47b2342023-06-02 11:51:57 +000026use alloc::vec::Vec;
Alice Wang93ee98a2023-06-08 08:20:39 +000027use buddy_system_allocator::{FrameAllocator, LockedFrameAllocator};
Alice Wangf47b2342023-06-02 11:51:57 +000028use core::alloc::Layout;
Alice Wang93ee98a2023-06-08 08:20:39 +000029use core::num::NonZeroUsize;
30use core::ops::Range;
Alice Wangf47b2342023-06-02 11:51:57 +000031use core::ptr::NonNull;
Alice Wangb73a81b2023-06-07 13:05:09 +000032use core::result;
Pierre-Clément Tosi92154762023-06-07 15:32:15 +000033use hyp::{get_hypervisor, MMIO_GUARD_GRANULE_SIZE};
Alice Wang93ee98a2023-06-08 08:20:39 +000034use log::{debug, error, trace};
35use once_cell::race::OnceBox;
36use spin::mutex::SpinMutex;
37use tinyvec::ArrayVec;
38
39/// A global static variable representing the system memory tracker, protected by a spin mutex.
40pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
41
42static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
43static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
44
45/// Memory range.
46pub type MemoryRange = Range<usize>;
Alice Wanga3931aa2023-07-05 12:52:09 +000047
48fn get_va_range(range: &MemoryRange) -> VaRange {
49 VaRange::new(range.start, range.end)
50}
51
Alice Wang93ee98a2023-06-08 08:20:39 +000052type Result<T> = result::Result<T, MemoryTrackerError>;
53
54#[derive(Clone, Copy, Debug, Default, PartialEq)]
55enum MemoryType {
56 #[default]
57 ReadOnly,
58 ReadWrite,
59}
60
61#[derive(Clone, Debug, Default)]
62struct MemoryRegion {
63 range: MemoryRange,
64 mem_type: MemoryType,
65}
66
67/// Tracks non-overlapping slices of main memory.
68pub struct MemoryTracker {
69 total: MemoryRange,
70 page_table: PageTable,
71 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
72 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
73 mmio_range: MemoryRange,
Alice Wang5bb79502023-06-12 09:25:07 +000074 payload_range: Option<MemoryRange>,
Alice Wang93ee98a2023-06-08 08:20:39 +000075}
76
Andrew Walbranc06e7342023-07-05 14:00:51 +000077// TODO: Remove this once aarch64-paging crate is updated.
78// SAFETY: Only `PageTable` doesn't implement Send, but it should.
Alice Wang93ee98a2023-06-08 08:20:39 +000079unsafe impl Send for MemoryTracker {}
80
81impl MemoryTracker {
82 const CAPACITY: usize = 5;
83 const MMIO_CAPACITY: usize = 5;
84
85 /// Creates a new instance from an active page table, covering the maximum RAM size.
86 pub fn new(
87 mut page_table: PageTable,
88 total: MemoryRange,
89 mmio_range: MemoryRange,
Alice Wanga3931aa2023-07-05 12:52:09 +000090 payload_range: Option<Range<VirtualAddress>>,
Alice Wang93ee98a2023-06-08 08:20:39 +000091 ) -> Self {
92 assert!(
93 !total.overlaps(&mmio_range),
94 "MMIO space should not overlap with the main memory region."
95 );
96
97 // Activate dirty state management first, otherwise we may get permission faults immediately
98 // after activating the new page table. This has no effect before the new page table is
99 // activated because none of the entries in the initial idmap have the DBM flag.
100 set_dbm_enabled(true);
101
102 debug!("Activating dynamic page table...");
Andrew Walbranc06e7342023-07-05 14:00:51 +0000103 // SAFETY: page_table duplicates the static mappings for everything that the Rust code is
Alice Wang93ee98a2023-06-08 08:20:39 +0000104 // aware of so activating it shouldn't have any visible effect.
105 unsafe { page_table.activate() }
106 debug!("... Success!");
107
108 Self {
109 total,
110 page_table,
111 regions: ArrayVec::new(),
112 mmio_regions: ArrayVec::new(),
113 mmio_range,
Alice Wanga3931aa2023-07-05 12:52:09 +0000114 payload_range: payload_range.map(|r| r.start.0..r.end.0),
Alice Wang93ee98a2023-06-08 08:20:39 +0000115 }
116 }
117
118 /// Resize the total RAM size.
119 ///
120 /// This function fails if it contains regions that are not included within the new size.
121 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
122 if range.start != self.total.start {
123 return Err(MemoryTrackerError::DifferentBaseAddress);
124 }
125 if self.total.end < range.end {
126 return Err(MemoryTrackerError::SizeTooLarge);
127 }
128 if !self.regions.iter().all(|r| r.range.is_within(range)) {
129 return Err(MemoryTrackerError::SizeTooSmall);
130 }
131
132 self.total = range.clone();
133 Ok(())
134 }
135
136 /// Allocate the address range for a const slice; returns None if failed.
137 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
138 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
139 self.check(&region)?;
Alice Wanga3931aa2023-07-05 12:52:09 +0000140 self.page_table.map_rodata(&get_va_range(range)).map_err(|e| {
Alice Wang93ee98a2023-06-08 08:20:39 +0000141 error!("Error during range allocation: {e}");
142 MemoryTrackerError::FailedToMap
143 })?;
144 self.add(region)
145 }
146
147 /// Allocate the address range for a mutable slice; returns None if failed.
148 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
149 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
150 self.check(&region)?;
Alice Wanga3931aa2023-07-05 12:52:09 +0000151 self.page_table.map_data_dbm(&get_va_range(range)).map_err(|e| {
Alice Wang93ee98a2023-06-08 08:20:39 +0000152 error!("Error during mutable range allocation: {e}");
153 MemoryTrackerError::FailedToMap
154 })?;
155 self.add(region)
156 }
157
158 /// Allocate the address range for a const slice; returns None if failed.
159 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
160 self.alloc_range(&(base..(base + size.get())))
161 }
162
163 /// Allocate the address range for a mutable slice; returns None if failed.
164 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
165 self.alloc_range_mut(&(base..(base + size.get())))
166 }
167
168 /// Checks that the given range of addresses is within the MMIO region, and then maps it
169 /// appropriately.
170 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
171 if !range.is_within(&self.mmio_range) {
172 return Err(MemoryTrackerError::OutOfRange);
173 }
174 if self.mmio_regions.iter().any(|r| range.overlaps(r)) {
175 return Err(MemoryTrackerError::Overlaps);
176 }
177 if self.mmio_regions.len() == self.mmio_regions.capacity() {
178 return Err(MemoryTrackerError::Full);
179 }
180
Alice Wanga3931aa2023-07-05 12:52:09 +0000181 self.page_table.map_device_lazy(&get_va_range(&range)).map_err(|e| {
Alice Wang93ee98a2023-06-08 08:20:39 +0000182 error!("Error during MMIO device mapping: {e}");
183 MemoryTrackerError::FailedToMap
184 })?;
185
186 if self.mmio_regions.try_push(range).is_some() {
187 return Err(MemoryTrackerError::Full);
188 }
189
190 Ok(())
191 }
192
193 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
194 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
195 /// add it.
196 fn check(&self, region: &MemoryRegion) -> Result<()> {
197 if !region.range.is_within(&self.total) {
198 return Err(MemoryTrackerError::OutOfRange);
199 }
200 if self.regions.iter().any(|r| region.range.overlaps(&r.range)) {
201 return Err(MemoryTrackerError::Overlaps);
202 }
203 if self.regions.len() == self.regions.capacity() {
204 return Err(MemoryTrackerError::Full);
205 }
206 Ok(())
207 }
208
209 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
210 if self.regions.try_push(region).is_some() {
211 return Err(MemoryTrackerError::Full);
212 }
213
214 Ok(self.regions.last().unwrap().range.clone())
215 }
216
217 /// Unmaps all tracked MMIO regions from the MMIO guard.
218 ///
219 /// Note that they are not unmapped from the page table.
220 pub fn mmio_unmap_all(&mut self) -> Result<()> {
221 for range in &self.mmio_regions {
222 self.page_table
Alice Wanga3931aa2023-07-05 12:52:09 +0000223 .modify_range(&get_va_range(range), &mmio_guard_unmap_page)
Alice Wang93ee98a2023-06-08 08:20:39 +0000224 .map_err(|_| MemoryTrackerError::FailedToUnmap)?;
225 }
226 Ok(())
227 }
228
229 /// Initialize the shared heap to dynamically share memory from the global allocator.
Alice Wangb6d2c642023-06-13 13:07:06 +0000230 pub fn init_dynamic_shared_pool(&mut self, granule: usize) -> Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000231 const INIT_CAP: usize = 10;
232
Alice Wang93ee98a2023-06-08 08:20:39 +0000233 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule, INIT_CAP));
234 if previous.is_some() {
235 return Err(MemoryTrackerError::SharedMemorySetFailure);
236 }
237
238 SHARED_POOL
239 .set(Box::new(LockedFrameAllocator::new()))
240 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
241
242 Ok(())
243 }
244
245 /// Initialize the shared heap from a static region of memory.
246 ///
247 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
248 /// to share its memory with host. Instead they allow host to designate part
249 /// of guest memory as "shared" ahead of guest starting its execution. The
250 /// shared memory region is indicated in swiotlb node. On such platforms use
251 /// a separate heap to allocate buffers that can be shared with host.
252 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
253 let size = NonZeroUsize::new(range.len()).unwrap();
254 let range = self.alloc_mut(range.start, size)?;
255 let shared_pool = LockedFrameAllocator::<32>::new();
256
257 shared_pool.lock().insert(range);
258
259 SHARED_POOL
260 .set(Box::new(shared_pool))
261 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
262
263 Ok(())
264 }
265
266 /// Unshares any memory that may have been shared.
267 pub fn unshare_all_memory(&mut self) {
268 drop(SHARED_MEMORY.lock().take());
269 }
270
271 /// Handles translation fault for blocks flagged for lazy MMIO mapping by enabling the page
272 /// table entry and MMIO guard mapping the block. Breaks apart a block entry if required.
Alice Wang88736462023-07-05 12:14:15 +0000273 pub fn handle_mmio_fault(&mut self, addr: VirtualAddress) -> Result<()> {
274 let page_start = VirtualAddress(page_4kb_of(addr.0));
Alice Wanga3931aa2023-07-05 12:52:09 +0000275 let page_range: VaRange = (page_start..page_start + MMIO_GUARD_GRANULE_SIZE).into();
Alice Wang93ee98a2023-06-08 08:20:39 +0000276 self.page_table
277 .modify_range(&page_range, &verify_lazy_mapped_block)
278 .map_err(|_| MemoryTrackerError::InvalidPte)?;
Alice Wanga3931aa2023-07-05 12:52:09 +0000279 get_hypervisor().mmio_guard_map(page_start.0)?;
Alice Wang93ee98a2023-06-08 08:20:39 +0000280 // Maps a single device page, breaking up block mappings if necessary.
281 self.page_table.map_device(&page_range).map_err(|_| MemoryTrackerError::FailedToMap)
282 }
283
284 /// Flush all memory regions marked as writable-dirty.
285 fn flush_dirty_pages(&mut self) -> Result<()> {
286 // Collect memory ranges for which dirty state is tracked.
287 let writable_regions =
288 self.regions.iter().filter(|r| r.mem_type == MemoryType::ReadWrite).map(|r| &r.range);
289 // Execute a barrier instruction to ensure all hardware updates to the page table have been
290 // observed before reading PTE flags to determine dirty state.
291 dsb!("ish");
292 // Now flush writable-dirty pages in those regions.
Alice Wang5bb79502023-06-12 09:25:07 +0000293 for range in writable_regions.chain(self.payload_range.as_ref().into_iter()) {
Alice Wang93ee98a2023-06-08 08:20:39 +0000294 self.page_table
Alice Wanga3931aa2023-07-05 12:52:09 +0000295 .modify_range(&get_va_range(range), &flush_dirty_range)
Alice Wang93ee98a2023-06-08 08:20:39 +0000296 .map_err(|_| MemoryTrackerError::FlushRegionFailed)?;
297 }
298 Ok(())
299 }
300
301 /// Handles permission fault for read-only blocks by setting writable-dirty state.
302 /// In general, this should be called from the exception handler when hardware dirty
303 /// state management is disabled or unavailable.
Alice Wang88736462023-07-05 12:14:15 +0000304 pub fn handle_permission_fault(&mut self, addr: VirtualAddress) -> Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000305 self.page_table
Alice Wanga3931aa2023-07-05 12:52:09 +0000306 .modify_range(&(addr..addr + 1).into(), &mark_dirty_block)
Alice Wang93ee98a2023-06-08 08:20:39 +0000307 .map_err(|_| MemoryTrackerError::SetPteDirtyFailed)
308 }
309}
310
311impl Drop for MemoryTracker {
312 fn drop(&mut self) {
313 set_dbm_enabled(false);
314 self.flush_dirty_pages().unwrap();
315 self.unshare_all_memory();
316 }
317}
318
319/// Allocates a memory range of at least the given size and alignment that is shared with the host.
320/// Returns a pointer to the buffer.
321pub fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
322 assert_ne!(layout.size(), 0);
323 let Some(buffer) = try_shared_alloc(layout) else {
324 handle_alloc_error(layout);
325 };
326
327 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
328 Ok(buffer)
329}
330
331fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
332 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
333
334 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
335 Some(NonNull::new(buffer as _).unwrap())
336 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
337 shared_memory.refill(&mut shared_pool, layout);
338 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
339 } else {
340 None
341 }
342}
343
344/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
345///
346/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
347///
348/// # Safety
349///
350/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
351/// deallocated.
352pub unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
353 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
354
355 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
356 Ok(())
357}
Alice Wangf47b2342023-06-02 11:51:57 +0000358
359/// Allocates memory on the heap and shares it with the host.
360///
361/// Unshares all pages when dropped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000362struct MemorySharer {
Alice Wangf47b2342023-06-02 11:51:57 +0000363 granule: usize,
364 shared_regions: Vec<(usize, Layout)>,
365}
366
367impl MemorySharer {
368 /// Constructs a new `MemorySharer` instance with the specified granule size and capacity.
369 /// `granule` must be a power of 2.
Alice Wang93ee98a2023-06-08 08:20:39 +0000370 fn new(granule: usize, capacity: usize) -> Self {
Alice Wangf47b2342023-06-02 11:51:57 +0000371 assert!(granule.is_power_of_two());
372 Self { granule, shared_regions: Vec::with_capacity(capacity) }
373 }
374
Alice Wang93ee98a2023-06-08 08:20:39 +0000375 /// Gets from the global allocator a granule-aligned region that suits `hint` and share it.
376 fn refill(&mut self, pool: &mut FrameAllocator<32>, hint: Layout) {
Alice Wangf47b2342023-06-02 11:51:57 +0000377 let layout = hint.align_to(self.granule).unwrap().pad_to_align();
378 assert_ne!(layout.size(), 0);
Andrew Walbranc06e7342023-07-05 14:00:51 +0000379 // SAFETY: layout has non-zero size.
Alice Wangf47b2342023-06-02 11:51:57 +0000380 let Some(shared) = NonNull::new(unsafe { alloc_zeroed(layout) }) else {
381 handle_alloc_error(layout);
382 };
383
384 let base = shared.as_ptr() as usize;
385 let end = base.checked_add(layout.size()).unwrap();
386 trace!("Sharing memory region {:#x?}", base..end);
387 for vaddr in (base..end).step_by(self.granule) {
388 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
389 get_hypervisor().mem_share(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
390 }
391 self.shared_regions.push((base, layout));
392
393 pool.add_frame(base, end);
394 }
395}
396
397impl Drop for MemorySharer {
398 fn drop(&mut self) {
399 while let Some((base, layout)) = self.shared_regions.pop() {
400 let end = base.checked_add(layout.size()).unwrap();
401 trace!("Unsharing memory region {:#x?}", base..end);
402 for vaddr in (base..end).step_by(self.granule) {
403 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
404 get_hypervisor().mem_unshare(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
405 }
406
Andrew Walbranc06e7342023-07-05 14:00:51 +0000407 // SAFETY: The region was obtained from alloc_zeroed() with the recorded layout.
Alice Wangf47b2342023-06-02 11:51:57 +0000408 unsafe { dealloc(base as *mut _, layout) };
409 }
410 }
411}
Alice Wangb73a81b2023-06-07 13:05:09 +0000412
413/// Checks whether block flags indicate it should be MMIO guard mapped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000414fn verify_lazy_mapped_block(
Alice Wangb73a81b2023-06-07 13:05:09 +0000415 _range: &VaRange,
416 desc: &mut Descriptor,
417 level: usize,
418) -> result::Result<(), ()> {
419 let flags = desc.flags().expect("Unsupported PTE flags set");
420 if !is_leaf_pte(&flags, level) {
421 return Ok(()); // Skip table PTEs as they aren't tagged with MMIO_LAZY_MAP_FLAG.
422 }
423 if flags.contains(MMIO_LAZY_MAP_FLAG) && !flags.contains(Attributes::VALID) {
424 Ok(())
425 } else {
426 Err(())
427 }
428}
429
430/// MMIO guard unmaps page
Alice Wang93ee98a2023-06-08 08:20:39 +0000431fn mmio_guard_unmap_page(
Alice Wangb73a81b2023-06-07 13:05:09 +0000432 va_range: &VaRange,
433 desc: &mut Descriptor,
434 level: usize,
435) -> result::Result<(), ()> {
436 let flags = desc.flags().expect("Unsupported PTE flags set");
437 if !is_leaf_pte(&flags, level) {
438 return Ok(());
439 }
440 // This function will be called on an address range that corresponds to a device. Only if a
441 // page has been accessed (written to or read from), will it contain the VALID flag and be MMIO
442 // guard mapped. Therefore, we can skip unmapping invalid pages, they were never MMIO guard
443 // mapped anyway.
444 if flags.contains(Attributes::VALID) {
445 assert!(
446 flags.contains(MMIO_LAZY_MAP_FLAG),
447 "Attempting MMIO guard unmap for non-device pages"
448 );
449 assert_eq!(
450 va_range.len(),
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000451 MMIO_GUARD_GRANULE_SIZE,
Alice Wangb73a81b2023-06-07 13:05:09 +0000452 "Failed to break down block mapping before MMIO guard mapping"
453 );
454 let page_base = va_range.start().0;
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000455 assert_eq!(page_base % MMIO_GUARD_GRANULE_SIZE, 0);
Alice Wangb73a81b2023-06-07 13:05:09 +0000456 // Since mmio_guard_map takes IPAs, if pvmfw moves non-ID address mapping, page_base
457 // should be converted to IPA. However, since 0x0 is a valid MMIO address, we don't use
458 // virt_to_phys here, and just pass page_base instead.
459 get_hypervisor().mmio_guard_unmap(page_base).map_err(|e| {
460 error!("Error MMIO guard unmapping: {e}");
461 })?;
462 }
463 Ok(())
464}