blob: 4a75b976f87eb6d427bf9a6a68269365b448eeb3 [file] [log] [blame]
Alice Wangf47b2342023-06-02 11:51:57 +00001// Copyright 2023, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Shared memory management.
16
Alice Wang93ee98a2023-06-08 08:20:39 +000017use super::dbm::{flush_dirty_range, mark_dirty_block, set_dbm_enabled};
18use super::error::MemoryTrackerError;
19use super::page_table::{is_leaf_pte, PageTable, MMIO_LAZY_MAP_FLAG};
20use super::util::{page_4kb_of, virt_to_phys};
21use crate::dsb;
22use crate::util::RangeExt as _;
Alice Wangb73a81b2023-06-07 13:05:09 +000023use aarch64_paging::paging::{Attributes, Descriptor, MemoryRegion as VaRange};
Alice Wangf47b2342023-06-02 11:51:57 +000024use alloc::alloc::{alloc_zeroed, dealloc, handle_alloc_error};
Alice Wang93ee98a2023-06-08 08:20:39 +000025use alloc::boxed::Box;
Alice Wangf47b2342023-06-02 11:51:57 +000026use alloc::vec::Vec;
Alice Wang93ee98a2023-06-08 08:20:39 +000027use buddy_system_allocator::{FrameAllocator, LockedFrameAllocator};
Alice Wangf47b2342023-06-02 11:51:57 +000028use core::alloc::Layout;
Alice Wang93ee98a2023-06-08 08:20:39 +000029use core::num::NonZeroUsize;
30use core::ops::Range;
Alice Wangf47b2342023-06-02 11:51:57 +000031use core::ptr::NonNull;
Alice Wangb73a81b2023-06-07 13:05:09 +000032use core::result;
Pierre-Clément Tosi92154762023-06-07 15:32:15 +000033use hyp::{get_hypervisor, MMIO_GUARD_GRANULE_SIZE};
Alice Wang93ee98a2023-06-08 08:20:39 +000034use log::{debug, error, trace};
35use once_cell::race::OnceBox;
36use spin::mutex::SpinMutex;
37use tinyvec::ArrayVec;
38
39/// A global static variable representing the system memory tracker, protected by a spin mutex.
40pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
41
42static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
43static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
44
45/// Memory range.
46pub type MemoryRange = Range<usize>;
47type Result<T> = result::Result<T, MemoryTrackerError>;
48
49#[derive(Clone, Copy, Debug, Default, PartialEq)]
50enum MemoryType {
51 #[default]
52 ReadOnly,
53 ReadWrite,
54}
55
56#[derive(Clone, Debug, Default)]
57struct MemoryRegion {
58 range: MemoryRange,
59 mem_type: MemoryType,
60}
61
62/// Tracks non-overlapping slices of main memory.
63pub struct MemoryTracker {
64 total: MemoryRange,
65 page_table: PageTable,
66 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
67 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
68 mmio_range: MemoryRange,
Alice Wang5bb79502023-06-12 09:25:07 +000069 payload_range: Option<MemoryRange>,
Alice Wang93ee98a2023-06-08 08:20:39 +000070}
71
Andrew Walbranc06e7342023-07-05 14:00:51 +000072// TODO: Remove this once aarch64-paging crate is updated.
73// SAFETY: Only `PageTable` doesn't implement Send, but it should.
Alice Wang93ee98a2023-06-08 08:20:39 +000074unsafe impl Send for MemoryTracker {}
75
76impl MemoryTracker {
77 const CAPACITY: usize = 5;
78 const MMIO_CAPACITY: usize = 5;
79
80 /// Creates a new instance from an active page table, covering the maximum RAM size.
81 pub fn new(
82 mut page_table: PageTable,
83 total: MemoryRange,
84 mmio_range: MemoryRange,
Alice Wang5bb79502023-06-12 09:25:07 +000085 payload_range: Option<MemoryRange>,
Alice Wang93ee98a2023-06-08 08:20:39 +000086 ) -> Self {
87 assert!(
88 !total.overlaps(&mmio_range),
89 "MMIO space should not overlap with the main memory region."
90 );
91
92 // Activate dirty state management first, otherwise we may get permission faults immediately
93 // after activating the new page table. This has no effect before the new page table is
94 // activated because none of the entries in the initial idmap have the DBM flag.
95 set_dbm_enabled(true);
96
97 debug!("Activating dynamic page table...");
Andrew Walbranc06e7342023-07-05 14:00:51 +000098 // SAFETY: page_table duplicates the static mappings for everything that the Rust code is
Alice Wang93ee98a2023-06-08 08:20:39 +000099 // aware of so activating it shouldn't have any visible effect.
100 unsafe { page_table.activate() }
101 debug!("... Success!");
102
103 Self {
104 total,
105 page_table,
106 regions: ArrayVec::new(),
107 mmio_regions: ArrayVec::new(),
108 mmio_range,
109 payload_range,
110 }
111 }
112
113 /// Resize the total RAM size.
114 ///
115 /// This function fails if it contains regions that are not included within the new size.
116 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
117 if range.start != self.total.start {
118 return Err(MemoryTrackerError::DifferentBaseAddress);
119 }
120 if self.total.end < range.end {
121 return Err(MemoryTrackerError::SizeTooLarge);
122 }
123 if !self.regions.iter().all(|r| r.range.is_within(range)) {
124 return Err(MemoryTrackerError::SizeTooSmall);
125 }
126
127 self.total = range.clone();
128 Ok(())
129 }
130
131 /// Allocate the address range for a const slice; returns None if failed.
132 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
133 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
134 self.check(&region)?;
135 self.page_table.map_rodata(range).map_err(|e| {
136 error!("Error during range allocation: {e}");
137 MemoryTrackerError::FailedToMap
138 })?;
139 self.add(region)
140 }
141
142 /// Allocate the address range for a mutable slice; returns None if failed.
143 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
144 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
145 self.check(&region)?;
146 self.page_table.map_data_dbm(range).map_err(|e| {
147 error!("Error during mutable range allocation: {e}");
148 MemoryTrackerError::FailedToMap
149 })?;
150 self.add(region)
151 }
152
153 /// Allocate the address range for a const slice; returns None if failed.
154 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
155 self.alloc_range(&(base..(base + size.get())))
156 }
157
158 /// Allocate the address range for a mutable slice; returns None if failed.
159 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
160 self.alloc_range_mut(&(base..(base + size.get())))
161 }
162
163 /// Checks that the given range of addresses is within the MMIO region, and then maps it
164 /// appropriately.
165 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
166 if !range.is_within(&self.mmio_range) {
167 return Err(MemoryTrackerError::OutOfRange);
168 }
169 if self.mmio_regions.iter().any(|r| range.overlaps(r)) {
170 return Err(MemoryTrackerError::Overlaps);
171 }
172 if self.mmio_regions.len() == self.mmio_regions.capacity() {
173 return Err(MemoryTrackerError::Full);
174 }
175
176 self.page_table.map_device_lazy(&range).map_err(|e| {
177 error!("Error during MMIO device mapping: {e}");
178 MemoryTrackerError::FailedToMap
179 })?;
180
181 if self.mmio_regions.try_push(range).is_some() {
182 return Err(MemoryTrackerError::Full);
183 }
184
185 Ok(())
186 }
187
188 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
189 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
190 /// add it.
191 fn check(&self, region: &MemoryRegion) -> Result<()> {
192 if !region.range.is_within(&self.total) {
193 return Err(MemoryTrackerError::OutOfRange);
194 }
195 if self.regions.iter().any(|r| region.range.overlaps(&r.range)) {
196 return Err(MemoryTrackerError::Overlaps);
197 }
198 if self.regions.len() == self.regions.capacity() {
199 return Err(MemoryTrackerError::Full);
200 }
201 Ok(())
202 }
203
204 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
205 if self.regions.try_push(region).is_some() {
206 return Err(MemoryTrackerError::Full);
207 }
208
209 Ok(self.regions.last().unwrap().range.clone())
210 }
211
212 /// Unmaps all tracked MMIO regions from the MMIO guard.
213 ///
214 /// Note that they are not unmapped from the page table.
215 pub fn mmio_unmap_all(&mut self) -> Result<()> {
216 for range in &self.mmio_regions {
217 self.page_table
218 .modify_range(range, &mmio_guard_unmap_page)
219 .map_err(|_| MemoryTrackerError::FailedToUnmap)?;
220 }
221 Ok(())
222 }
223
224 /// Initialize the shared heap to dynamically share memory from the global allocator.
Alice Wangb6d2c642023-06-13 13:07:06 +0000225 pub fn init_dynamic_shared_pool(&mut self, granule: usize) -> Result<()> {
Alice Wang93ee98a2023-06-08 08:20:39 +0000226 const INIT_CAP: usize = 10;
227
Alice Wang93ee98a2023-06-08 08:20:39 +0000228 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule, INIT_CAP));
229 if previous.is_some() {
230 return Err(MemoryTrackerError::SharedMemorySetFailure);
231 }
232
233 SHARED_POOL
234 .set(Box::new(LockedFrameAllocator::new()))
235 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
236
237 Ok(())
238 }
239
240 /// Initialize the shared heap from a static region of memory.
241 ///
242 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
243 /// to share its memory with host. Instead they allow host to designate part
244 /// of guest memory as "shared" ahead of guest starting its execution. The
245 /// shared memory region is indicated in swiotlb node. On such platforms use
246 /// a separate heap to allocate buffers that can be shared with host.
247 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
248 let size = NonZeroUsize::new(range.len()).unwrap();
249 let range = self.alloc_mut(range.start, size)?;
250 let shared_pool = LockedFrameAllocator::<32>::new();
251
252 shared_pool.lock().insert(range);
253
254 SHARED_POOL
255 .set(Box::new(shared_pool))
256 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
257
258 Ok(())
259 }
260
261 /// Unshares any memory that may have been shared.
262 pub fn unshare_all_memory(&mut self) {
263 drop(SHARED_MEMORY.lock().take());
264 }
265
266 /// Handles translation fault for blocks flagged for lazy MMIO mapping by enabling the page
267 /// table entry and MMIO guard mapping the block. Breaks apart a block entry if required.
268 pub fn handle_mmio_fault(&mut self, addr: usize) -> Result<()> {
269 let page_range = page_4kb_of(addr)..page_4kb_of(addr) + MMIO_GUARD_GRANULE_SIZE;
270 self.page_table
271 .modify_range(&page_range, &verify_lazy_mapped_block)
272 .map_err(|_| MemoryTrackerError::InvalidPte)?;
273 get_hypervisor().mmio_guard_map(page_range.start)?;
274 // Maps a single device page, breaking up block mappings if necessary.
275 self.page_table.map_device(&page_range).map_err(|_| MemoryTrackerError::FailedToMap)
276 }
277
278 /// Flush all memory regions marked as writable-dirty.
279 fn flush_dirty_pages(&mut self) -> Result<()> {
280 // Collect memory ranges for which dirty state is tracked.
281 let writable_regions =
282 self.regions.iter().filter(|r| r.mem_type == MemoryType::ReadWrite).map(|r| &r.range);
283 // Execute a barrier instruction to ensure all hardware updates to the page table have been
284 // observed before reading PTE flags to determine dirty state.
285 dsb!("ish");
286 // Now flush writable-dirty pages in those regions.
Alice Wang5bb79502023-06-12 09:25:07 +0000287 for range in writable_regions.chain(self.payload_range.as_ref().into_iter()) {
Alice Wang93ee98a2023-06-08 08:20:39 +0000288 self.page_table
289 .modify_range(range, &flush_dirty_range)
290 .map_err(|_| MemoryTrackerError::FlushRegionFailed)?;
291 }
292 Ok(())
293 }
294
295 /// Handles permission fault for read-only blocks by setting writable-dirty state.
296 /// In general, this should be called from the exception handler when hardware dirty
297 /// state management is disabled or unavailable.
298 pub fn handle_permission_fault(&mut self, addr: usize) -> Result<()> {
299 self.page_table
300 .modify_range(&(addr..addr + 1), &mark_dirty_block)
301 .map_err(|_| MemoryTrackerError::SetPteDirtyFailed)
302 }
303}
304
305impl Drop for MemoryTracker {
306 fn drop(&mut self) {
307 set_dbm_enabled(false);
308 self.flush_dirty_pages().unwrap();
309 self.unshare_all_memory();
310 }
311}
312
313/// Allocates a memory range of at least the given size and alignment that is shared with the host.
314/// Returns a pointer to the buffer.
315pub fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
316 assert_ne!(layout.size(), 0);
317 let Some(buffer) = try_shared_alloc(layout) else {
318 handle_alloc_error(layout);
319 };
320
321 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
322 Ok(buffer)
323}
324
325fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
326 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
327
328 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
329 Some(NonNull::new(buffer as _).unwrap())
330 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
331 shared_memory.refill(&mut shared_pool, layout);
332 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
333 } else {
334 None
335 }
336}
337
338/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
339///
340/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
341///
342/// # Safety
343///
344/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
345/// deallocated.
346pub unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
347 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
348
349 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
350 Ok(())
351}
Alice Wangf47b2342023-06-02 11:51:57 +0000352
353/// Allocates memory on the heap and shares it with the host.
354///
355/// Unshares all pages when dropped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000356struct MemorySharer {
Alice Wangf47b2342023-06-02 11:51:57 +0000357 granule: usize,
358 shared_regions: Vec<(usize, Layout)>,
359}
360
361impl MemorySharer {
362 /// Constructs a new `MemorySharer` instance with the specified granule size and capacity.
363 /// `granule` must be a power of 2.
Alice Wang93ee98a2023-06-08 08:20:39 +0000364 fn new(granule: usize, capacity: usize) -> Self {
Alice Wangf47b2342023-06-02 11:51:57 +0000365 assert!(granule.is_power_of_two());
366 Self { granule, shared_regions: Vec::with_capacity(capacity) }
367 }
368
Alice Wang93ee98a2023-06-08 08:20:39 +0000369 /// Gets from the global allocator a granule-aligned region that suits `hint` and share it.
370 fn refill(&mut self, pool: &mut FrameAllocator<32>, hint: Layout) {
Alice Wangf47b2342023-06-02 11:51:57 +0000371 let layout = hint.align_to(self.granule).unwrap().pad_to_align();
372 assert_ne!(layout.size(), 0);
Andrew Walbranc06e7342023-07-05 14:00:51 +0000373 // SAFETY: layout has non-zero size.
Alice Wangf47b2342023-06-02 11:51:57 +0000374 let Some(shared) = NonNull::new(unsafe { alloc_zeroed(layout) }) else {
375 handle_alloc_error(layout);
376 };
377
378 let base = shared.as_ptr() as usize;
379 let end = base.checked_add(layout.size()).unwrap();
380 trace!("Sharing memory region {:#x?}", base..end);
381 for vaddr in (base..end).step_by(self.granule) {
382 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
383 get_hypervisor().mem_share(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
384 }
385 self.shared_regions.push((base, layout));
386
387 pool.add_frame(base, end);
388 }
389}
390
391impl Drop for MemorySharer {
392 fn drop(&mut self) {
393 while let Some((base, layout)) = self.shared_regions.pop() {
394 let end = base.checked_add(layout.size()).unwrap();
395 trace!("Unsharing memory region {:#x?}", base..end);
396 for vaddr in (base..end).step_by(self.granule) {
397 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
398 get_hypervisor().mem_unshare(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
399 }
400
Andrew Walbranc06e7342023-07-05 14:00:51 +0000401 // SAFETY: The region was obtained from alloc_zeroed() with the recorded layout.
Alice Wangf47b2342023-06-02 11:51:57 +0000402 unsafe { dealloc(base as *mut _, layout) };
403 }
404 }
405}
Alice Wangb73a81b2023-06-07 13:05:09 +0000406
407/// Checks whether block flags indicate it should be MMIO guard mapped.
Alice Wang93ee98a2023-06-08 08:20:39 +0000408fn verify_lazy_mapped_block(
Alice Wangb73a81b2023-06-07 13:05:09 +0000409 _range: &VaRange,
410 desc: &mut Descriptor,
411 level: usize,
412) -> result::Result<(), ()> {
413 let flags = desc.flags().expect("Unsupported PTE flags set");
414 if !is_leaf_pte(&flags, level) {
415 return Ok(()); // Skip table PTEs as they aren't tagged with MMIO_LAZY_MAP_FLAG.
416 }
417 if flags.contains(MMIO_LAZY_MAP_FLAG) && !flags.contains(Attributes::VALID) {
418 Ok(())
419 } else {
420 Err(())
421 }
422}
423
424/// MMIO guard unmaps page
Alice Wang93ee98a2023-06-08 08:20:39 +0000425fn mmio_guard_unmap_page(
Alice Wangb73a81b2023-06-07 13:05:09 +0000426 va_range: &VaRange,
427 desc: &mut Descriptor,
428 level: usize,
429) -> result::Result<(), ()> {
430 let flags = desc.flags().expect("Unsupported PTE flags set");
431 if !is_leaf_pte(&flags, level) {
432 return Ok(());
433 }
434 // This function will be called on an address range that corresponds to a device. Only if a
435 // page has been accessed (written to or read from), will it contain the VALID flag and be MMIO
436 // guard mapped. Therefore, we can skip unmapping invalid pages, they were never MMIO guard
437 // mapped anyway.
438 if flags.contains(Attributes::VALID) {
439 assert!(
440 flags.contains(MMIO_LAZY_MAP_FLAG),
441 "Attempting MMIO guard unmap for non-device pages"
442 );
443 assert_eq!(
444 va_range.len(),
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000445 MMIO_GUARD_GRANULE_SIZE,
Alice Wangb73a81b2023-06-07 13:05:09 +0000446 "Failed to break down block mapping before MMIO guard mapping"
447 );
448 let page_base = va_range.start().0;
Pierre-Clément Tosi92154762023-06-07 15:32:15 +0000449 assert_eq!(page_base % MMIO_GUARD_GRANULE_SIZE, 0);
Alice Wangb73a81b2023-06-07 13:05:09 +0000450 // Since mmio_guard_map takes IPAs, if pvmfw moves non-ID address mapping, page_base
451 // should be converted to IPA. However, since 0x0 is a valid MMIO address, we don't use
452 // virt_to_phys here, and just pass page_base instead.
453 get_hypervisor().mmio_guard_unmap(page_base).map_err(|e| {
454 error!("Error MMIO guard unmapping: {e}");
455 })?;
456 }
457 Ok(())
458}