diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d89924a9..3d21da74 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,8 @@ jobs: - run: cargo doc --no-default-features - run: cargo test --no-default-features --features instructions - run: cargo test --no-default-features --features memory_encryption + - run: cargo test --no-default-features --features virt_addr_57,virt_addr_rt + - run: cargo test --no-default-features --features default_virt_addr_57,virt_addr_rt - run: cargo test --no-default-features test: diff --git a/.gitignore b/.gitignore index b5c891e7..de81fe70 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ /target/ Cargo.lock /testing/target + +# VS Code local config +.vscode/ diff --git a/Cargo.toml b/Cargo.toml index 39519f02..83360c6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,9 @@ default = ["nightly", "instructions"] instructions = [] memory_encryption = [] nightly = ["const_fn", "step_trait", "abi_x86_interrupt", "asm_const"] +virt_addr_57 = [] +virt_addr_rt = [] +default_virt_addr_57 = ["virt_addr_57"] abi_x86_interrupt = [] asm_const = [] step_trait = [] diff --git a/Changelog.md b/Changelog.md index cd17fadd..53692e26 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,37 @@ # Unreleased +## New Features + +- Add the sealed `FixedValidity<48>` policy and feature-gated `FixedValidity<57>` and + `RuntimeValidity` policies for virtual addresses. +- Add the generic `VirtAddrGeneric` type and the `VirtAddr48`, `VirtAddr57`, and `VirtAddrRT` + aliases. +- Add `virt_addr_57` for enabling the 57-bit fixed validity policy and `VirtAddr57` alias, and + `virt_addr_rt` for enabling the `RuntimeValidity` policy and `VirtAddrRT` alias. Add + `default_virt_addr_57` for selecting fixed 57-bit validity as the default. +- Propagate virtual-address validity through pages, descriptor pointers, TSS, GDT, IDT, handler + types, interrupt stack frames, TLB commands, and CET legacy bitmap pages. +- Add `is_valid_currently` for explicitly checking an existing address against the active mode. +- Cache the active virtual-address width after its first use by `VirtAddrRT`. Add + `VirtAddrRT::refetch_virtual_address_bits` for refreshing the cache after changing `CR4.LA57`. + +## Compatibility Notes + +- Without new features, `VirtAddr` remains an alias for a fixed 48-bit virtual address. Existing + constructors retain their names. Generic validity bounds require Rust 1.61 for these methods to + remain `const`, so they are non-const on Rust 1.59 and 1.60. +- Enabling `virt_addr_57` makes the 57-bit fixed policy and its alias available. Enabling + `virt_addr_rt` makes both the runtime policy and its alias available. Enabling + `default_virt_addr_57` also enables `virt_addr_57` and changes `VirtAddr` and validity-aware + aggregate defaults to fixed 57-bit validity. +- Runtime checked construction and address-producing operations are available only on `x86_64` + with the `instructions` feature and require ring 0. The first such operation caches `CR4.LA57`. + Call `VirtAddrRT::refetch_virtual_address_bits` after changing `CR4.LA57`. Storage-only + operations such as `zero`, `new_unsafe`, formatting, and comparison remain available elsewhere. +- Validity is checked when a value is created. Later address-space mode changes do not + retroactively invalidate existing values. +- The mapper stack remains limited to four-level page tables and explicitly accepts VA48 pages. + # 0.15.5 – 2026-07-11 This release is compatible with Rust nightlies starting with `nightly-2026-07-10` (this only applies when the `nightly` feature is used). diff --git a/src/addr.rs b/src/addr/mod.rs similarity index 62% rename from src/addr.rs rename to src/addr/mod.rs index e3645034..387545ac 100644 --- a/src/addr.rs +++ b/src/addr/mod.rs @@ -2,8 +2,10 @@ use core::convert::TryFrom; use core::fmt; +use core::hash::Hash; #[cfg(feature = "step_trait")] use core::iter::Step; +use core::marker::PhantomData; use core::ops::{Add, AddAssign, Sub, SubAssign}; #[cfg(feature = "memory_encryption")] use core::sync::atomic::Ordering; @@ -13,10 +15,51 @@ use crate::structures::mem_encrypt::ENC_BIT_MASK; use crate::structures::paging::page_table::PageTableLevel; use crate::structures::paging::{PageOffset, PageTableIndex}; -use bit_field::BitField; use dep_const_fn::const_fn; -const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000; +#[cfg(feature = "virt_addr_rt")] +mod rt; +mod validity; + +#[cfg(feature = "virt_addr_rt")] +pub use rt::{RuntimeValidity, VirtAddrRT}; +pub(crate) use validity::ArithmeticValidity; +pub use validity::{FixedValidity, VirtAddrValidity}; + +/// Canonicalizes the given address with the given number of bits. +#[inline] +const fn canonicalize_with_bits(addr: u64, bits: usize) -> u64 { + let shift = 64 - bits; + ((addr << shift) as i64 >> shift) as u64 +} + +/// Tries to create a new canonical virtual address with the given number of bits. +/// +/// # Safety +/// +/// The caller must ensure that `bits` is valid for the selected validity policy. This is not +/// checked. +#[inline] +#[rustversion::attr(since(1.61), const)] +unsafe fn try_new_with_bits( + addr: u64, + bits: usize, +) -> Result, VirtAddrNotValid> { + let canonicalized = canonicalize_with_bits(addr, bits); + if canonicalized == addr { + Ok(VirtAddrGeneric(canonicalized, PhantomData)) + } else { + Err(VirtAddrNotValid(addr)) + } +} + +/// Creates a canonical virtual address by discarding invalid high bits, with the given number of +/// bits. +#[inline] +#[rustversion::attr(since(1.61), const)] +fn new_truncate_with_bits(addr: u64, bits: usize) -> VirtAddrGeneric { + VirtAddrGeneric(canonicalize_with_bits(addr, bits), PhantomData) +} /// A canonical 64-bit virtual memory address. /// @@ -25,12 +68,86 @@ const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000; /// [`TryFrom`](https://doc.rust-lang.org/std/convert/trait.TryFrom.html) trait can be used for performing conversions /// between `u64` and `usize`. /// -/// On `x86_64`, only the 48 lower bits of a virtual address can be used. The top 16 bits need -/// to be copies of bit 47, i.e. the most significant bit. Addresses that fulfil this criterion -/// are called “canonical”. This type guarantees that it always represents a canonical address. +/// On `x86_64`, virtual addresses are canonical when all bits above the most significant valid bit +/// are copies of that bit. Currently, two address-space modes are supported on `x86_64`: +/// +/// - Four-level paging (48-bit): The most significant valid bit is bit 47. +/// - Five-level paging (57-bit): The most significant valid bit is bit 56. +/// +/// [`VirtAddrGeneric`] uses [`VirtAddrValidity`] to create different types of virtual addresses for +/// different modes: +/// +/// - [`VirtAddr48`]: A virtual address that is canonical under four-level paging. (A 48-bit +/// canonical virtual address.) +/// - `VirtAddr57` (with `virt_addr_57`): A virtual address that is canonical under five-level +/// paging. (A 57-bit canonical virtual address.) +/// - `VirtAddrRT` (with `virt_addr_rt`): A virtual address that is canonical under the currently +/// active address-space mode. Validity is checked only when an address is created. A later +/// address-space mode change does not invalidate existing values. +/// +/// [`VirtAddr48`] and `VirtAddr57` provide const-capable constructors and accessors. +/// `VirtAddrRT` can be stored, compared, formatted, inspected, and created through +/// [`zero`](Self::zero) or unsafe [`new_unsafe`](Self::new_unsafe) on all targets. Operations that +/// check the current address-space mode or produce a new runtime-valid address use a cached +/// virtual-address width. They require the `instructions` feature and an `x86_64` target, and they +/// must execute in Ring 0. The first such operation initializes the cache from `CR4.LA57`. Call +/// `VirtAddrRT::refetch_virtual_address_bits` after changing the active address-space mode. +/// +/// Validity is checked only when an address is created. A later address-space mode change does not +/// invalidate existing values. Operations that subsequently produce a new address check the +/// result against the cached mode at that time. After changing `CR4.LA57`, update the cache before +/// creating or validating runtime-valid addresses. Use `is_valid_currently` to explicitly +/// revalidate an existing address when current-mode checks are available. +/// +/// The validity parameter is intentionally required. Use [`VirtAddr`] when the validity should +/// follow the crate's feature-selected default. +/// +/// ```compile_fail +/// use x86_64::addr::VirtAddrGeneric; +/// +/// let _ = VirtAddrGeneric::zero(); +/// ``` #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] -pub struct VirtAddr(u64); +pub struct VirtAddrGeneric(u64, PhantomData); + +/// A virtual address that is canonical under four-level paging. +pub type VirtAddr48 = VirtAddrGeneric>; + +/// A virtual address that is canonical under five-level paging. +/// +/// This alias is available with the `virt_addr_57` feature. +#[cfg(feature = "virt_addr_57")] +#[cfg_attr(feature = "doc_cfg", doc(cfg(feature = "virt_addr_57")))] +pub type VirtAddr57 = VirtAddrGeneric>; + +/// The default virtual-address validity policy. +/// +/// This is [`FixedValidity<48>`] by default and [`FixedValidity<57>`] when the +/// `default_virt_addr_57` feature is enabled. +#[cfg(not(feature = "default_virt_addr_57"))] +pub type DefaultVirtAddrValidity = FixedValidity<48>; + +/// The default virtual-address validity policy. +/// +/// This is [`FixedValidity<48>`] by default and [`FixedValidity<57>`] when the +/// `default_virt_addr_57` feature is enabled. +#[cfg(feature = "default_virt_addr_57")] +pub type DefaultVirtAddrValidity = FixedValidity<57>; + +/// The default virtual address type. +/// +/// This is an alias for [`VirtAddr48`] by default and `VirtAddr57` when the +/// `default_virt_addr_57` feature is enabled. +#[cfg(not(feature = "default_virt_addr_57"))] +pub type VirtAddr = VirtAddr48; + +/// The default virtual address type. +/// +/// This is an alias for [`VirtAddr48`] by default and [`VirtAddr57`] when the +/// `default_virt_addr_57` feature is enabled. +#[cfg(feature = "default_virt_addr_57")] +pub type VirtAddr = VirtAddr57; /// A 64-bit physical memory address. /// @@ -47,10 +164,8 @@ pub struct PhysAddr(u64); /// A passed `u64` was not a valid virtual address. /// -/// This means that bits 48 to 64 are not -/// a valid sign extension and are not null either. So automatic sign extension would have -/// overwritten possibly meaningful bits. This likely indicates a bug, for example an invalid -/// address calculation. +/// Automatic sign extension for the selected validity policy would have overwritten possibly +/// meaningful bits. This likely indicates a bug, for example an invalid address calculation. /// /// Contains the invalid address. pub struct VirtAddrNotValid(pub u64); @@ -63,137 +178,194 @@ impl core::fmt::Debug for VirtAddrNotValid { } } -impl VirtAddr { - /// Creates a new canonical virtual address. +impl VirtAddrGeneric> +where + FixedValidity: VirtAddrValidity, +{ + /// Creates a new canonical virtual address, with provided fixed width. /// /// The provided address should already be canonical. If you want to check /// whether an address is canonical, use [`try_new`](Self::try_new). /// /// ## Panics /// - /// This function panics if the bits in the range 48 to 64 are invalid - /// (i.e. are not a proper sign extension of bit 47). + /// This function panics if the address is not canonical for the selected fixed width. #[inline] - pub const fn new(addr: u64) -> VirtAddr { + #[rustversion::attr(since(1.61), const)] + pub fn new(addr: u64) -> Self { // TODO: Replace with .ok().expect(msg) when that works on stable. match Self::try_new(addr) { Ok(v) => v, - Err(_) => panic!("virtual address must be sign extended in bits 48 to 64"), + Err(_) => panic!("virtual address must be canonical for the selected fixed width"), } } - /// Tries to create a new canonical virtual address. + /// Tries to create a new canonical virtual address, with provided fixed width. /// - /// This function checks whether the given address is canonical - /// and returns an error otherwise. An address is canonical - /// if bits 48 to 64 are a correct sign - /// extension (i.e. copies of bit 47). - #[inline] - pub const fn try_new(addr: u64) -> Result { - let v = Self::new_truncate(addr); - if v.0 == addr { - Ok(v) - } else { - Err(VirtAddrNotValid(addr)) - } + /// This function checks whether the given address is canonical for the selected fixed width + /// and returns an error otherwise. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn try_new(addr: u64) -> Result { + // SAFETY: `BITS` is valid for `FixedValidity`, so this is safe. + unsafe { try_new_with_bits(addr, BITS) } + } + + /// Creates a canonical virtual address by discarding invalid high bits, with provided fixed + /// width. + /// + /// This function sign-extends the selected fixed-width sign bit. If you want to check whether + /// an address is canonical, use [`new`](Self::new) or [`try_new`](Self::try_new). + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn new_truncate(addr: u64) -> Self { + new_truncate_with_bits(addr, BITS) } - /// Creates a new canonical virtual address, throwing out bits 48..64. + /// Creates a fixed-width virtual address from the given pointer. /// - /// This function performs sign extension of bit 47 to make the address - /// canonical, overwriting bits 48 to 64. If you want to check whether an - /// address is canonical, use [`new`](Self::new) or [`try_new`](Self::try_new). + /// The pointer address must be canonical under the selected fixed validity policy. + #[cfg(target_pointer_width = "64")] #[inline] - pub const fn new_truncate(addr: u64) -> VirtAddr { - // By doing the right shift as a signed operation (on a i64), it will - // sign extend the value, repeating the leftmost bit. - VirtAddr(((addr << 16) as i64 >> 16) as u64) + pub fn from_ptr(ptr: *const T) -> Self { + Self::new(ptr as *const () as u64) } + /// Aligns the virtual address upwards to the given alignment. + /// + /// See the [`align_up`] function for more information. + #[inline] + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + Self::new_truncate(align_up(self.0, align.into())) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// See the [`align_down`] function for more information. + #[inline] + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + self.align_down_u64(align.into()) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// This variant accepts the alignment as a `u64` for internal users. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub(crate) fn align_down_u64(self, align: u64) -> Self { + Self::new_truncate(align_down(self.0, align)) + } +} + +impl VirtAddrGeneric { /// Creates a new virtual address, without any checks. /// /// ## Safety /// - /// You must make sure bits 48..64 are equal to bit 47. This is not checked. + /// The caller must ensure that `addr` is valid for `V`. This is not checked. #[inline] - pub const unsafe fn new_unsafe(addr: u64) -> VirtAddr { - VirtAddr(addr) + #[rustversion::attr(since(1.61), const)] + pub unsafe fn new_unsafe(addr: u64) -> Self { + VirtAddrGeneric(addr, PhantomData) } /// Creates a virtual address that points to `0`. #[inline] - pub const fn zero() -> VirtAddr { - VirtAddr(0) + #[rustversion::attr(since(1.61), const)] + pub fn zero() -> Self { + VirtAddrGeneric(0, PhantomData) } /// Converts the address to an `u64`. #[inline] - pub const fn as_u64(self) -> u64 { + #[rustversion::attr(since(1.61), const)] + pub fn as_u64(self) -> u64 { self.0 } - /// Creates a virtual address from the given pointer - #[cfg(target_pointer_width = "64")] - #[inline] - pub fn from_ptr(ptr: *const T) -> Self { - Self::new(ptr as *const () as u64) - } - /// Converts the address to a raw pointer. #[cfg(target_pointer_width = "64")] #[inline] - pub const fn as_ptr(self) -> *const T { + #[rustversion::attr(since(1.61), const)] + pub fn as_ptr(self) -> *const T { self.as_u64() as *const T } /// Converts the address to a mutable raw pointer. #[cfg(target_pointer_width = "64")] #[inline] - pub const fn as_mut_ptr(self) -> *mut T { + #[rustversion::attr(since(1.61), const)] + pub fn as_mut_ptr(self) -> *mut T { self.as_ptr::() as *mut T } /// Convenience method for checking if a virtual address is null. #[inline] - pub const fn is_null(self) -> bool { + #[rustversion::attr(since(1.61), const)] + pub fn is_null(self) -> bool { self.0 == 0 } - /// Aligns the virtual address upwards to the given alignment. - /// - /// See the `align_up` function for more information. - /// - /// # Panics - /// - /// This function panics if the resulting address is higher than - /// `0xffff_ffff_ffff_ffff`. + /// Returns the 12-bit page offset of this virtual address. #[inline] - pub fn align_up(self, align: U) -> Self - where - U: Into, - { - VirtAddr::new_truncate(align_up(self.0, align.into())) + #[rustversion::attr(since(1.61), const)] + pub fn page_offset(self) -> PageOffset { + PageOffset::new_truncate(self.0 as u16) } - /// Aligns the virtual address downwards to the given alignment. - /// - /// See the `align_down` function for more information. + /// Returns the 9-bit level 1 page table index. #[inline] - pub fn align_down(self, align: U) -> Self - where - U: Into, - { - self.align_down_u64(align.into()) + #[rustversion::attr(since(1.61), const)] + pub fn p1_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12) as u16) } - /// Aligns the virtual address downwards to the given alignment. - /// - /// See the `align_down` function for more information. + /// Returns the 9-bit level 2 page table index. #[inline] - pub(crate) const fn align_down_u64(self, align: u64) -> Self { - VirtAddr::new_truncate(align_down(self.0, align)) + #[rustversion::attr(since(1.61), const)] + pub fn p2_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> 9) as u16) } + /// Returns the 9-bit level 3 page table index. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn p3_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9) as u16) + } + + /// Returns the 9-bit level 4 page table index. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn p4_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9 >> 9) as u16) + } + + /// Returns the 9-bit level page table index. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn page_table_index(self, level: PageTableLevel) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> ((level as u8 - 1) * 9)) as u16) + } +} + +#[cfg(feature = "virt_addr_57")] +impl VirtAddrGeneric> { + /// Returns the 9-bit level 5 page table index. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn p5_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9 >> 9 >> 9) as u16) + } +} + +impl VirtAddrGeneric { /// Checks whether the virtual address has the demanded alignment. #[inline] pub fn is_aligned(self, align: U) -> bool @@ -205,44 +377,53 @@ impl VirtAddr { /// Checks whether the virtual address has the demanded alignment. #[inline] - pub(crate) const fn is_aligned_u64(self, align: u64) -> bool { - self.align_down_u64(align).as_u64() == self.as_u64() + #[rustversion::attr(since(1.61), const)] + pub(crate) fn is_aligned_u64(self, align: u64) -> bool { + align_down(self.0, align) == self.0 } - /// Returns the 12-bit page offset of this virtual address. + /// Creates a checked virtual address for an internal policy-generic API. + /// + /// Runtime policies use the cached current address-space mode during this construction. #[inline] - pub const fn page_offset(self) -> PageOffset { - PageOffset::new_truncate(self.0 as u16) + pub(crate) fn new_with_validity(addr: u64) -> Self { + // SAFETY: `V::bits()` is valid for `V`, so this is safe. + match unsafe { try_new_with_bits(addr, V::bits()) } { + Ok(address) => address, + Err(_) => panic!("virtual address must be canonical for its validity policy"), + } } - /// Returns the 9-bit level 1 page table index. + /// Returns the first address in the upper canonical half for this policy. #[inline] - pub const fn p1_index(self) -> PageTableIndex { - PageTableIndex::new_truncate((self.0 >> 12) as u16) + pub(crate) fn upper_half_start() -> Self { + new_truncate_with_bits(1u64 << (V::bits() - 1), V::bits()) } - /// Returns the 9-bit level 2 page table index. + /// Returns the final address in the lower canonical half for this policy. #[inline] - pub const fn p2_index(self) -> PageTableIndex { - PageTableIndex::new_truncate((self.0 >> 12 >> 9) as u16) + pub(crate) fn lower_half_end() -> Self { + unsafe { Self::new_unsafe((1u64 << (V::bits() - 1)) - 1) } } - /// Returns the 9-bit level 3 page table index. + /// Returns the greatest canonical address for this policy. #[inline] - pub const fn p3_index(self) -> PageTableIndex { - PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9) as u16) + pub(crate) fn max_value() -> Self { + unsafe { Self::new_unsafe(u64::MAX) } } - /// Returns the 9-bit level 4 page table index. + /// Tries to create a checked virtual address for an internal policy-generic API. + /// + /// Runtime policies use the cached current address-space mode during this construction. #[inline] - pub const fn p4_index(self) -> PageTableIndex { - PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9 >> 9) as u16) + pub(crate) fn try_new_with_validity(addr: u64) -> Result { + // SAFETY: `V::bits()` is valid for `V`, so this is safe. + unsafe { try_new_with_bits(addr, V::bits()) } } - /// Returns the 9-bit level page table index. #[inline] - pub const fn page_table_index(self, level: PageTableLevel) -> PageTableIndex { - PageTableIndex::new_truncate((self.0 >> 12 >> ((level as u8 - 1) * 9)) as u16) + fn new_truncate_with_validity(addr: u64) -> Self { + VirtAddrGeneric(canonicalize_with_bits(addr, V::bits()), PhantomData) } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. @@ -260,12 +441,8 @@ impl VirtAddr { /// function always returns the exact bound, so it doesn't need to return a /// lower and upper bound like steps_between does. pub(crate) fn steps_between_u64(start: &Self, end: &Self) -> Option { - let mut steps = end.0.checked_sub(start.0)?; - - // Mask away extra bits that appear while jumping the gap. - steps &= 0xffff_ffff_ffff; - - Some(steps) + let mask = (1u64 << V::bits()) - 1; + (end.0 & mask).checked_sub(start.0 & mask) } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. @@ -277,54 +454,26 @@ impl VirtAddr { /// An implementation of forward_checked that takes u64 instead of usize. #[inline] pub(crate) fn forward_checked_u64(start: Self, count: u64) -> Option { - if count > ADDRESS_SPACE_SIZE { - return None; - } - - let mut addr = start.0.checked_add(count)?; - - match addr.get_bits(47..) { - 0x1 => { - // Jump the gap by sign extending the 47th bit. - addr.set_bits(47.., 0x1ffff); - } - 0x2 => { - // Address overflow - return None; - } - _ => {} + let mask = (1u64 << V::bits()) - 1; + let addr = (start.0 & mask).checked_add(count)?; + if addr > mask { + None + } else { + Some(Self::new_truncate_with_validity(addr)) } - - Some(unsafe { Self::new_unsafe(addr) }) } /// An implementation of backward_checked that takes u64 instead of usize. #[cfg(feature = "step_trait")] #[inline] pub(crate) fn backward_checked_u64(start: Self, count: u64) -> Option { - if count > ADDRESS_SPACE_SIZE { - return None; - } - - let mut addr = start.0.checked_sub(count)?; - - match addr.get_bits(47..) { - 0x1fffe => { - // Jump the gap by sign extending the 47th bit. - addr.set_bits(47.., 0); - } - 0x1fffd => { - // Address underflow - return None; - } - _ => {} - } - - Some(unsafe { Self::new_unsafe(addr) }) + let mask = (1u64 << V::bits()) - 1; + let addr = (start.0 & mask).checked_sub(count)?; + Some(Self::new_truncate_with_validity(addr)) } } -impl fmt::Debug for VirtAddr { +impl fmt::Debug for VirtAddrGeneric { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_tuple("VirtAddr") .field(&format_args!("{:#x}", self.0)) @@ -332,42 +481,42 @@ impl fmt::Debug for VirtAddr { } } -impl fmt::Binary for VirtAddr { +impl fmt::Binary for VirtAddrGeneric { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Binary::fmt(&self.0, f) } } -impl fmt::LowerHex for VirtAddr { +impl fmt::LowerHex for VirtAddrGeneric { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::LowerHex::fmt(&self.0, f) } } -impl fmt::Octal for VirtAddr { +impl fmt::Octal for VirtAddrGeneric { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Octal::fmt(&self.0, f) } } -impl fmt::UpperHex for VirtAddr { +impl fmt::UpperHex for VirtAddrGeneric { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::UpperHex::fmt(&self.0, f) } } -impl fmt::Pointer for VirtAddr { +impl fmt::Pointer for VirtAddrGeneric { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Pointer::fmt(&(self.0 as *const ()), f) } } -impl Add for VirtAddr { +impl Add for VirtAddrGeneric { type Output = Self; #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] @@ -383,7 +532,7 @@ impl Add for VirtAddr { /// canonical address. #[inline] fn add(self, rhs: u64) -> Self::Output { - VirtAddr::try_new( + Self::try_new_with_validity( self.0 .checked_add(rhs) .expect("attempt to add with overflow"), @@ -392,7 +541,7 @@ impl Add for VirtAddr { } } -impl AddAssign for VirtAddr { +impl AddAssign for VirtAddrGeneric { #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] /// Add an offset to a virtual address. /// @@ -410,7 +559,7 @@ impl AddAssign for VirtAddr { } } -impl Sub for VirtAddr { +impl Sub for VirtAddrGeneric { type Output = Self; #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] @@ -426,7 +575,7 @@ impl Sub for VirtAddr { /// canonical address. #[inline] fn sub(self, rhs: u64) -> Self::Output { - VirtAddr::try_new( + Self::try_new_with_validity( self.0 .checked_sub(rhs) .expect("attempt to subtract with overflow"), @@ -435,7 +584,7 @@ impl Sub for VirtAddr { } } -impl SubAssign for VirtAddr { +impl SubAssign for VirtAddrGeneric { #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] /// Subtract an offset from a virtual address. /// @@ -453,7 +602,7 @@ impl SubAssign for VirtAddr { } } -impl Sub for VirtAddr { +impl Sub> for VirtAddrGeneric { type Output = u64; /// Returns the difference between two addresses. @@ -462,15 +611,33 @@ impl Sub for VirtAddr { /// /// This function will panic on overflow. #[inline] - fn sub(self, rhs: VirtAddr) -> Self::Output { + fn sub(self, rhs: VirtAddrGeneric) -> Self::Output { self.as_u64() .checked_sub(rhs.as_u64()) .expect("attempt to subtract with overflow") } } +#[cfg(feature = "virt_addr_57")] +impl From for VirtAddr57 { + #[inline] + fn from(address: VirtAddr48) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +#[cfg(feature = "virt_addr_57")] +impl TryFrom for VirtAddr48 { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddr57) -> Result { + Self::try_new(address.as_u64()) + } +} + #[cfg(feature = "step_trait")] -impl Step for VirtAddr { +impl Step for VirtAddrGeneric { #[inline] fn steps_between(start: &Self, end: &Self) -> (usize, Option) { Self::steps_between_impl(start, end) @@ -512,7 +679,10 @@ impl Step for VirtAddr { } #[cfg(kani)] -impl kani::Arbitrary for VirtAddr { +impl kani::Arbitrary for VirtAddrGeneric> +where + FixedValidity: VirtAddrValidity, +{ fn any() -> Self { Self::new_truncate(kani::any()) } @@ -785,10 +955,149 @@ pub const fn align_up(addr: u64, align: u64) -> u64 { mod tests { use super::*; + /// Constructs an unchecked VA48 value for tests of internal arithmetic behavior. + /// + /// This helper preserves the concise tuple-constructor spelling used by the original tests. + #[allow(non_snake_case)] + fn VirtAddr(addr: u64) -> VirtAddr48 { + unsafe { VirtAddr48::new_unsafe(addr) } + } + + #[rustversion::since(1.61)] + const UNSAFE_VIRT_ADDR_48: VirtAddr48 = unsafe { VirtAddr48::new_unsafe(0x1234) }; + #[rustversion::since(1.61)] + #[cfg(feature = "virt_addr_57")] + const UNSAFE_VIRT_ADDR_57: VirtAddr57 = unsafe { VirtAddr57::new_unsafe(0x1234) }; + #[rustversion::since(1.61)] + #[cfg(feature = "virt_addr_rt")] + const UNSAFE_VIRT_ADDR_RT: VirtAddrRT = unsafe { VirtAddrRT::new_unsafe(0x1234) }; + + #[rustversion::since(1.61)] + #[test] + #[cfg(not(feature = "default_virt_addr_57"))] + fn default_virtaddr_is_va48() { + let _: fn(u64) -> VirtAddr48 = crate::VirtAddr::new; + + const FIXED48: VirtAddr48 = VirtAddr48::new(0x1234); + const GENERIC48: VirtAddrGeneric> = + VirtAddrGeneric::>::new(0x1234); + assert_eq!(FIXED48.as_u64(), 0x1234); + assert_eq!(GENERIC48.as_u64(), 0x1234); + } + + #[rustversion::since(1.61)] + #[test] + #[cfg(feature = "default_virt_addr_57")] + fn configured_default_virtaddr_is_va57() { + let _: fn(u64) -> VirtAddr57 = crate::VirtAddr::new; + + const FIXED57: VirtAddr57 = VirtAddr57::new(0x00ff_0000_0000_0000); + assert_eq!(FIXED57.as_u64(), 0x00ff_0000_0000_0000); + } + + #[test] + fn fixed_virtaddr_canonicality() { + assert!(VirtAddr48::try_new(0x0000_7fff_ffff_ffff).is_ok()); + assert!(VirtAddr48::try_new(0x0000_8000_0000_0000).is_err()); + assert!(VirtAddr48::try_new(0xffff_8000_0000_0000).is_ok()); + + #[cfg(feature = "virt_addr_57")] + { + assert!(VirtAddr57::try_new(0x00ff_ffff_ffff_ffff).is_ok()); + assert!(VirtAddr57::try_new(0x0100_0000_0000_0000).is_err()); + assert!(VirtAddr57::try_new(0xff00_0000_0000_0000).is_ok()); + assert!(VirtAddr57::try_new(0x0000_8000_0000_0000).is_ok()); + } + } + + #[test] + fn pure_canonicalization_uses_selected_width() { + assert_eq!(canonicalize_with_bits(1 << 47, 48), 0xffff_8000_0000_0000); + assert_eq!(canonicalize_with_bits(1 << 56, 57), 0xff00_0000_0000_0000); + assert_eq!(canonicalize_with_bits((1 << 47) - 1, 48), (1 << 47) - 1); + assert_eq!(canonicalize_with_bits((1 << 56) - 1, 57), (1 << 56) - 1); + } + + #[test] + #[cfg(all(feature = "step_trait", feature = "virt_addr_57"))] + fn fixed_virtaddr_operations_use_policy_width() { + let low_end = VirtAddr57::new(0x00ff_ffff_ffff_fffe); + assert_eq!((low_end + 1).as_u64(), 0x00ff_ffff_ffff_ffff); + assert_eq!( + Step::forward(low_end + 1, 1).as_u64(), + 0xff00_0000_0000_0000 + ); + assert_eq!( + Step::backward(VirtAddr57::new(0xff00_0000_0000_0000), 1).as_u64(), + 0x00ff_ffff_ffff_ffff + ); + assert_eq!( + VirtAddr57::new(0x00ff_ffff_ffff_ffff) + .align_up(2u64) + .as_u64(), + 0xff00_0000_0000_0000 + ); + } + + #[test] + #[cfg(feature = "virt_addr_57")] + fn fixed_virtaddr_conversions_preserve_or_check_values() { + let address48 = VirtAddr48::new(0xffff_8000_0000_1234); + let address57 = VirtAddr57::from(address48); + + assert_eq!(address57.as_u64(), address48.as_u64()); + assert_eq!(VirtAddr48::try_from(address57).unwrap(), address48); + + let la57_only = VirtAddr57::new(0x0000_8000_0000_0000); + assert!(VirtAddr48::try_from(la57_only).is_err()); + } + + #[test] + fn virtaddr_policy_layout_is_transparent() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + } + + #[rustversion::since(1.61)] + #[test] + fn new_unsafe_is_const_for_all_policies() { + assert_eq!(UNSAFE_VIRT_ADDR_48.as_u64(), 0x1234); + #[cfg(feature = "virt_addr_57")] + assert_eq!(UNSAFE_VIRT_ADDR_57.as_u64(), 0x1234); + #[cfg(feature = "virt_addr_rt")] + assert_eq!(UNSAFE_VIRT_ADDR_RT.as_u64(), 0x1234); + } + #[test] #[should_panic] pub fn add_overflow_virtaddr() { - let _ = VirtAddr::new(0xffff_ffff_ffff_ffff) + 1; + let _ = VirtAddr48::new(0xffff_ffff_ffff_ffff) + 1; } #[test] @@ -800,7 +1109,7 @@ mod tests { #[test] #[should_panic] pub fn sub_underflow_virtaddr() { - let _ = VirtAddr::new(0) - 1; + let _ = VirtAddr48::new(0) - 1; } #[test] @@ -811,10 +1120,10 @@ mod tests { #[test] pub fn virtaddr_new_truncate() { - assert_eq!(VirtAddr::new_truncate(0), VirtAddr(0)); - assert_eq!(VirtAddr::new_truncate(1 << 47), VirtAddr(0xfffff << 47)); - assert_eq!(VirtAddr::new_truncate(123), VirtAddr(123)); - assert_eq!(VirtAddr::new_truncate(123 << 47), VirtAddr(0xfffff << 47)); + assert_eq!(VirtAddr48::new_truncate(0), VirtAddr(0)); + assert_eq!(VirtAddr48::new_truncate(1 << 47), VirtAddr(0xfffff << 47)); + assert_eq!(VirtAddr48::new_truncate(123), VirtAddr(123)); + assert_eq!(VirtAddr48::new_truncate(123 << 47), VirtAddr(0xfffff << 47)); } #[test] @@ -1002,8 +1311,8 @@ mod tests { fn test_virt_addr_align_up() { // Make sure the 47th bit is extended. assert_eq!( - VirtAddr::new(0x7fff_ffff_ffff).align_up(2u64), - VirtAddr::new(0xffff_8000_0000_0000) + VirtAddr48::new(0x7fff_ffff_ffff).align_up(2u64), + VirtAddr48::new(0xffff_8000_0000_0000) ); } @@ -1011,15 +1320,15 @@ mod tests { fn test_virt_addr_align_down() { // Make sure the 47th bit is extended. assert_eq!( - VirtAddr::new(0xffff_8000_0000_0000).align_down(1u64 << 48), - VirtAddr::new(0) + VirtAddr48::new(0xffff_8000_0000_0000).align_down(1u64 << 48), + VirtAddr48::new(0) ); } #[test] #[should_panic] fn test_virt_addr_align_up_overflow() { - VirtAddr::new(0xffff_ffff_ffff_ffff).align_up(2u64); + VirtAddr48::new(0xffff_ffff_ffff_ffff).align_up(2u64); } #[test] @@ -1034,8 +1343,8 @@ mod tests { let slice = &[1, 2, 3, 4, 5]; // Make sure that from_ptr(slice) is the address of the first element assert_eq!( - VirtAddr::from_ptr(slice.as_slice()), - VirtAddr::from_ptr(&slice[0]) + VirtAddr48::from_ptr(slice.as_slice()), + VirtAddr48::from_ptr(&slice[0]) ); } } @@ -1075,7 +1384,7 @@ mod proofs { }; if let Some(expected) = expected { // Verify that `expected` is a valid address. - assert!(VirtAddr::try_new(expected).is_ok()); + assert!(VirtAddr48::try_new(expected).is_ok()); } // Verify `forward_checked`. let next = Step::forward_checked(start, 1); diff --git a/src/addr/rt/instr.rs b/src/addr/rt/instr.rs new file mode 100644 index 00000000..48f0042b --- /dev/null +++ b/src/addr/rt/instr.rs @@ -0,0 +1,266 @@ +//! Runtime virtual-address width operations for the `RuntimeValidity` policy. + +#[cfg(feature = "virt_addr_57")] +use core::convert::TryFrom; +use core::sync::atomic::{AtomicU8, Ordering}; + +#[cfg(feature = "virt_addr_57")] +use crate::addr::VirtAddr57; +use crate::addr::{ + align_down, align_up, canonicalize_with_bits, new_truncate_with_bits, try_new_with_bits, + ArithmeticValidity, VirtAddrGeneric, VirtAddrNotValid, VirtAddrValidity, +}; + +use super::RuntimeValidity; +#[cfg(any(test, feature = "virt_addr_57"))] +use super::VirtAddrRT; + +impl ArithmeticValidity for RuntimeValidity {} + +/// The cached virtual-address width for the active address-space mode. +/// +/// Zero indicates that the cache has not been initialized yet. +static CURRENT_VIRTUAL_ADDRESS_BITS: AtomicU8 = AtomicU8::new(0); + +/// Returns a lazily initialized virtual-address width from the given cache. +#[inline] +fn cached_virtual_address_bits_with( + cache: &AtomicU8, + read_current_bits: impl FnOnce() -> u8, +) -> usize { + let cached = cache.load(Ordering::Relaxed); + if cached != 0 { + return usize::from(cached); + } + + let current = read_current_bits(); + debug_assert!(current == 48 || current == 57); + usize::from( + match cache.compare_exchange(0, current, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => current, + Err(initialized) => initialized, + }, + ) +} + +/// Replaces the virtual-address width in the given cache. +#[inline] +fn refetch_virtual_address_bits_with(cache: &AtomicU8, read_current_bits: impl FnOnce() -> u8) { + let current = read_current_bits(); + debug_assert!(current == 48 || current == 57); + cache.store(current, Ordering::Relaxed); +} + +/// Reads the virtual-address width for the currently active address-space mode. +/// +/// This function must execute in Ring 0. +#[inline] +fn read_current_virtual_address_bits() -> u8 { + use crate::registers::control::{Cr4, Cr4Flags}; + + if Cr4::read().contains(Cr4Flags::L5_PAGING) { + 57 + } else { + 48 + } +} + +/// Refetches and caches the virtual-address width for the active address-space mode. +/// +/// This function must execute in Ring 0. +#[inline] +fn refetch_virtual_address_bits() { + refetch_virtual_address_bits_with( + &CURRENT_VIRTUAL_ADDRESS_BITS, + read_current_virtual_address_bits, + ); +} + +/// Returns the cached virtual-address width for the active address-space mode. +/// +/// This function must execute in Ring 0 if the cache has not been initialized yet. +#[inline] +pub(super) fn cached_virtual_address_bits() -> usize { + cached_virtual_address_bits_with( + &CURRENT_VIRTUAL_ADDRESS_BITS, + read_current_virtual_address_bits, + ) +} + +impl VirtAddrGeneric { + /// Refetches the virtual-address width from the active address-space mode and updates the + /// cached value. + /// + /// The first runtime-valid address operation initializes the cache automatically. Call this + /// method after changing `CR4.LA57` and before resuming operations that create or validate + /// runtime-valid addresses. The caller is responsible for synchronizing the mode change with + /// other processors and threads. + /// + /// This method reads `CR4.LA57`, so it must execute in Ring 0. + #[inline] + pub fn refetch_virtual_address_bits() { + refetch_virtual_address_bits(); + } + + /// Creates a new virtual address valid in the current address-space mode. + /// + /// # Panics + /// + /// This function panics if the address is not canonical under the currently active mode. + #[inline] + pub fn new(addr: u64) -> Self { + match Self::try_new(addr) { + Ok(address) => address, + Err(_) => panic!("virtual address must be canonical in the current address-space mode"), + } + } + + /// Tries to create a virtual address valid in the current address-space mode. + /// + /// This function checks the address using the cached active canonical width. The first runtime + /// address operation initializes the cache from `CR4.LA57`. + #[inline] + pub fn try_new(addr: u64) -> Result { + // SAFETY: `cached_virtual_address_bits()` is valid, at least when the cache is initialized, + // so this is safe. + unsafe { try_new_with_bits(addr, cached_virtual_address_bits()) } + } + + /// Creates a virtual address by canonicalizing it for the current address-space mode. + /// + /// This function uses the cached active canonical width to sign-extend the address. The first + /// runtime address operation initializes the cache from `CR4.LA57`. + #[inline] + pub fn new_truncate(addr: u64) -> Self { + new_truncate_with_bits(addr, cached_virtual_address_bits()) + } + + /// Creates a virtual address from the given pointer. + /// + /// The pointer address must be canonical in the current address-space mode. + #[cfg(target_pointer_width = "64")] + #[inline] + pub fn from_ptr(ptr: *const T) -> Self { + Self::new(ptr as *const () as u64) + } + + /// Aligns the virtual address upwards to the given alignment. + /// + /// The result is canonicalized using the current address-space mode. + #[inline] + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + Self::new_truncate(align_up(self.0, align.into())) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// The result is canonicalized using the current address-space mode. + #[inline] + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + self.align_down_u64(align.into()) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// This variant accepts the alignment as a `u64` for internal users. + #[inline] + pub(crate) fn align_down_u64(self, align: u64) -> Self { + Self::new_truncate(align_down(self.0, align)) + } +} + +impl VirtAddrGeneric { + /// Checks whether the address is canonical in the currently active address-space mode. + /// + /// This method checks the address against the cached active canonical width even though it was + /// valid for its policy when created. + #[inline] + pub fn is_valid_currently(self) -> bool { + canonicalize_with_bits(self.0, cached_virtual_address_bits()) == self.0 + } +} + +#[cfg(feature = "virt_addr_57")] +impl TryFrom for VirtAddrRT { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddr57) -> Result { + Self::try_new(address.as_u64()) + } +} + +#[cfg(test)] +mod tests { + use core::ops::{Add, AddAssign, Sub, SubAssign}; + use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; + + #[cfg(feature = "step_trait")] + use core::iter::Step; + + use crate::addr::VirtAddr48; + + use super::*; + + #[test] + fn runtime_virtaddr_arithmetic_traits_are_available() { + fn assert_arithmetic() + where + T: Add + AddAssign + Sub + SubAssign, + { + } + + assert_arithmetic::(); + + #[cfg(feature = "step_trait")] + { + fn assert_step() {} + assert_step::(); + } + } + + #[test] + fn runtime_virtual_address_bits_are_cached_and_updateable() { + let cache = AtomicU8::new(0); + let reads = AtomicUsize::new(0); + + assert_eq!( + cached_virtual_address_bits_with(&cache, || { + reads.fetch_add(1, Ordering::Relaxed); + 57 + }), + 57 + ); + assert_eq!( + cached_virtual_address_bits_with(&cache, || { + reads.fetch_add(1, Ordering::Relaxed); + 48 + }), + 57 + ); + assert_eq!(reads.load(Ordering::Relaxed), 1); + + refetch_virtual_address_bits_with(&cache, || { + reads.fetch_add(1, Ordering::Relaxed); + 48 + }); + assert_eq!(cached_virtual_address_bits_with(&cache, || 57), 48); + assert_eq!(reads.load(Ordering::Relaxed), 2); + + let _: fn() = VirtAddrRT::refetch_virtual_address_bits; + } + + #[test] + fn current_validity_check_is_available_for_fixed_addresses() { + let _: fn(VirtAddr48) -> bool = VirtAddr48::is_valid_currently; + + #[cfg(feature = "virt_addr_57")] + let _: fn(VirtAddr57) -> bool = VirtAddr57::is_valid_currently; + } +} diff --git a/src/addr/rt/mod.rs b/src/addr/rt/mod.rs new file mode 100644 index 00000000..834a738c --- /dev/null +++ b/src/addr/rt/mod.rs @@ -0,0 +1,116 @@ +//! Runtime virtual-address validity policy. + +use core::convert::TryFrom; + +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +mod instr; + +use crate::structures::paging::PageTableIndex; + +#[cfg(feature = "virt_addr_57")] +use super::VirtAddr57; +use super::{VirtAddr48, VirtAddrGeneric, VirtAddrNotValid, VirtAddrValidity}; + +/// The runtime virtual-address validity policy. +/// +/// This policy checks the currently active address-space mode using a global cache of +/// `CR4.LA57`. The first operation that needs the active mode initializes the cache. The policy +/// type itself is available with the `virt_addr_rt` feature on all targets. Operations that do not +/// consult the active mode are available wherever the policy is available. Checked construction, +/// canonicalization, and address-producing arithmetic additionally require the `instructions` +/// feature and an `x86_64` target, and they must execute in Ring 0. +#[cfg_attr(feature = "doc_cfg", doc(cfg(feature = "virt_addr_rt")))] +#[cfg_attr( + not(all(feature = "instructions", target_arch = "x86_64")), + doc = r#" +Address-producing arithmetic is unavailable when the current address-space mode cannot be read: + +```compile_fail +use x86_64::{RuntimeValidity, VirtAddrGeneric}; + +let address = VirtAddrGeneric::::zero(); +let _ = address + 1u64; +``` +"# +)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RuntimeValidity; + +/// A virtual address checked against the current address-space mode when created. +/// +/// This alias is available with the `virt_addr_rt` feature. +#[cfg_attr(feature = "doc_cfg", doc(cfg(feature = "virt_addr_rt")))] +pub type VirtAddrRT = VirtAddrGeneric; + +impl crate::sealed::Sealed for RuntimeValidity {} + +impl VirtAddrValidity for RuntimeValidity { + fn bits() -> usize { + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + { + instr::cached_virtual_address_bits() + } + + #[cfg(not(all(feature = "instructions", target_arch = "x86_64")))] + { + // All callers of this function are expected to be disabled on non-x86_64 targets or + // when the instructions feature is disabled. + unreachable!( + "runtime virtual-address width requires x86_64 and the instructions feature" + ) + } + } +} + +impl VirtAddrGeneric { + /// Returns the 9-bit level 5 page table index. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn p5_index(self) -> PageTableIndex { + PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9 >> 9 >> 9) as u16) + } +} + +impl From for VirtAddrRT { + #[inline] + fn from(address: VirtAddr48) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +#[cfg(feature = "virt_addr_57")] +impl From for VirtAddr57 { + #[inline] + fn from(address: VirtAddrRT) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +impl TryFrom for VirtAddr48 { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddrRT) -> Result { + Self::try_new(address.as_u64()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runtime_fixed_conversions_preserve_or_check_values() { + let address48 = VirtAddr48::new(0xffff_8000_0000_1234); + let address_rt = VirtAddrRT::from(address48); + + assert_eq!(address_rt.as_u64(), address48.as_u64()); + assert_eq!(VirtAddr48::try_from(address_rt).unwrap(), address48); + + #[cfg(feature = "virt_addr_57")] + { + let address57 = VirtAddr57::from(address_rt); + assert_eq!(address57.as_u64(), address48.as_u64()); + } + } +} diff --git a/src/addr/validity.rs b/src/addr/validity.rs new file mode 100644 index 00000000..38b87efe --- /dev/null +++ b/src/addr/validity.rs @@ -0,0 +1,87 @@ +//! Virtual-address validity and fixed-width validity policies. + +use core::hash::Hash; + +/// A policy for virtual-address validity. +/// +/// This trait is sealed and cannot be implemented outside this crate. Three validities are +/// supported: +/// +/// - [`FixedValidity<48>`]: 48-bit fixed width. +/// - [`FixedValidity<57>`]: 57-bit fixed width (requires the `virt_addr_57` feature). +/// - `RuntimeValidity`: Runtime validity (requires the `virt_addr_rt` feature). +/// +/// This trait is used by [`VirtAddrGeneric`](super::VirtAddrGeneric) to construct different +/// virtual-address types. +/// +/// # Examples +/// +/// ``` +/// use x86_64::addr::{FixedValidity, VirtAddrGeneric}; +/// +/// let addr = VirtAddrGeneric::>::new(0x1000); +/// ``` +/// +/// The set of validity policies is closed: +/// +/// ```compile_fail +/// struct CustomValidity; +/// +/// let _ = x86_64::addr::VirtAddrGeneric::::zero(); +/// ``` +#[cfg_attr( + not(feature = "virt_addr_57"), + doc = r#" +`FixedValidity<57>` requires the `virt_addr_57` feature: + +```compile_fail +use x86_64::{FixedValidity, VirtAddrGeneric}; + +let _ = VirtAddrGeneric::>::zero(); +``` +"# +)] +pub trait VirtAddrValidity: crate::sealed::Sealed + Copy + Ord + Hash { + /// Returns the number of valid bits in the virtual address. + /// + /// This function is not used in const contexts for `FixedValidity`, where the number of bits is + /// known as a const parameter. It is used for common code that is generic over validity + /// policies, including `RuntimeValidity`. + fn bits() -> usize; +} + +/// A fixed-width virtual-address validity policy. +/// +/// `FixedValidity<48>` is always supported. `FixedValidity<57>` is supported with the +/// `virt_addr_57` feature. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FixedValidity; + +impl crate::sealed::Sealed for FixedValidity<48> {} + +#[cfg(feature = "virt_addr_57")] +impl crate::sealed::Sealed for FixedValidity<57> {} + +impl VirtAddrValidity for FixedValidity<48> { + fn bits() -> usize { + 48 + } +} + +#[cfg(feature = "virt_addr_57")] +impl VirtAddrValidity for FixedValidity<57> { + fn bits() -> usize { + 57 + } +} + +/// A [`VirtAddrValidity`] for which arithmetic operations are supported. +/// +/// Enabled fixed validity policies always support arithmetic. `RuntimeValidity` supports +/// arithmetic when the `instructions` feature is enabled and the target is `x86_64`. +pub(crate) trait ArithmeticValidity: VirtAddrValidity {} + +impl ArithmeticValidity for FixedValidity where + FixedValidity: VirtAddrValidity +{ +} diff --git a/src/instructions/tables.rs b/src/instructions/tables.rs index 611d6117..34be735f 100644 --- a/src/instructions/tables.rs +++ b/src/instructions/tables.rs @@ -42,30 +42,51 @@ pub unsafe fn lidt(idt: &DescriptorTablePointer) { } } +/// A raw descriptor-table register value. +/// +/// Assembly writes plain integers here before the base is checked and wrapped in a semantic type. +#[repr(C, packed(2))] +struct RawDescriptorTablePointer { + limit: u16, + base: u64, +} + +#[inline] +fn read_raw_gdt() -> RawDescriptorTablePointer { + let mut pointer = RawDescriptorTablePointer { limit: 0, base: 0 }; + unsafe { + asm!("sgdt [{}]", in(reg) &mut pointer, options(nostack, preserves_flags)); + } + pointer +} + +#[inline] +fn read_raw_idt() -> RawDescriptorTablePointer { + let mut pointer = RawDescriptorTablePointer { limit: 0, base: 0 }; + unsafe { + asm!("sidt [{}]", in(reg) &mut pointer, options(nostack, preserves_flags)); + } + pointer +} + /// Get the address of the current GDT. #[inline] pub fn sgdt() -> DescriptorTablePointer { - let mut gdt: DescriptorTablePointer = DescriptorTablePointer { - limit: 0, - base: VirtAddr::new(0), - }; - unsafe { - asm!("sgdt [{}]", in(reg) &mut gdt, options(nostack, preserves_flags)); + let raw = read_raw_gdt(); + DescriptorTablePointer { + limit: raw.limit, + base: VirtAddr::new(raw.base), } - gdt } /// Get the address of the current IDT. #[inline] pub fn sidt() -> DescriptorTablePointer { - let mut idt: DescriptorTablePointer = DescriptorTablePointer { - limit: 0, - base: VirtAddr::new(0), - }; - unsafe { - asm!("sidt [{}]", in(reg) &mut idt, options(nostack, preserves_flags)); + let raw = read_raw_idt(); + DescriptorTablePointer { + limit: raw.limit, + base: VirtAddr::new(raw.base), } - idt } /// Load the task state register using the `ltr` instruction. diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index d96cc895..b882a104 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -2,6 +2,7 @@ use bit_field::BitField; +use crate::addr::{DefaultVirtAddrValidity, VirtAddrGeneric, VirtAddrValidity}; use crate::{ instructions::segmentation::{Segment, CS}, structures::paging::{ @@ -30,9 +31,9 @@ pub fn flush_all() { /// The Invalidate PCID Command to execute. #[derive(Debug)] -pub enum InvPcidCommand { +pub enum InvPcidCommand { /// The logical processor invalidates mappings—except global translations—for the linear address and PCID specified. - Address(VirtAddr, Pcid), + Address(VirtAddrGeneric, Pcid), /// The logical processor invalidates all mappings—except global translations—associated with the PCID. Single(Pcid), @@ -47,7 +48,7 @@ pub enum InvPcidCommand { // TODO: Remove this in the next breaking release. #[deprecated = "please use `InvPcidCommand` instead"] #[doc(hidden)] -pub type InvPicdCommand = InvPcidCommand; +pub type InvPicdCommand = InvPcidCommand; /// The INVPCID descriptor comprises 128 bits and consists of a PCID and a linear address. /// For INVPCID type 0, the processor uses the full 64 bits of the linear address even outside 64-bit mode; the linear address is not used for other INVPCID types. @@ -99,6 +100,11 @@ impl fmt::Display for PcidTooBig { /// This function is unsafe as it requires CPUID.(EAX=07H, ECX=0H):EBX.INVPCID to be 1. #[inline] pub unsafe fn flush_pcid(command: InvPcidCommand) { + unsafe { flush_pcid_inner(command) } +} + +#[inline] +unsafe fn flush_pcid_inner(command: InvPcidCommand) { let mut desc = InvpcidDescriptor { pcid: 0, address: 0, @@ -225,12 +231,13 @@ impl Invlpgb { /// A builder struct to construct the parameters for the `invlpgb` instruction. #[derive(Debug, Clone)] #[must_use] -pub struct InvlpgbFlushBuilder<'a, S = Size4KiB> +pub struct InvlpgbFlushBuilder<'a, S = Size4KiB, V = DefaultVirtAddrValidity> where S: NotGiantPageSize, + V: VirtAddrValidity, { invlpgb: &'a Invlpgb, - page_range: Option>, + page_range: Option>, pcid: Option, asid: Option, include_global: bool, @@ -238,15 +245,16 @@ where include_nested_translations: bool, } -impl<'a, S> InvlpgbFlushBuilder<'a, S> +impl<'a, S, V> InvlpgbFlushBuilder<'a, S, V> where S: NotGiantPageSize, + V: VirtAddrValidity, { /// Flush a range of pages. /// /// If the range doesn't fit within `invlpgb_count_max`, `invlpgb` is /// executed multiple times. - pub fn pages(self, page_range: PageRange) -> InvlpgbFlushBuilder<'a, T> + pub fn pages(self, page_range: PageRange) -> InvlpgbFlushBuilder<'a, T, V> where T: NotGiantPageSize, { @@ -317,11 +325,14 @@ where if let Some(mut pages) = self.page_range { while !pages.is_empty() { // Calculate out how many pages we still need to flush. - let count = Page::::steps_between_impl(&pages.start, &pages.end).0; + let count = Page::::steps_between_impl(&pages.start, &pages.end).0; // Make sure that we never jump the gap in the address space when flushing. - let second_half_start = - Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked( + VirtAddrGeneric::::upper_half_start(), + ) + }; let count = if pages.start < second_half_start { let count_to_second_half = Page::steps_between_impl(&pages.start, &second_half_start).0; @@ -355,7 +366,7 @@ where } } else { unsafe { - flush_broadcast::( + flush_broadcast::( None, self.pcid, self.asid, @@ -389,8 +400,8 @@ impl fmt::Display for AsidOutOfRangeError { /// See `INVLPGB` in AMD64 Architecture Programmer's Manual Volume 3 #[inline] -unsafe fn flush_broadcast( - va_and_count: Option<(Page, u16)>, +unsafe fn flush_broadcast( + va_and_count: Option<(Page, u16)>, pcid: Option, asid: Option, include_global: bool, @@ -398,6 +409,7 @@ unsafe fn flush_broadcast( include_nested_translations: bool, ) where S: NotGiantPageSize, + V: VirtAddrValidity, { let mut rax = 0; let mut ecx = 0; diff --git a/src/structures/gdt.rs b/src/structures/gdt.rs index bf267e5b..77268f60 100644 --- a/src/structures/gdt.rs +++ b/src/structures/gdt.rs @@ -1,11 +1,12 @@ //! Types for the Global Descriptor Table and segment selectors. +use crate::addr::{DefaultVirtAddrValidity, VirtAddrValidity}; pub use crate::registers::segmentation::SegmentSelector; use crate::structures::tss::{InvalidIoMap, TaskStateSegment}; use crate::PrivilegeLevel; use bit_field::BitField; use bitflags::bitflags; -use core::{cmp, fmt, mem}; +use core::{cmp, fmt, marker::PhantomData, mem}; // imports for intra-doc links #[cfg(doc)] use crate::registers::segmentation::{Segment, CS, SS}; @@ -105,29 +106,30 @@ impl fmt::Debug for Entry { /// ``` #[derive(Debug, Clone)] -pub struct GlobalDescriptorTable { +pub struct GlobalDescriptorTable { table: [Entry; MAX], len: usize, + validity: PhantomData, } -impl GlobalDescriptorTable { +impl GlobalDescriptorTable<8, DefaultVirtAddrValidity> { /// Creates an empty GDT with the default length of 8. pub const fn new() -> Self { Self::empty() } } -impl Default for GlobalDescriptorTable { +impl Default for GlobalDescriptorTable { #[inline] fn default() -> Self { - Self::new() + Self::empty_with_validity() } } -impl GlobalDescriptorTable { +impl GlobalDescriptorTable { /// Creates an empty GDT which can hold `MAX` number of [`Entry`]s. #[inline] - pub const fn empty() -> Self { + pub const fn empty_with_validity() -> Self { // TODO: Replace with compiler error when feature(generic_const_exprs) is stable. assert!(MAX > 0, "A GDT cannot have 0 entries"); assert!(MAX <= (1 << 13), "A GDT can only have at most 2^13 entries"); @@ -138,6 +140,7 @@ impl GlobalDescriptorTable { Self { table: [NULL; MAX], len: 1, + validity: PhantomData, } } @@ -158,9 +161,9 @@ impl GlobalDescriptorTable { allow(rustdoc::broken_intra_doc_links) )] #[inline] - pub const fn from_raw_entries(slice: &[u64]) -> Self { + pub const fn from_raw_entries_with_validity(slice: &[u64]) -> Self { let len = slice.len(); - let mut table = Self::empty().table; + let mut table = Self::empty_with_validity().table; let mut idx = 0; assert!(len > 0, "cannot initialize GDT with empty slice"); @@ -175,7 +178,11 @@ impl GlobalDescriptorTable { idx += 1; } - Self { table, len } + Self { + table, + len, + validity: PhantomData, + } } /// Get a reference to the internal [`Entry`] table. @@ -214,37 +221,6 @@ impl GlobalDescriptorTable { SegmentSelector::new(index as u16, entry.dpl()) } - /// Loads the GDT in the CPU using the `lgdt` instruction. This does **not** alter any of the - /// segment registers; you **must** (re)load them yourself using [the appropriate - /// functions](crate::instructions::segmentation): - /// [`SS::set_reg()`] and [`CS::set_reg()`]. - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub fn load(&'static self) { - // SAFETY: static lifetime ensures no modification after loading. - unsafe { self.load_unsafe() }; - } - - /// Loads the GDT in the CPU using the `lgdt` instruction. This does **not** alter any of the - /// segment registers; you **must** (re)load them yourself using [the appropriate - /// functions](crate::instructions::segmentation): - /// [`SS::set_reg()`] and [`CS::set_reg()`]. - /// - /// # Safety - /// - /// Unlike `load` this function will not impose a static lifetime constraint - /// this means its up to the user to ensure that there will be no modifications - /// after loading and that the GDT will live for as long as it's loaded. - /// - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub unsafe fn load_unsafe(&self) { - use crate::instructions::tables::lgdt; - unsafe { - lgdt(&self.pointer()); - } - } - #[inline] #[rustversion::attr(since(1.83), const)] fn push(&mut self, value: u64) -> usize { @@ -265,14 +241,55 @@ impl GlobalDescriptorTable { /// Creates the descriptor pointer for this table. This pointer can only be /// safely used if the table is never modified or destroyed while in use. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - fn pointer(&self) -> super::DescriptorTablePointer { + fn pointer(&self) -> super::DescriptorTablePointer + where + V: VirtAddrValidity, + { super::DescriptorTablePointer { - base: crate::VirtAddr::new(self.table.as_ptr() as u64), + base: crate::addr::VirtAddrGeneric::::new_with_validity(self.table.as_ptr() as u64), limit: self.limit(), } } } +impl GlobalDescriptorTable { + /// Loads the GDT in the CPU using the `lgdt` instruction. + /// + /// The static lifetime ensures that the table is not destroyed while loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub fn load(&'static self) { + unsafe { self.load_unsafe() }; + } + + /// Loads the GDT without imposing a static lifetime. + /// + /// # Safety + /// + /// The caller must keep the GDT alive and unmodified while it is loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub unsafe fn load_unsafe(&self) { + unsafe { crate::instructions::tables::lgdt(&self.pointer()) }; + } + + /// Creates an empty GDT with the selected default virtual-address validity. + /// + /// This method preserves the legacy `GlobalDescriptorTable::::empty` API. + #[inline] + pub const fn empty() -> Self { + Self::empty_with_validity() + } + + /// Forms a GDT with the default virtual-address validity from a slice of raw entries. + /// + /// This method preserves the legacy `from_raw_entries` API. + #[inline] + pub const fn from_raw_entries(slice: &[u64]) -> Self { + Self::from_raw_entries_with_validity(slice) + } +} + /// A 64-bit mode segment descriptor. /// /// Segmentation is no longer supported in 64-bit mode, so most of the descriptor @@ -431,6 +448,16 @@ impl Descriptor { unsafe { Self::tss_segment_unchecked(tss) } } + /// Creates a TSS system descriptor with the selected validity type. + #[inline] + pub fn tss_segment_with_validity(tss: &'static TaskStateSegment) -> Descriptor + where + V: VirtAddrValidity, + { + // SAFETY: The pointer is derived from a &'static reference, which ensures its validity. + unsafe { Self::tss_segment_unchecked_with_validity(tss) } + } + /// Similar to [`Descriptor::tss_segment`], but unsafe since it does not enforce a lifetime /// constraint on the provided TSS. /// @@ -443,6 +470,22 @@ impl Descriptor { unsafe { Self::tss_segment_raw(tss, 0) } } + /// Creates a TSS descriptor with the selected validity type from a raw pointer. + /// + /// # Safety + /// The caller must ensure that the passed pointer is valid for as long as the descriptor is + /// being used. + #[inline] + pub unsafe fn tss_segment_unchecked_with_validity( + tss: *const TaskStateSegment, + ) -> Descriptor + where + V: VirtAddrValidity, + { + // SAFETY: if iomap_size is zero, there are no requirements to uphold. + unsafe { Self::tss_segment_raw(tss, 0) } + } + /// Creates a TSS system descriptor for the given TSS, setting up the IO permissions bitmap. /// /// # Example @@ -450,25 +493,38 @@ impl Descriptor { /// ``` /// use x86_64::structures::gdt::Descriptor; /// use x86_64::structures::tss::TaskStateSegment; + /// use x86_64::addr::FixedValidity; /// /// /// A helper that places some I/O map bytes behind a TSS. /// #[repr(C)] /// struct TssWithIOMap { - /// tss: TaskStateSegment, + /// tss: TaskStateSegment>, /// iomap: [u8; 5], /// } /// - /// static TSS: TssWithIOMap = TssWithIOMap { - /// tss: TaskStateSegment::new(), + /// let tss = Box::leak(Box::new(TssWithIOMap { + /// tss: TaskStateSegment::new_with_validity(), /// iomap: [0xff, 0xff, 0x00, 0x80, 0xff], - /// }; + /// })); /// - /// let tss = Descriptor::tss_segment_with_iomap(&TSS.tss, &TSS.iomap).unwrap(); + /// let descriptor = + /// Descriptor::tss_segment_with_iomap_with_validity(&tss.tss, &tss.iomap).unwrap(); /// ``` pub fn tss_segment_with_iomap( tss: &'static TaskStateSegment, iomap: &'static [u8], ) -> Result { + Self::tss_segment_with_iomap_with_validity(tss, iomap) + } + + /// Creates a TSS descriptor with an I/O bitmap and the selected validity type. + pub fn tss_segment_with_iomap_with_validity( + tss: &'static TaskStateSegment, + iomap: &'static [u8], + ) -> Result + where + V: VirtAddrValidity, + { if iomap.len() > 8193 { return Err(InvalidIoMap::TooLong { len: iomap.len() }); } @@ -508,10 +564,13 @@ impl Descriptor { /// There must be a valid IO map at `(tss as *const u8).offset(tss.iomap_base)` /// of length `iomap_size`, with the terminating `0xFF` byte. Additionally, `iomap_base` must /// not exceed `0xDFFF`. - unsafe fn tss_segment_raw(tss: *const TaskStateSegment, iomap_size: u16) -> Descriptor { + unsafe fn tss_segment_raw(tss: *const TaskStateSegment, iomap_size: u16) -> Descriptor + where + V: VirtAddrValidity, + { use self::DescriptorFlags as Flags; - let ptr = tss as u64; + let ptr = crate::addr::VirtAddrGeneric::::new_with_validity(tss as u64).as_u64(); let mut low = Flags::PRESENT.bits(); // base @@ -521,7 +580,7 @@ impl Descriptor { let iomap_limit = u64::from(unsafe { (*tss).iomap_base }) + u64::from(iomap_size); low.set_bits( 0..16, - cmp::max(mem::size_of::() as u64, iomap_limit) - 1, + cmp::max(mem::size_of::>() as u64, iomap_limit) - 1, ); // type (0b1001 = available 64-bit tss) low.set_bits(40..44, 0b1001); @@ -538,6 +597,31 @@ mod tests { use super::DescriptorFlags as Flags; use super::*; + #[test] + fn policy_does_not_change_gdt_layout() { + #[cfg(target_pointer_width = "64")] + const EXPECTED_SIZE: usize = 72; + #[cfg(target_pointer_width = "32")] + const EXPECTED_SIZE: usize = 68; + + assert_eq!(mem::size_of::(), EXPECTED_SIZE); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + mem::size_of::>>(), + EXPECTED_SIZE + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + mem::size_of::>(), + EXPECTED_SIZE + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + mem::align_of::>(), + 8 + ); + } + #[test] #[rustfmt::skip] pub fn linux_kernel_defaults() { @@ -563,11 +647,13 @@ mod tests { gdt } - static TSS: TaskStateSegment = TaskStateSegment::new(); + fn tss() -> &'static TaskStateSegment> { + Box::leak(Box::new(TaskStateSegment::new_with_validity())) + } fn make_full_gdt() -> GlobalDescriptorTable { let mut gdt = make_six_entry_gdt(); - gdt.append(Descriptor::tss_segment(&TSS)); + gdt.append(Descriptor::tss_segment_with_validity(tss())); assert_eq!(gdt.len, 8); gdt } @@ -597,7 +683,7 @@ mod tests { let mut gdt = make_six_entry_gdt(); gdt.append(Descriptor::user_data_segment()); // We have one free slot, but the GDT requires two - gdt.append(Descriptor::tss_segment(&TSS)); + gdt.append(Descriptor::tss_segment_with_validity(tss())); } #[test] diff --git a/src/structures/idt.rs b/src/structures/idt.rs index f15cedcc..5d1dcaa3 100644 --- a/src/structures/idt.rs +++ b/src/structures/idt.rs @@ -20,8 +20,9 @@ //! //! These types are defined for the compatibility with the Nightly Rust build. +use crate::addr::{DefaultVirtAddrValidity, VirtAddrGeneric, VirtAddrValidity}; use crate::registers::rflags::RFlags; -use crate::{PrivilegeLevel, VirtAddr}; +use crate::PrivilegeLevel; use bit_field::BitField; use bitflags::bitflags; use core::convert::TryFrom; @@ -54,7 +55,7 @@ use super::gdt::SegmentSelector; #[derive(Clone, Debug)] #[repr(C)] #[repr(align(16))] -pub struct InterruptDescriptorTable { +pub struct InterruptDescriptorTable { /// A divide error (`#DE`) occurs when the denominator of a DIV instruction or /// an IDIV instruction is 0. A `#DE` also occurs if the result is too large to be /// represented in the destination. @@ -62,7 +63,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#DE`. /// /// The vector number of the `#DE` exception is 0. - pub divide_error: Entry, + pub divide_error: Entry, V>, /// When the debug-exception mechanism is enabled, a `#DB` exception can occur under any /// of the following circumstances: @@ -94,7 +95,7 @@ pub struct InterruptDescriptorTable { /// instruction pointer points to the instruction after the one that caused the `#DB`. /// /// The vector number of the `#DB` exception is 1. - pub debug: Entry, + pub debug: Entry, V>, /// An non maskable interrupt exception (NMI) occurs as a result of system logic /// signaling a non-maskable interrupt to the processor. @@ -104,7 +105,7 @@ pub struct InterruptDescriptorTable { /// boundary where the NMI was recognized. /// /// The vector number of the NMI exception is 2. - pub non_maskable_interrupt: Entry, + pub non_maskable_interrupt: Entry, V>, /// A breakpoint (`#BP`) exception occurs when an `INT3` instruction is executed. The /// `INT3` is normally used by debug software to set instruction breakpoints by replacing @@ -112,7 +113,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the byte after the `INT3` instruction. /// /// The vector number of the `#BP` exception is 3. - pub breakpoint: Entry, + pub breakpoint: Entry, V>, /// An overflow exception (`#OF`) occurs as a result of executing an `INTO` instruction /// while the overflow bit in `RFLAGS` is set to 1. @@ -121,7 +122,7 @@ pub struct InterruptDescriptorTable { /// instruction that caused the `#OF`. /// /// The vector number of the `#OF` exception is 4. - pub overflow: Entry, + pub overflow: Entry, V>, /// A bound-range exception (`#BR`) exception can occur as a result of executing /// the `BOUND` instruction. The `BOUND` instruction compares an array index (first @@ -131,7 +132,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the `BOUND` instruction that caused the `#BR`. /// /// The vector number of the `#BR` exception is 5. - pub bound_range_exceeded: Entry, + pub bound_range_exceeded: Entry, V>, /// An invalid opcode exception (`#UD`) occurs when an attempt is made to execute an /// invalid or undefined opcode. The validity of an opcode often depends on the @@ -165,7 +166,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#UD`. /// /// The vector number of the `#UD` exception is 6. - pub invalid_opcode: Entry, + pub invalid_opcode: Entry, V>, /// A device not available exception (`#NM`) occurs under any of the following conditions: /// @@ -182,7 +183,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#NM`. /// /// The vector number of the `#NM` exception is 7. - pub device_not_available: Entry, + pub device_not_available: Entry, V>, /// A double fault (`#DF`) exception can occur when a second exception occurs during /// the handling of a prior (first) exception or interrupt handler. @@ -216,14 +217,14 @@ pub struct InterruptDescriptorTable { /// and the program cannot be restarted. /// /// The vector number of the `#DF` exception is 8. - pub double_fault: Entry, + pub double_fault: Entry, V>, /// This interrupt vector is reserved. It is for a discontinued exception originally used /// by processors that supported external x87-instruction coprocessors. On those processors, /// the exception condition is caused by an invalid-segment or invalid-page access on an /// x87-instruction coprocessor-instruction operand. On current processors, this condition /// causes a general-protection exception to occur. - coprocessor_segment_overrun: Entry, + coprocessor_segment_overrun: Entry, V>, /// An invalid TSS exception (`#TS`) occurs only as a result of a control transfer through /// a gate descriptor that results in an invalid stack-segment reference using an `SS` @@ -233,7 +234,7 @@ pub struct InterruptDescriptorTable { /// points to the control-transfer instruction that caused the `#TS`. /// /// The vector number of the `#TS` exception is 10. - pub invalid_tss: Entry, + pub invalid_tss: Entry, V>, /// An segment-not-present exception (`#NP`) occurs when an attempt is made to load a /// segment or gate with a clear present bit. @@ -243,7 +244,7 @@ pub struct InterruptDescriptorTable { /// that loaded the segment selector resulting in the `#NP`. /// /// The vector number of the `#NP` exception is 11. - pub segment_not_present: Entry, + pub segment_not_present: Entry, V>, /// An stack segment exception (`#SS`) can occur in the following situations: /// @@ -260,7 +261,7 @@ pub struct InterruptDescriptorTable { /// caused the `#SS`. /// /// The vector number of the `#NP` exception is 12. - pub stack_segment_fault: Entry, + pub stack_segment_fault: Entry, V>, /// A general protection fault (`#GP`) can occur in various situations. Common causes include: /// @@ -276,7 +277,7 @@ pub struct InterruptDescriptorTable { /// the instruction that caused the `#GP`. /// /// The vector number of the `#GP` exception is 13. - pub general_protection_fault: Entry, + pub general_protection_fault: Entry, V>, /// A page fault (`#PF`) can occur during a memory access in any of the following situations: /// @@ -297,10 +298,10 @@ pub struct InterruptDescriptorTable { /// [`PageFaultErrorCode`](struct.PageFaultErrorCode.html) struct. /// /// The vector number of the `#PF` exception is 14. - pub page_fault: Entry, + pub page_fault: Entry, V>, /// vector nr. 15 - reserved_1: Entry, + reserved_1: Entry, V>, /// The x87 Floating-Point Exception-Pending exception (`#MF`) is used to handle unmasked x87 /// floating-point exceptions. In 64-bit mode, the x87 floating point unit is not used @@ -308,7 +309,7 @@ pub struct InterruptDescriptorTable { /// compatibility mode. /// /// The vector number of the `#MF` exception is 16. - pub x87_floating_point: Entry, + pub x87_floating_point: Entry, V>, /// An alignment check exception (`#AC`) occurs when an unaligned-memory data reference /// is performed while alignment checking is enabled. An `#AC` can occur only when CPL=3. @@ -317,7 +318,7 @@ pub struct InterruptDescriptorTable { /// instruction that caused the `#AC`. /// /// The vector number of the `#AC` exception is 17. - pub alignment_check: Entry, + pub alignment_check: Entry, V>, /// The machine check exception (`#MC`) is model specific. Processor implementations /// are not required to support the `#MC` exception, and those implementations that do @@ -326,7 +327,7 @@ pub struct InterruptDescriptorTable { /// There is no reliable way to restart the program. /// /// The vector number of the `#MC` exception is 18. - pub machine_check: Entry, + pub machine_check: Entry, V>, /// The SIMD Floating-Point Exception (`#XF`) is used to handle unmasked SSE /// floating-point exceptions. The SSE floating-point exceptions reported by @@ -342,10 +343,10 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#XF`. /// /// The vector number of the `#XF` exception is 19. - pub simd_floating_point: Entry, + pub simd_floating_point: Entry, V>, /// vector nr. 20 - pub virtualization: Entry, + pub virtualization: Entry, V>, /// A #CP exception is generated when shadow stacks are enabled and mismatch /// scenarios are detected (possible error code cases below). @@ -358,10 +359,10 @@ pub struct InterruptDescriptorTable { /// - A missing ENDBRANCH instruction if indirect branch tracking is enabled. /// /// vector nr. 21 - pub cp_protection_exception: Entry, + pub cp_protection_exception: Entry, V>, /// vector nr. 22-27 - reserved_2: [Entry; 6], + reserved_2: [Entry, V>; 6], /// The Hypervisor Injection Exception (`#HV`) is injected by a hypervisor /// as a doorbell to inform an `SEV-SNP` enabled guest running with the @@ -378,7 +379,7 @@ pub struct InterruptDescriptorTable { /// software-managed para-virtualization interface. /// /// The vector number of the ``#HV`` exception is 28. - pub hv_injection_exception: Entry, + pub hv_injection_exception: Entry, V>, /// The VMM Communication Exception (`#VC`) is always generated by hardware when an `SEV-ES` /// enabled guest is running and an `NAE` event occurs. @@ -407,7 +408,7 @@ pub struct InterruptDescriptorTable { /// setting intercept bits for events that would occur in the `#VC` handler (such as `IRET`). /// /// The vector number of the ``#VC`` exception is 29. - pub vmm_communication_exception: Entry, + pub vmm_communication_exception: Entry, V>, /// The Security Exception (`#SX`) signals security-sensitive events that occur while /// executing the VMM, in the form of an exception so that the VMM may take appropriate @@ -418,10 +419,10 @@ pub struct InterruptDescriptorTable { /// The only error code currently defined is 1, and indicates redirection of INIT has occurred. /// /// The vector number of the ``#SX`` exception is 30. - pub security_exception: Entry, + pub security_exception: Entry, V>, /// vector nr. 31 - reserved_3: Entry, + reserved_3: Entry, V>, /// User-defined interrupts can be initiated either by system logic or software. They occur /// when: @@ -443,14 +444,14 @@ pub struct InterruptDescriptorTable { /// external interrupt was recognized. /// - If the interrupt occurs as a result of executing the INTn instruction, the saved /// instruction pointer points to the instruction after the INTn. - interrupts: [Entry; 256 - 32], + interrupts: [Entry, V>; 256 - 32], } -impl InterruptDescriptorTable { +impl InterruptDescriptorTable { /// Creates a new IDT filled with non-present entries. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn new() -> InterruptDescriptorTable { + pub fn new_with_validity() -> Self { InterruptDescriptorTable { divide_error: Entry::missing(), debug: Entry::missing(), @@ -486,42 +487,19 @@ impl InterruptDescriptorTable { /// Resets all entries of this IDT in place. #[inline] pub fn reset(&mut self) { - *self = Self::new(); - } - - /// Loads the IDT in the CPU using the `lidt` command. - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub fn load(&'static self) { - unsafe { self.load_unsafe() } - } - - /// Loads the IDT in the CPU using the `lidt` command. - /// - /// # Safety - /// - /// As long as it is the active IDT, you must ensure that: - /// - /// - `self` is never destroyed. - /// - `self` always stays at the same memory location. It is recommended to wrap it in - /// a `Box`. - /// - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub unsafe fn load_unsafe(&self) { - use crate::instructions::tables::lidt; - unsafe { - lidt(&self.pointer()); - } + *self = Self::new_with_validity(); } /// Creates the descriptor pointer for this table. This pointer can only be /// safely used if the table is never modified or destroyed while in use. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - fn pointer(&self) -> crate::structures::DescriptorTablePointer { + fn pointer(&self) -> crate::structures::DescriptorTablePointer + where + V: VirtAddrValidity, + { use core::mem::size_of; crate::structures::DescriptorTablePointer { - base: VirtAddr::new(self as *const _ as u64), + base: VirtAddrGeneric::::new_with_validity(self as *const _ as u64), limit: (size_of::() - 1) as u16, } } @@ -551,7 +529,7 @@ impl InterruptDescriptorTable { /// /// Panics if the entry is an exception. #[inline] - pub fn slice(&self, bounds: impl RangeBounds) -> &[Entry] { + pub fn slice(&self, bounds: impl RangeBounds) -> &[Entry, V>] { let (lower_idx, upper_idx) = self.condition_slice_bounds(bounds); &self.interrupts[(lower_idx - 32)..(upper_idx - 32)] } @@ -560,21 +538,52 @@ impl InterruptDescriptorTable { /// /// Panics if the entry is an exception. #[inline] - pub fn slice_mut(&mut self, bounds: impl RangeBounds) -> &mut [Entry] { + pub fn slice_mut(&mut self, bounds: impl RangeBounds) -> &mut [Entry, V>] { let (lower_idx, upper_idx) = self.condition_slice_bounds(bounds); &mut self.interrupts[(lower_idx - 32)..(upper_idx - 32)] } } -impl Default for InterruptDescriptorTable { +impl InterruptDescriptorTable { + /// Creates a new IDT with the default virtual-address validity. + /// + /// Handler addresses assigned later retain their creation-time validity guarantees. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn new() -> Self { + Self::new_with_validity() + } + + /// Loads the IDT in the CPU using the `lidt` command. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub fn load(&'static self) { + unsafe { self.load_unsafe() } + } + + /// Loads the IDT without imposing a static lifetime. + /// + /// # Safety + /// + /// The caller must keep the IDT alive and unmodified while it is loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub unsafe fn load_unsafe(&self) { + use crate::instructions::tables::lidt; + + unsafe { lidt(&self.pointer()) } + } +} + +impl Default for InterruptDescriptorTable { #[inline] fn default() -> Self { - Self::new() + Self::new_with_validity() } } -impl Index for InterruptDescriptorTable { - type Output = Entry; +impl Index for InterruptDescriptorTable { + type Output = Entry, V>; /// Returns the IDT entry with the specified index. /// @@ -605,7 +614,7 @@ impl Index for InterruptDescriptorTable { } } -impl IndexMut for InterruptDescriptorTable { +impl IndexMut for InterruptDescriptorTable { /// Returns a mutable reference to the IDT entry with the specified index. /// /// Panics if the entry is an exception that pushes an error code (use the struct fields for accessing these entries). @@ -637,8 +646,8 @@ impl IndexMut for InterruptDescriptorTable { macro_rules! impl_index_for_idt { ($ty:ty) => { - impl Index<$ty> for InterruptDescriptorTable { - type Output = [Entry]; + impl Index<$ty> for InterruptDescriptorTable { + type Output = [Entry, V>]; /// Returns the IDT entry with the specified index. /// @@ -650,7 +659,7 @@ macro_rules! impl_index_for_idt { } } - impl IndexMut<$ty> for InterruptDescriptorTable { + impl IndexMut<$ty> for InterruptDescriptorTable { /// Returns a mutable reference to the IDT entry with the specified index. /// /// Panics if the entry is an exception that pushes an error code (use the struct fields for accessing these entries). @@ -682,16 +691,16 @@ impl_index_for_idt!(RangeFull); /// The generic parameter is some [`HandlerFuncType`], depending on the interrupt vector. #[derive(Clone, Copy)] #[repr(C)] -pub struct Entry { +pub struct Entry { pointer_low: u16, options: EntryOptions, pointer_middle: u16, pointer_high: u32, reserved: u32, - phantom: PhantomData, + phantom: PhantomData<(F, V)>, } -impl fmt::Debug for Entry { +impl fmt::Debug for Entry { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Entry") .field("handler_addr", &format_args!("{:#x}", self.handler_addr())) @@ -700,7 +709,7 @@ impl fmt::Debug for Entry { } } -impl PartialEq for Entry { +impl PartialEq for Entry { fn eq(&self, other: &Self) -> bool { self.pointer_low == other.pointer_low && self.options == other.options @@ -717,14 +726,15 @@ impl PartialEq for Entry { any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type HandlerFunc = extern "x86-interrupt" fn(InterruptStackFrame); +pub type HandlerFunc = + extern "x86-interrupt" fn(InterruptStackFrame); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct HandlerFunc(()); +pub struct HandlerFunc(PhantomData); /// A handler function for an exception that pushes an error code. /// @@ -733,14 +743,15 @@ pub struct HandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type HandlerFuncWithErrCode = extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64); +pub type HandlerFuncWithErrCode = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct HandlerFuncWithErrCode(()); +pub struct HandlerFuncWithErrCode(PhantomData); /// A page fault handler function that pushes a page fault error code. /// @@ -749,15 +760,15 @@ pub struct HandlerFuncWithErrCode(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type PageFaultHandlerFunc = - extern "x86-interrupt" fn(InterruptStackFrame, error_code: PageFaultErrorCode); +pub type PageFaultHandlerFunc = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: PageFaultErrorCode); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct PageFaultHandlerFunc(()); +pub struct PageFaultHandlerFunc(PhantomData); /// A handler function that must not return, e.g. for a machine check exception. /// @@ -766,14 +777,15 @@ pub struct PageFaultHandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type DivergingHandlerFunc = extern "x86-interrupt" fn(InterruptStackFrame) -> !; +pub type DivergingHandlerFunc = + extern "x86-interrupt" fn(InterruptStackFrame) -> !; /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct DivergingHandlerFunc(()); +pub struct DivergingHandlerFunc(PhantomData); /// A handler function with an error code that must not return, e.g. for a double fault exception. /// @@ -782,20 +794,23 @@ pub struct DivergingHandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type DivergingHandlerFuncWithErrCode = - extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64) -> !; +pub type DivergingHandlerFuncWithErrCode = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64) -> !; /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct DivergingHandlerFuncWithErrCode(()); +pub struct DivergingHandlerFuncWithErrCode( + PhantomData, +); /// A general handler function for an interrupt or an exception with the interrupt/exceptions's index and an optional error code. -pub type GeneralHandlerFunc = fn(InterruptStackFrame, index: u8, error_code: Option); +pub type GeneralHandlerFunc = + fn(InterruptStackFrame, index: u8, error_code: Option); -impl Entry { +impl Entry { /// Creates a non-present IDT entry (but sets the must-be-one bits). #[inline] pub const fn missing() -> Self { @@ -808,7 +823,9 @@ impl Entry { phantom: PhantomData, } } +} +impl Entry { /// Sets the handler address for the IDT entry and sets the following defaults: /// - The code selector is the code segment currently active in the CPU /// - The present bit is set @@ -825,7 +842,7 @@ impl Entry { /// and the signature of such a function is correct for the entry type. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] #[inline] - pub unsafe fn set_handler_addr(&mut self, addr: VirtAddr) -> &mut EntryOptions { + pub unsafe fn set_handler_addr(&mut self, addr: VirtAddrGeneric) -> &mut EntryOptions { use crate::instructions::segmentation::{Segment, CS}; let addr = addr.as_u64(); @@ -842,18 +859,21 @@ impl Entry { /// Returns the virtual address of this IDT entry's handler function. #[inline] - pub fn handler_addr(&self) -> VirtAddr { + pub fn handler_addr(&self) -> VirtAddrGeneric { let addr = self.pointer_low as u64 | ((self.pointer_middle as u64) << 16) | ((self.pointer_high as u64) << 32); // addr is a valid VirtAddr, as the pointer members are either all zero, // or have been set by set_handler_addr (which takes a VirtAddr). - VirtAddr::new_truncate(addr) + unsafe { VirtAddrGeneric::::new_unsafe(addr) } } } #[cfg(all(feature = "instructions", target_arch = "x86_64"))] -impl Entry { +impl Entry +where + F: HandlerFuncType, +{ /// Sets the handler function for the IDT entry and sets the following defaults: /// - The code selector is the code segment currently active in the CPU /// - The present bit is set @@ -877,27 +897,30 @@ impl Entry { /// # Safety /// /// Implementors have to ensure that `to_virt_addr` returns a valid address. -pub unsafe trait HandlerFuncType { +pub unsafe trait HandlerFuncType { /// Get the virtual address of the handler function. - fn to_virt_addr(self) -> VirtAddr; + fn to_virt_addr(self) -> VirtAddrGeneric; } macro_rules! impl_handler_func_type { - ($f:ty) => { + ($f:ident) => { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] - unsafe impl HandlerFuncType for $f { + unsafe impl HandlerFuncType for $f + where + V: VirtAddrValidity, + { #[inline] - fn to_virt_addr(self) -> VirtAddr { + fn to_virt_addr(self) -> VirtAddrGeneric { // Casting a function pointer to u64 is fine, if the pointer // width doesn't exceed 64 bits. #[cfg_attr( any(target_pointer_width = "32", target_pointer_width = "64"), allow(clippy::fn_to_numeric_cast) )] - VirtAddr::new(self as u64) + VirtAddrGeneric::::new_with_validity(self as u64) } } }; @@ -1015,16 +1038,18 @@ impl EntryOptions { /// occurs, which can cause undefined behavior (see the [`as_mut`](InterruptStackFrame::as_mut) /// method for more information). #[repr(transparent)] -pub struct InterruptStackFrame(InterruptStackFrameValue); +pub struct InterruptStackFrame( + InterruptStackFrameValue, +); -impl InterruptStackFrame { +impl InterruptStackFrame { /// Creates a new interrupt stack frame with the given values. #[inline] pub fn new( - instruction_pointer: VirtAddr, + instruction_pointer: VirtAddrGeneric, code_segment: SegmentSelector, cpu_flags: RFlags, - stack_pointer: VirtAddr, + stack_pointer: VirtAddrGeneric, stack_segment: SegmentSelector, ) -> Self { Self(InterruptStackFrameValue::new( @@ -1051,13 +1076,13 @@ impl InterruptStackFrame { /// Also, it is not fully clear yet whether modifications of the interrupt stack frame are /// officially supported by LLVM's x86 interrupt calling convention. #[inline] - pub unsafe fn as_mut(&mut self) -> Volatile<&mut InterruptStackFrameValue> { + pub unsafe fn as_mut(&mut self) -> Volatile<&mut InterruptStackFrameValue> { Volatile::new(&mut self.0) } } -impl Deref for InterruptStackFrame { - type Target = InterruptStackFrameValue; +impl Deref for InterruptStackFrame { + type Target = InterruptStackFrameValue; #[inline] fn deref(&self) -> &Self::Target { @@ -1065,7 +1090,7 @@ impl Deref for InterruptStackFrame { } } -impl fmt::Debug for InterruptStackFrame { +impl fmt::Debug for InterruptStackFrame { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.0.fmt(f) @@ -1075,33 +1100,33 @@ impl fmt::Debug for InterruptStackFrame { /// Represents the interrupt stack frame pushed by the CPU on interrupt or exception entry. #[derive(Clone, Copy)] #[repr(C)] -pub struct InterruptStackFrameValue { +pub struct InterruptStackFrameValue { /// This value points to the instruction that should be executed when the interrupt /// handler returns. For most interrupts, this value points to the instruction immediately /// following the last executed instruction. However, for some exceptions (e.g., page faults), /// this value points to the faulting instruction, so that the instruction is restarted on /// return. See the documentation of the [`InterruptDescriptorTable`] fields for more details. - pub instruction_pointer: VirtAddr, + pub instruction_pointer: VirtAddrGeneric, /// The code segment selector at the time of the interrupt. pub code_segment: SegmentSelector, _reserved1: [u8; 6], /// The flags register before the interrupt handler was invoked. pub cpu_flags: RFlags, /// The stack pointer at the time of the interrupt. - pub stack_pointer: VirtAddr, + pub stack_pointer: VirtAddrGeneric, /// The stack segment descriptor at the time of the interrupt (often zero in 64-bit mode). pub stack_segment: SegmentSelector, _reserved2: [u8; 6], } -impl InterruptStackFrameValue { +impl InterruptStackFrameValue { /// Creates a new interrupt stack frame with the given values. #[inline] pub fn new( - instruction_pointer: VirtAddr, + instruction_pointer: VirtAddrGeneric, code_segment: SegmentSelector, cpu_flags: RFlags, - stack_pointer: VirtAddr, + stack_pointer: VirtAddrGeneric, stack_segment: SegmentSelector, ) -> Self { Self { @@ -1148,7 +1173,7 @@ impl InterruptStackFrameValue { } } -impl fmt::Debug for InterruptStackFrameValue { +impl fmt::Debug for InterruptStackFrameValue { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut s = f.debug_struct("InterruptStackFrame"); s.field("instruction_pointer", &self.instruction_pointer); @@ -1419,7 +1444,7 @@ impl TryFrom for ExceptionVector { ))] #[macro_export] /// Set a general handler in an [`InterruptDescriptorTable`]. -/// ``` +/// ```no_run /// #![feature(abi_x86_interrupt)] /// use x86_64::set_general_handler; /// use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame}; @@ -1666,9 +1691,88 @@ mod test { fn size_test() { use core::mem::size_of; assert_eq!(size_of::>(), 16); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::< + Entry>, crate::addr::FixedValidity<57>>, + >(), + 16 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::, crate::addr::RuntimeValidity>>( + ), + 16 + ); assert_eq!(size_of::(), 256 * 16); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::>>(), + 256 * 16 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::>(), + 256 * 16 + ); assert_eq!(size_of::(), 40); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::>>(), + 40 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::>(), + 40 + ); assert_eq!(size_of::(), 40); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::>>(), + 40 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::>(), + 40 + ); + } + + #[test] + fn explicit_policy_idt_and_frames_construct() { + #[cfg(feature = "virt_addr_57")] + let _: InterruptDescriptorTable> = + InterruptDescriptorTable::new_with_validity(); + + #[cfg(feature = "virt_addr_57")] + { + let address57 = crate::addr::VirtAddr57::new(0x0000_8000_0000_0000); + let frame57 = InterruptStackFrame::new( + address57, + SegmentSelector(0), + RFlags::empty(), + address57, + SegmentSelector(0), + ); + assert_eq!(frame57.instruction_pointer, address57); + } + + #[cfg(feature = "virt_addr_rt")] + { + let _: InterruptDescriptorTable = + InterruptDescriptorTable::new_with_validity(); + + let address_rt = unsafe { crate::addr::VirtAddrRT::new_unsafe(0x1234) }; + let frame_rt = InterruptStackFrame::new( + address_rt, + SegmentSelector(0), + RFlags::empty(), + address_rt, + SegmentSelector(0), + ); + assert_eq!(frame_rt.stack_pointer, address_rt); + } } #[cfg(all( @@ -1680,6 +1784,7 @@ mod test { // https://github.com/rust-osdev/x86_64/pull/285#issuecomment-962642984 #[cfg(not(windows))] #[test] + #[ignore = "runtime-valid handler construction requires ring 0"] fn default_handlers() { fn general_handler( _stack_frame: InterruptStackFrame, @@ -1745,15 +1850,16 @@ mod test { #[test] fn isr_frame_manipulation() { - let mut frame = InterruptStackFrame(InterruptStackFrameValue { - instruction_pointer: VirtAddr::new(0x1000), - code_segment: SegmentSelector(0), - cpu_flags: RFlags::empty(), - stack_pointer: VirtAddr::new(0x2000), - stack_segment: SegmentSelector(0), - _reserved1: Default::default(), - _reserved2: Default::default(), - }); + let mut frame: InterruptStackFrame> = + InterruptStackFrame(InterruptStackFrameValue { + instruction_pointer: crate::addr::VirtAddr48::new(0x1000), + code_segment: SegmentSelector(0), + cpu_flags: RFlags::empty(), + stack_pointer: crate::addr::VirtAddr48::new(0x2000), + stack_segment: SegmentSelector(0), + _reserved1: Default::default(), + _reserved2: Default::default(), + }); unsafe { frame.as_mut().update(|f| f.instruction_pointer += 2u64); diff --git a/src/structures/mod.rs b/src/structures/mod.rs index 084bbafa..a2e42baa 100644 --- a/src/structures/mod.rs +++ b/src/structures/mod.rs @@ -1,6 +1,6 @@ //! Representations of various x86 specific structures and descriptor tables. -use crate::VirtAddr; +use crate::addr::{DefaultVirtAddrValidity, VirtAddrGeneric, VirtAddrValidity}; pub mod gdt; @@ -14,18 +14,40 @@ pub mod tss; /// A struct describing a pointer to a descriptor table (GDT / IDT). /// This is in a format suitable for giving to 'lgdt' or 'lidt'. -#[derive(Debug, Clone, Copy)] #[repr(C, packed(2))] -pub struct DescriptorTablePointer { +pub struct DescriptorTablePointer { /// Size of the DT in bytes - 1. pub limit: u16, /// Pointer to the memory region containing the DT. - pub base: VirtAddr, + pub base: VirtAddrGeneric, +} + +// These traits are implemented manually because Rust 1.59 has limited derive support for generic +// packed structs. They can use derive once the MSRV is raised to Rust 1.69. +impl Copy for DescriptorTablePointer {} + +impl Clone for DescriptorTablePointer { + fn clone(&self) -> Self { + *self + } +} + +impl core::fmt::Debug for DescriptorTablePointer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let limit = self.limit; + let base = self.base; + + f.debug_struct("DescriptorTablePointer") + .field("limit", &limit) + .field("base", &base) + .finish() + } } #[cfg(test)] mod tests { use super::*; + use crate::VirtAddr; use std::mem::size_of; #[test] @@ -33,10 +55,23 @@ mod tests { // Per the SDM, a descriptor pointer has to be 2+8=10 bytes assert_eq!(size_of::(), 10); // Make sure that we can reference a pointer's limit - let p = DescriptorTablePointer { + let p: DescriptorTablePointer = DescriptorTablePointer { limit: 5, base: VirtAddr::zero(), }; let _: &u16 = &p.limit; + + let _: DescriptorTablePointer = p; + + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::>>(), + 10 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::>(), + 10 + ); } } diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 5f673c55..1021114b 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -4,13 +4,13 @@ use crate::structures::paging::{ page_table::{FrameError, PageTable, PageTableEntry, PageTableLevel}, }; -/// A Mapper implementation that relies on a PhysAddr to VirtAddr conversion function. +/// A Mapper implementation that relies on a PhysAddr to VirtAddr48 conversion function. /// /// This type requires that the all physical page table frames are mapped to some virtual /// address. Normally, this is done by mapping the complete physical address space into /// the virtual address space at some offset. Other mappings between physical and virtual /// memory are possible too, as long as they can be calculated as an `PhysAddr` to -/// `VirtAddr` closure. +/// `VirtAddr48` closure. #[derive(Debug)] pub struct MappedPageTable<'a, P: PageTableFrameMapping> { page_table_walker: PageTableWalker

, @@ -55,7 +55,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -81,7 +81,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -107,7 +107,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -125,7 +125,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -142,7 +142,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) @@ -150,13 +150,16 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; @@ -175,7 +178,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -206,7 +209,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -235,7 +238,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -257,7 +260,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -274,7 +277,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -294,13 +297,16 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?; @@ -320,7 +326,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -356,7 +362,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -382,7 +388,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -407,7 +413,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -424,7 +430,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -444,7 +450,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -465,7 +471,10 @@ impl Mapper for MappedPageTable<'_, P> { Ok(MapperFlushAll::new()) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?; @@ -484,7 +493,7 @@ impl Mapper for MappedPageTable<'_, P> { impl Translate for MappedPageTable<'_, P> { #[allow(clippy::inconsistent_digit_grouping)] - fn translate(&self, addr: VirtAddr) -> TranslateResult { + fn translate(&self, addr: VirtAddr48) -> TranslateResult { let p4 = &self.level_4_table; let p3 = match self.page_table_walker.next_table(&p4[addr.p4_index()]) { Ok(page_table) => page_table, @@ -555,8 +564,8 @@ impl CleanUp for MappedPageTable<'_, P> { unsafe { self.clean_up_addr_range( PageRangeInclusive { - start: Page::from_start_address(VirtAddr::new(0)).unwrap(), - end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(), + start: Page::from_start_address(VirtAddr48::new(0)).unwrap(), + end: Page::from_start_address(VirtAddr48::new(0xffff_ffff_ffff_f000)).unwrap(), }, frame_deallocator, ) @@ -565,7 +574,7 @@ impl CleanUp for MappedPageTable<'_, P> { unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, @@ -574,7 +583,7 @@ impl CleanUp for MappedPageTable<'_, P> { page_table: &mut PageTable, page_table_walker: &PageTableWalker

, level: PageTableLevel, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut impl FrameDeallocator, ) -> bool { if range.is_empty() { @@ -598,15 +607,21 @@ impl CleanUp for MappedPageTable<'_, P> { .skip(usize::from(start)) { if let Ok(page_table) = page_table_walker.next_table_mut(entry) { - let start = VirtAddr::forward_checked_impl( + let start = VirtAddr48::forward_checked_impl( table_addr, (offset_per_entry as usize) * i, ) .unwrap(); let end = start + (offset_per_entry - 1); - let start = Page::::containing_address(start); + let start = + Page::>::containing_address_with_validity( + start, + ); let start = start.max(range.start); - let end = Page::::containing_address(end); + let end = + Page::>::containing_address_with_validity( + end, + ); let end = end.min(range.end); unsafe { if clean_up( diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index d0f21716..0b83c9fe 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -6,13 +6,14 @@ pub use self::offset_page_table::OffsetPageTable; #[cfg(all(feature = "instructions", target_arch = "x86_64"))] pub use self::recursive_page_table::{InvalidPageTable, RecursivePageTable}; +use crate::addr::{FixedValidity, VirtAddr48}; use crate::structures::paging::{ frame_alloc::{FrameAllocator, FrameDeallocator}, page::PageRangeInclusive, page_table::PageTableFlags, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, }; -use crate::{PhysAddr, VirtAddr}; +use crate::PhysAddr; mod mapped_page_table; mod offset_page_table; @@ -33,7 +34,7 @@ pub trait Translate { /// frame is returned. Otherwise an error value is returned. /// /// This function works with huge pages of all sizes. - fn translate(&self, addr: VirtAddr) -> TranslateResult; + fn translate(&self, addr: VirtAddr48) -> TranslateResult; /// Translates the given virtual address to the physical address that it maps to. /// @@ -42,7 +43,7 @@ pub trait Translate { /// This is a convenience method. For more information about a mapping see the /// [`translate`](Translate::translate) method. #[inline] - fn translate_addr(&self, addr: VirtAddr) -> Option { + fn translate_addr(&self, addr: VirtAddr48) -> Option { match self.translate(addr) { TranslateResult::NotMapped | TranslateResult::InvalidFrameAddress(_) => None, TranslateResult::Mapped { frame, offset, .. } => Some(frame.start_address() + offset), @@ -159,9 +160,10 @@ pub trait Mapper { /// # Mapper, Page, PhysFrame, FrameAllocator, /// # Size4KiB, OffsetPageTable, page_table::PageTableFlags /// # }; + /// # use x86_64::addr::FixedValidity; /// # #[cfg(all(feature = "instructions", target_arch = "x86_64"))] /// # unsafe fn test(mapper: &mut OffsetPageTable, frame_allocator: &mut impl FrameAllocator, - /// # page: Page, frame: PhysFrame) { + /// # page: Page>, frame: PhysFrame) { /// mapper /// .map_to( /// page, @@ -178,7 +180,7 @@ pub trait Mapper { #[inline] unsafe fn map_to( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, frame_allocator: &mut A, @@ -248,9 +250,10 @@ pub trait Mapper { /// # Mapper, PhysFrame, Page, FrameAllocator, /// # Size4KiB, OffsetPageTable, page_table::PageTableFlags /// # }; + /// # use x86_64::addr::FixedValidity; /// # #[cfg(all(feature = "instructions", target_arch = "x86_64"))] /// # unsafe fn test(mapper: &mut OffsetPageTable, frame_allocator: &mut impl FrameAllocator, - /// # page: Page, frame: PhysFrame) { + /// # page: Page>, frame: PhysFrame) { /// mapper /// .map_to_with_table_flags( /// page, @@ -269,7 +272,7 @@ pub trait Mapper { /// ``` unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -282,7 +285,10 @@ pub trait Mapper { /// Removes a mapping from the page table and returns the frame that used to be mapped. /// /// Note that no page tables or pages are deallocated. - fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + fn unmap( + &mut self, + page: Page>, + ) -> Result<(PhysFrame, MapperFlush), UnmapError>; /// Updates the flags of an existing mapping. /// @@ -297,7 +303,7 @@ pub trait Mapper { /// spaces. unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError>; @@ -312,7 +318,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -327,7 +333,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -342,7 +348,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -350,7 +356,10 @@ pub trait Mapper { /// /// This function assumes that the page is mapped to a frame of size `S` and returns an /// error otherwise. - fn translate_page(&self, page: Page) -> Result, TranslateError>; + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError>; /// Maps the given frame to the virtual page with the same address. /// @@ -371,7 +380,8 @@ pub trait Mapper { S: PageSize, Self: Mapper, { - let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); + let page = + Page::containing_address_with_validity(VirtAddr48::new(frame.start_address().as_u64())); unsafe { self.map_to(page, frame, flags, frame_allocator) } } } @@ -387,7 +397,7 @@ pub trait Mapper { not(all(feature = "instructions", target_arch = "x86_64")), allow(dead_code) )] // FIXME -pub struct MapperFlush(Page); +pub struct MapperFlush(Page>); impl MapperFlush { /// Create a new flush promise @@ -395,7 +405,7 @@ impl MapperFlush { /// Note that this method is intended for implementing the [`Mapper`] trait and no other uses /// are expected. #[inline] - pub fn new(page: Page) -> Self { + pub fn new(page: Page>) -> Self { MapperFlush(page) } @@ -403,7 +413,11 @@ impl MapperFlush { #[cfg(all(feature = "instructions", target_arch = "x86_64"))] #[inline] pub fn flush(self) { + #[cfg(not(feature = "default_virt_addr_57"))] crate::instructions::tlb::flush(self.0.start_address()); + + #[cfg(feature = "default_virt_addr_57")] + crate::instructions::tlb::flush(self.0.start_address().into()); } /// Don't flush the TLB and silence the “must be used” warning. @@ -412,7 +426,7 @@ impl MapperFlush { /// Returns the page to be flushed. #[inline] - pub fn page(&self) -> Page { + pub fn page(&self) -> Page> { self.0 } } @@ -513,14 +527,14 @@ pub trait CleanUp { /// Remove all empty P1-P3 tables in a certain range /// ``` /// # use core::ops::RangeInclusive; - /// # use x86_64::{VirtAddr, structures::paging::{ + /// # use x86_64::{addr::VirtAddr48, structures::paging::{ /// # FrameDeallocator, Size4KiB, mapper::CleanUp, page::Page, /// # }}; /// # unsafe fn test(page_table: &mut impl CleanUp, frame_deallocator: &mut impl FrameDeallocator) { /// // clean up all page tables in the lower half of the address space /// let lower_half = Page::range_inclusive( - /// Page::containing_address(VirtAddr::new(0)), - /// Page::containing_address(VirtAddr::new(0x0000_7fff_ffff_ffff)), + /// Page::containing_address_with_validity(VirtAddr48::new(0)), + /// Page::containing_address_with_validity(VirtAddr48::new(0x0000_7fff_ffff_ffff)), /// ); /// page_table.clean_up_addr_range(lower_half, frame_deallocator); /// # } @@ -533,7 +547,7 @@ pub trait CleanUp { /// (e.g. no reference counted page tables or reusing the same page tables for different virtual addresses ranges in the same page table). unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator; diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index 546a8299..56542b7d 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -26,7 +26,7 @@ impl<'a> OffsetPageTable<'a> { /// of a valid page table hierarchy. Otherwise this function might break memory safety, e.g. /// by writing to an illegal memory location. #[inline] - pub unsafe fn new(level_4_table: &'a mut PageTable, phys_offset: VirtAddr) -> Self { + pub unsafe fn new(level_4_table: &'a mut PageTable, phys_offset: VirtAddr48) -> Self { let phys_offset = PhysOffset { offset: phys_offset, }; @@ -46,14 +46,14 @@ impl<'a> OffsetPageTable<'a> { } /// Returns the offset used for converting virtual to physical addresses. - pub fn phys_offset(&self) -> VirtAddr { + pub fn phys_offset(&self) -> VirtAddr48 { self.inner.page_table_frame_mapping().offset } } #[derive(Debug)] struct PhysOffset { - offset: VirtAddr, + offset: VirtAddr48, } unsafe impl PageTableFrameMapping for PhysOffset { @@ -69,7 +69,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -87,7 +87,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -95,7 +95,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -104,7 +104,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -113,7 +113,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -122,14 +122,17 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } @@ -138,7 +141,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -156,7 +159,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -164,7 +167,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -173,7 +176,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -182,7 +185,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -191,14 +194,17 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } @@ -207,7 +213,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -225,7 +231,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -233,7 +239,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -242,7 +248,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -251,7 +257,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -260,21 +266,24 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } impl Translate for OffsetPageTable<'_> { #[inline] - fn translate(&self, addr: VirtAddr) -> TranslateResult { + fn translate(&self, addr: VirtAddr48) -> TranslateResult { self.inner.translate(addr) } } @@ -291,7 +300,7 @@ impl CleanUp for OffsetPageTable<'_> { #[inline] unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index bd3c5981..7e3087ab 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -55,7 +55,8 @@ impl<'a> RecursivePageTable<'a> { /// and [in the `unsafe-code-guidelines ` repo](https://github.com/rust-lang/unsafe-code-guidelines/issues/420). #[inline] pub fn new(table: &'a mut PageTable) -> Result { - let page = Page::containing_address(VirtAddr::new(table as *const _ as u64)); + let page = + Page::containing_address_with_validity(VirtAddr48::new(table as *const _ as u64)); let recursive_index = page.p4_index(); if page.p3_index() != recursive_index @@ -116,7 +117,7 @@ impl<'a> RecursivePageTable<'a> { /// in the passed entry. unsafe fn create_next_table<'b, A, S: PageSize>( entry: &'b mut PageTableEntry, - next_table_page: Page, + next_table_page: Page>, insert_flags: PageTableFlags, allocator: &mut A, ) -> Result<&'b mut PageTable, MapToError> @@ -128,7 +129,7 @@ impl<'a> RecursivePageTable<'a> { /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe. fn inner<'b, A, S: PageSize>( entry: &'b mut PageTableEntry, - next_table_page: Page, + next_table_page: Page>, insert_flags: PageTableFlags, allocator: &mut A, ) -> Result<&'b mut PageTable, MapToError> @@ -172,7 +173,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -204,7 +205,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -234,7 +235,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { use crate::structures::paging::PageTableFlags as Flags; @@ -256,7 +257,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -273,7 +274,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) @@ -281,13 +282,16 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -310,7 +314,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -352,7 +356,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -388,7 +392,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { use crate::structures::paging::PageTableFlags as Flags; @@ -417,7 +421,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -434,7 +438,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -457,13 +461,16 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -493,7 +500,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -544,7 +551,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -581,7 +588,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.p4; @@ -615,7 +622,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -632,7 +639,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -655,7 +662,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -682,7 +689,10 @@ impl Mapper for RecursivePageTable<'_> { Ok(MapperFlushAll::new()) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -717,8 +727,8 @@ impl Mapper for RecursivePageTable<'_> { impl Translate for RecursivePageTable<'_> { #[allow(clippy::inconsistent_digit_grouping)] - fn translate(&self, addr: VirtAddr) -> TranslateResult { - let page = Page::containing_address(addr); + fn translate(&self, addr: VirtAddr48) -> TranslateResult { + let page = Page::containing_address_with_validity(addr); let p4 = &self.p4; let p4_entry = &p4[addr.p4_index()]; @@ -797,8 +807,8 @@ impl CleanUp for RecursivePageTable<'_> { unsafe { self.clean_up_addr_range( PageRangeInclusive { - start: Page::from_start_address(VirtAddr::new(0)).unwrap(), - end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(), + start: Page::from_start_address(VirtAddr48::new(0)).unwrap(), + end: Page::from_start_address(VirtAddr48::new(0xffff_ffff_ffff_f000)).unwrap(), }, frame_deallocator, ) @@ -807,7 +817,7 @@ impl CleanUp for RecursivePageTable<'_> { unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, @@ -816,7 +826,7 @@ impl CleanUp for RecursivePageTable<'_> { recursive_index: PageTableIndex, page_table: &mut PageTable, level: PageTableLevel, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut impl FrameDeallocator, ) -> bool { if range.is_empty() { @@ -843,15 +853,21 @@ impl CleanUp for RecursivePageTable<'_> { }) { if let Ok(frame) = entry.frame() { - let start = VirtAddr::forward_checked_impl( + let start = VirtAddr48::forward_checked_impl( table_addr, (offset_per_entry as usize) * i, ) .unwrap(); let end = start + (offset_per_entry - 1); - let start = Page::::containing_address(start); + let start = + Page::>::containing_address_with_validity( + start, + ); let start = start.max(range.start); - let end = Page::::containing_address(end); + let end = + Page::>::containing_address_with_validity( + end, + ); let end = end.min(range.end); let page_table = [p1_ptr, p2_ptr, p3_ptr][level as usize - 2](start, recursive_index); @@ -913,12 +929,18 @@ impl fmt::Display for InvalidPageTable { } #[inline] -fn p3_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p3_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p3_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p3_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p3_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, recursive_index, @@ -928,12 +950,18 @@ fn p3_page(page: Page, recursive_index: PageTableIndex) -> Page } #[inline] -fn p2_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p2_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p2_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p2_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p2_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, recursive_index, @@ -943,12 +971,18 @@ fn p2_page(page: Page, recursive_index: PageTableIndex) } #[inline] -fn p1_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p1_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p1_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p1_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p1_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, page.p4_index(), diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index b4e7a4e6..eb983661 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -1,9 +1,10 @@ //! Abstractions for default-sized and huge virtual memory pages. +use crate::addr::ArithmeticValidity; +use crate::addr::{DefaultVirtAddrValidity, FixedValidity, VirtAddrGeneric, VirtAddrValidity}; use crate::sealed::Sealed; use crate::structures::paging::page_table::PageTableLevel; use crate::structures::paging::PageTableIndex; -use crate::VirtAddr; use core::convert::TryFrom; use core::fmt; #[cfg(feature = "step_trait")] @@ -65,12 +66,12 @@ impl Sealed for super::Size1GiB {} /// A virtual memory page. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(C)] -pub struct Page { - start_address: VirtAddr, +pub struct Page { + start_address: VirtAddrGeneric, size: PhantomData, } -impl Page { +impl Page { /// The page size in bytes. pub const SIZE: u64 = S::SIZE; @@ -79,11 +80,14 @@ impl Page { /// Returns an error if the address is not correctly aligned (i.e. is not a valid page start). #[inline] #[rustversion::attr(since(1.61), const)] - pub fn from_start_address(address: VirtAddr) -> Result { + pub fn from_start_address(address: VirtAddrGeneric) -> Result { if !address.is_aligned_u64(S::SIZE) { return Err(AddressNotAligned); } - Ok(Page::containing_address(address)) + Ok(Page { + start_address: address, + size: PhantomData, + }) } /// Returns the page that starts at the given virtual address. @@ -93,27 +97,17 @@ impl Page { /// The address must be correctly aligned. #[inline] #[rustversion::attr(since(1.61), const)] - pub unsafe fn from_start_address_unchecked(start_address: VirtAddr) -> Self { + pub unsafe fn from_start_address_unchecked(start_address: VirtAddrGeneric) -> Self { Page { start_address, size: PhantomData, } } - /// Returns the page that contains the given virtual address. - #[inline] - #[rustversion::attr(since(1.61), const)] - pub fn containing_address(address: VirtAddr) -> Self { - Page { - start_address: address.align_down_u64(S::SIZE), - size: PhantomData, - } - } - /// Returns the start address of the page. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn start_address(self) -> VirtAddr { + pub fn start_address(self) -> VirtAddrGeneric { self.start_address } @@ -148,20 +142,64 @@ impl Page { /// Returns a range of pages, exclusive `end`. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn range(start: Self, end: Self) -> PageRange { + pub fn range(start: Self, end: Self) -> PageRange { PageRange { start, end } } /// Returns a range of pages, inclusive `end`. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { + pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { PageRangeInclusive { start, end } } +} + +impl Page { + /// Returns the page that contains the given default virtual address. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn containing_address(address: VirtAddrGeneric) -> Self { + Self::containing_address_with_validity(address) + } +} + +impl Page> +where + FixedValidity: VirtAddrValidity, +{ + /// Returns the page that contains the given fixed-width virtual address. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn containing_address_with_validity(address: VirtAddrGeneric>) -> Self { + Page { + start_address: address.align_down_u64(S::SIZE), + size: PhantomData, + } + } +} + +#[cfg(all( + feature = "instructions", + feature = "virt_addr_rt", + target_arch = "x86_64" +))] +impl Page { + /// Returns the page that contains the given runtime-valid virtual address. + #[inline] + pub fn containing_address_current( + address: VirtAddrGeneric, + ) -> Self { + Page { + start_address: address.align_down_u64(S::SIZE), + size: PhantomData, + } + } +} +impl Page { // FIXME: Move this into the `Step` impl, once `Step` is stabilized. pub(crate) fn steps_between_u64(start: &Self, end: &Self) -> Option { - VirtAddr::steps_between_u64(&start.start_address(), &end.start_address()) + VirtAddrGeneric::::steps_between_u64(&start.start_address(), &end.start_address()) .map(|steps| steps / S::SIZE) } @@ -180,7 +218,7 @@ impl Page { #[cfg(any(feature = "instructions", feature = "step_trait"))] pub(crate) fn forward_checked_impl(start: Self, count: usize) -> Option { let count = u64::try_from(count).ok()?.checked_mul(S::SIZE)?; - let start_address = VirtAddr::forward_checked_u64(start.start_address, count)?; + let start_address = VirtAddrGeneric::::forward_checked_u64(start.start_address, count)?; Some(Self { start_address, size: PhantomData, @@ -188,7 +226,7 @@ impl Page { } } -impl Page { +impl Page { /// Returns the level 2 page table index of this page. #[inline] #[rustversion::attr(since(1.61), const)] @@ -197,7 +235,7 @@ impl Page { } } -impl Page { +impl Page> { /// Returns the 1GiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -208,11 +246,11 @@ impl Page { let mut addr = 0; addr |= p4_index.into_u64() << 39; addr |= p3_index.into_u64() << 30; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_with_validity(crate::addr::VirtAddr48::new_truncate(addr)) } } -impl Page { +impl Page> { /// Returns the 2MiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -225,11 +263,11 @@ impl Page { addr |= p4_index.into_u64() << 39; addr |= p3_index.into_u64() << 30; addr |= p2_index.into_u64() << 21; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_with_validity(crate::addr::VirtAddr48::new_truncate(addr)) } } -impl Page { +impl Page> { /// Returns the 4KiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -244,17 +282,18 @@ impl Page { addr |= p3_index.into_u64() << 30; addr |= p2_index.into_u64() << 21; addr |= p1_index.into_u64() << 12; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_with_validity(crate::addr::VirtAddr48::new_truncate(addr)) } /// Returns the level 1 page table index of this page. #[inline] - pub const fn p1_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p1_index(self) -> PageTableIndex { self.start_address.p1_index() } } -impl fmt::Debug for Page { +impl fmt::Debug for Page { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_fmt(format_args!( "Page[{}]({:#x})", @@ -264,37 +303,37 @@ impl fmt::Debug for Page { } } -impl Add for Page { +impl Add for Page { type Output = Self; #[inline] fn add(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() + rhs * S::SIZE) + unsafe { Page::from_start_address_unchecked(self.start_address() + rhs * S::SIZE) } } } -impl AddAssign for Page { +impl AddAssign for Page { #[inline] fn add_assign(&mut self, rhs: u64) { *self = *self + rhs; } } -impl Sub for Page { +impl Sub for Page { type Output = Self; #[inline] fn sub(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() - rhs * S::SIZE) + unsafe { Page::from_start_address_unchecked(self.start_address() - rhs * S::SIZE) } } } -impl SubAssign for Page { +impl SubAssign for Page { #[inline] fn sub_assign(&mut self, rhs: u64) { *self = *self - rhs; } } -impl Sub for Page { +impl Sub for Page { type Output = u64; #[inline] fn sub(self, rhs: Self) -> Self::Output { @@ -303,7 +342,7 @@ impl Sub for Page { } #[cfg(feature = "step_trait")] -impl Step for Page { +impl Step for Page { fn steps_between(start: &Self, end: &Self) -> (usize, Option) { Self::steps_between_impl(start, end) } @@ -316,7 +355,7 @@ impl Step for Page { use core::convert::TryFrom; let count = u64::try_from(count).ok()?.checked_mul(S::SIZE)?; - let start_address = VirtAddr::backward_checked_u64(start.start_address, count)?; + let start_address = VirtAddrGeneric::::backward_checked_u64(start.start_address, count)?; Some(Self { start_address, size: PhantomData, @@ -349,14 +388,14 @@ impl Step for Page { /// A range of pages with exclusive upper bound. #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[repr(C)] -pub struct PageRange { +pub struct PageRange { /// The start of the range, inclusive. - pub start: Page, + pub start: Page, /// The end of the range, exclusive. - pub end: Page, + pub end: Page, } -impl PageRange { +impl PageRange { /// Returns whether this range contains no pages. #[inline] pub fn is_empty(&self) -> bool { @@ -380,8 +419,8 @@ impl PageRange { } } -impl Iterator for PageRange { - type Item = Page; +impl Iterator for PageRange { + type Item = Page; #[inline] fn next(&mut self) -> Option { @@ -413,7 +452,9 @@ impl Iterator for PageRange { } // Figure out how many steps there are until the address range gap. - let second_half_start = Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked(VirtAddrGeneric::::upper_half_start()) + }; let steps_until_gap = Page::steps_between_u64(&self.start, &second_half_start) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -436,7 +477,7 @@ impl Iterator for PageRange { } } -impl DoubleEndedIterator for PageRange { +impl DoubleEndedIterator for PageRange { #[inline] fn next_back(&mut self) -> Option { if self.start < self.end { @@ -466,7 +507,11 @@ impl DoubleEndedIterator for PageRange { } // Figure out how many steps there are until the address range gap. - let first_half_end = Page::::containing_address(VirtAddr::new(0x7fff_ffff_f000)); + let first_half_end = unsafe { + Page::::from_start_address_unchecked(VirtAddrGeneric::::new_unsafe( + VirtAddrGeneric::::lower_half_end().as_u64() & !(S::SIZE - 1), + )) + }; let steps_until_gap = Page::steps_between_u64(&first_half_end, &self.end) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -482,18 +527,18 @@ impl DoubleEndedIterator for PageRange { } } -impl PageRange { +impl PageRange { /// Converts the range of 2MiB pages to a range of 4KiB pages. #[inline] - pub fn as_4kib_page_range(self) -> PageRange { + pub fn as_4kib_page_range(self) -> PageRange { PageRange { - start: Page::containing_address(self.start.start_address()), - end: Page::containing_address(self.end.start_address()), + start: unsafe { Page::from_start_address_unchecked(self.start.start_address()) }, + end: unsafe { Page::from_start_address_unchecked(self.end.start_address()) }, } } } -impl fmt::Debug for PageRange { +impl fmt::Debug for PageRange { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRange") .field("start", &self.start) @@ -505,14 +550,15 @@ impl fmt::Debug for PageRange { /// A range of pages with inclusive upper bound. #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[repr(C)] -pub struct PageRangeInclusive { +pub struct PageRangeInclusive +{ /// The start of the range, inclusive. - pub start: Page, + pub start: Page, /// The end of the range, inclusive. - pub end: Page, + pub end: Page, } -impl PageRangeInclusive { +impl PageRangeInclusive { /// Returns whether this range contains no pages. #[inline] pub fn is_empty(&self) -> bool { @@ -536,8 +582,8 @@ impl PageRangeInclusive { } } -impl Iterator for PageRangeInclusive { - type Item = Page; +impl Iterator for PageRangeInclusive { + type Item = Page; #[inline] fn next(&mut self) -> Option { @@ -547,7 +593,7 @@ impl Iterator for PageRangeInclusive { // If the end of the inclusive range is the maximum page possible for size S, // incrementing start until it is greater than the end will cause an integer overflow. // So instead, in that case we decrement end rather than incrementing start. - let max_page_addr = VirtAddr::new(u64::MAX) - (S::SIZE - 1); + let max_page_addr = VirtAddrGeneric::::max_value() - (S::SIZE - 1); if self.start.start_address() < max_page_addr { self.start += 1; } else { @@ -578,7 +624,9 @@ impl Iterator for PageRangeInclusive { } // Figure out how many steps there are until the address range gap. - let second_half_start = Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked(VirtAddrGeneric::::upper_half_start()) + }; let steps_until_gap = Page::steps_between_u64(&self.start, &second_half_start) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -601,7 +649,7 @@ impl Iterator for PageRangeInclusive { } } -impl DoubleEndedIterator for PageRangeInclusive { +impl DoubleEndedIterator for PageRangeInclusive { #[inline] fn next_back(&mut self) -> Option { if self.start <= self.end { @@ -640,7 +688,11 @@ impl DoubleEndedIterator for PageRangeInclusive { } // Figure out how many steps there are until the address range gap. - let first_half_end = Page::::containing_address(VirtAddr::new(0x7fff_ffff_f000)); + let first_half_end = unsafe { + Page::::from_start_address_unchecked(VirtAddrGeneric::::new_unsafe( + VirtAddrGeneric::::lower_half_end().as_u64() & !(S::SIZE - 1), + )) + }; let steps_until_gap = Page::steps_between_u64(&first_half_end, &self.end) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -656,7 +708,7 @@ impl DoubleEndedIterator for PageRangeInclusive { } } -impl fmt::Debug for PageRangeInclusive { +impl fmt::Debug for PageRangeInclusive { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRangeInclusive") .field("start", &self.start) @@ -666,9 +718,12 @@ impl fmt::Debug for PageRangeInclusive { } #[cfg(kani)] -impl kani::Arbitrary for Page { +impl kani::Arbitrary for Page> +where + FixedValidity: VirtAddrValidity, +{ fn any() -> Self { - Self::containing_address(kani::any()) + Self::containing_address_with_validity(kani::any()) } } @@ -686,6 +741,58 @@ impl fmt::Display for AddressNotAligned { mod tests { use super::*; + /// A fixed-VA48 page used by Ring 3 arithmetic tests. + type Page = super::Page>; + + /// A fixed-VA48 address used by Ring 3 arithmetic tests. + type VirtAddr = crate::addr::VirtAddr48; + + #[test] + fn page_validity_defaults_and_explicit_policy() { + let _: super::Page = + super::Page::containing_address(crate::VirtAddr::new(0x1000)); + let _: super::Page = unsafe { + super::Page::from_start_address_unchecked(crate::VirtAddr::new_unsafe(0x20_0000)) + }; + + #[cfg(feature = "virt_addr_57")] + { + let page57: super::Page> = + super::Page::containing_address_with_validity(crate::addr::VirtAddr57::new( + 0x00ff_0000_0000_1000, + )); + assert_eq!(page57.start_address().as_u64(), 0x00ff_0000_0000_1000); + } + } + + #[test] + #[cfg(all(feature = "step_trait", feature = "virt_addr_57"))] + fn page57_step_uses_la57_gap() { + let low_end = super::Page::>::from_start_address( + crate::addr::VirtAddr57::new(0x00ff_ffff_ffff_f000), + ) + .unwrap(); + let upper_start = + super::Page::>::from_start_address( + crate::addr::VirtAddr57::new(0xff00_0000_0000_0000), + ) + .unwrap(); + + assert_eq!(Step::forward(low_end, 1), upper_start); + assert_eq!(Step::backward(upper_start, 1), low_end); + } + + #[test] + fn p4_constructors_remain_va48() { + let page = super::Page::from_page_table_indices( + PageTableIndex::new(1), + PageTableIndex::new(2), + PageTableIndex::new(3), + PageTableIndex::new(4), + ); + let _: super::Page> = page; + } + fn test_is_hash() {} #[test] @@ -701,14 +808,16 @@ mod tests { let number = 1000; let start_addr = VirtAddr::new(0xdead_beaf); - let start: Page = Page::containing_address(start_addr); + let start: Page = Page::containing_address_with_validity(start_addr); let end = start + number; let mut range = Page::range(start, end); for i in 0..number { assert_eq!( range.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_with_validity( + start_addr + page_size * i, + )) ); } assert_eq!(range.next(), None); @@ -717,7 +826,9 @@ mod tests { for i in 0..=number { assert_eq!( range_inclusive.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_with_validity( + start_addr + page_size * i, + )) ); } assert_eq!(range_inclusive.next(), None); @@ -729,14 +840,16 @@ mod tests { let number = 1000; let start_addr = VirtAddr::new(u64::MAX).align_down(page_size) - number * page_size; - let start: Page = Page::containing_address(start_addr); + let start: Page = Page::containing_address_with_validity(start_addr); let end = start + number; let mut range_inclusive = Page::range_inclusive(start, end); for i in 0..=number { assert_eq!( range_inclusive.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_with_validity( + start_addr + page_size * i, + )) ); } assert_eq!(range_inclusive.next(), None); @@ -821,7 +934,7 @@ mod tests { #[test] pub fn test_page_range_len() { let start_addr = VirtAddr::new(0xdead_beaf); - let start = Page::::containing_address(start_addr); + let start = Page::::containing_address_with_validity(start_addr); let end = start + 50; let range = PageRange { start, end }; diff --git a/src/structures/tss.rs b/src/structures/tss.rs index f0174f3f..3825ce28 100644 --- a/src/structures/tss.rs +++ b/src/structures/tss.rs @@ -1,25 +1,25 @@ //! Provides a type for the task state segment structure. -use crate::VirtAddr; use core::{ fmt::{self, Display}, mem::size_of, }; +use crate::addr::{DefaultVirtAddrValidity, VirtAddrGeneric, VirtAddrValidity}; + /// In 64-bit mode the TSS holds information that is not /// directly related to the task-switch mechanism, /// but is used for stack switching when an interrupt or exception occurs. -#[derive(Debug, Clone, Copy)] #[repr(C, packed(4))] -pub struct TaskStateSegment { +pub struct TaskStateSegment { reserved_1: u32, /// The full 64-bit canonical forms of the stack pointers (RSP) for privilege levels 0-2. /// The stack pointers used when a privilege level change occurs from a lower privilege level to a higher one. - pub privilege_stack_table: [VirtAddr; 3], + pub privilege_stack_table: [VirtAddrGeneric; 3], reserved_2: u64, /// The full 64-bit canonical forms of the interrupt stack table (IST) pointers. /// The stack pointers used when an entry in the Interrupt Descriptor Table has an IST value other than 0. - pub interrupt_stack_table: [VirtAddr; 7], + pub interrupt_stack_table: [VirtAddrGeneric; 7], reserved_3: u64, reserved_4: u16, /// The 16-bit offset to the I/O permission bit map from the 64-bit TSS base. It must not @@ -27,7 +27,7 @@ pub struct TaskStateSegment { pub iomap_base: u16, } -impl TaskStateSegment { +impl TaskStateSegment { /// Creates a new TSS with zeroed privilege and interrupt stack table and an /// empty I/O-Permission Bitmap. /// @@ -35,11 +35,12 @@ impl TaskStateSegment { /// `size_of::() - 1`, this means that `iomap_base` is /// initialized to `size_of::()`. #[inline] - pub const fn new() -> TaskStateSegment { + #[rustversion::attr(since(1.61), const)] + pub fn new_with_validity() -> Self { TaskStateSegment { - privilege_stack_table: [VirtAddr::zero(); 3], - interrupt_stack_table: [VirtAddr::zero(); 7], - iomap_base: size_of::() as u16, + privilege_stack_table: [VirtAddrGeneric::zero(); 3], + interrupt_stack_table: [VirtAddrGeneric::zero(); 7], + iomap_base: size_of::() as u16, reserved_1: 0, reserved_2: 0, reserved_3: 0, @@ -48,10 +49,53 @@ impl TaskStateSegment { } } -impl Default for TaskStateSegment { +// These traits are implemented manually because Rust 1.59 has limited derive support for generic +// packed structs. They can use derive once the MSRV is raised to Rust 1.69. +impl Copy for TaskStateSegment {} + +impl Clone for TaskStateSegment { + fn clone(&self) -> Self { + *self + } +} + +impl fmt::Debug for TaskStateSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let reserved_1 = self.reserved_1; + let privilege_stack_table = self.privilege_stack_table; + let reserved_2 = self.reserved_2; + let interrupt_stack_table = self.interrupt_stack_table; + let reserved_3 = self.reserved_3; + let reserved_4 = self.reserved_4; + let iomap_base = self.iomap_base; + + f.debug_struct("TaskStateSegment") + .field("reserved_1", &reserved_1) + .field("privilege_stack_table", &privilege_stack_table) + .field("reserved_2", &reserved_2) + .field("interrupt_stack_table", &interrupt_stack_table) + .field("reserved_3", &reserved_3) + .field("reserved_4", &reserved_4) + .field("iomap_base", &iomap_base) + .finish() + } +} + +impl TaskStateSegment { + /// Creates a new TSS with the default virtual-address validity. + /// + /// Stack addresses assigned later retain their creation-time validity guarantees. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn new() -> Self { + Self::new_with_validity() + } +} + +impl Default for TaskStateSegment { #[inline] fn default() -> Self { - Self::new() + Self::new_with_validity() } } @@ -123,5 +167,15 @@ mod tests { // Per the SDM, the minimum size of a TSS is 0x68 bytes, giving a // minimum limit of 0x67. assert_eq!(size_of::(), 0x68); + #[cfg(feature = "virt_addr_57")] + assert_eq!( + size_of::>>(), + 0x68 + ); + #[cfg(feature = "virt_addr_rt")] + assert_eq!( + size_of::>(), + 0x68 + ); } } diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 7a32657c..1b08c3ef 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -24,6 +24,7 @@ spin = "0.5.0" # Overwrite the x86_64 crate for both direct and indirect dependencies. [dependencies.x86_64] path = ".." +features = ["virt_addr_57", "virt_addr_rt"] [patch.crates-io] x86_64 = { path = ".." } diff --git a/testing/src/tests.rs b/testing/src/tests.rs index d221fefc..65b0ab8e 100644 --- a/testing/src/tests.rs +++ b/testing/src/tests.rs @@ -4,3 +4,17 @@ fn example_test() { assert_eq!(0, 0); serial_println!("[ok]"); } + +#[test_case] +fn runtime_virtual_address_validity_in_la48() { + use x86_64::registers::control::{Cr4, Cr4Flags}; + use x86_64::{VirtAddr48, VirtAddr57, VirtAddrRT}; + + serial_print!("runtime_virtual_address_validity_in_la48... "); + assert!(!Cr4::read().contains(Cr4Flags::L5_PAGING)); + assert!(VirtAddrRT::try_new(0x0000_7fff_ffff_ffff).is_ok()); + assert!(VirtAddrRT::try_new(0x00ff_ffff_ffff_ffff).is_err()); + assert!(VirtAddr48::new(0x1234).is_valid_currently()); + assert!(!VirtAddr57::new(0x00ff_0000_0000_0000).is_valid_currently()); + serial_println!("[ok]"); +}