diff --git a/Cargo.lock b/Cargo.lock index 163090f441..234fec1585 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5120,6 +5120,7 @@ version = "0.0.0" dependencies = [ "aarch64defs", "arrayvec", + "bitfield-struct 0.10.1", "cfg-if", "crc32fast", "fdt", diff --git a/openhcl/openhcl_boot/Cargo.toml b/openhcl/openhcl_boot/Cargo.toml index 67e9deec66..b10f5034ba 100644 --- a/openhcl/openhcl_boot/Cargo.toml +++ b/openhcl/openhcl_boot/Cargo.toml @@ -8,6 +8,7 @@ rust-version.workspace = true [dependencies] aarch64defs.workspace = true +bitfield-struct.workspace = true minimal_rt.workspace = true underhill_confidentiality.workspace = true host_fdt_parser.workspace = true diff --git a/openhcl/openhcl_boot/src/arch/aarch64/mod.rs b/openhcl/openhcl_boot/src/arch/aarch64/mod.rs index a26231f9f8..788b039d56 100644 --- a/openhcl/openhcl_boot/src/arch/aarch64/mod.rs +++ b/openhcl/openhcl_boot/src/arch/aarch64/mod.rs @@ -10,12 +10,17 @@ mod memory; mod vp; mod vsm; +use crate::host_params::shim_params::ShimParams; pub use memory::physical_address_bits; pub use memory::setup_vtl2_memory; pub use memory::verify_imported_regions_hash; pub use vp::setup_vtl2_vp; pub use vsm::get_isolation_type; +pub fn initialize(_: &ShimParams) {} + +pub fn uninitialize(_: &ShimParams) {} + // Entry point. #[cfg(minimal_rt)] core::arch::global_asm! { diff --git a/openhcl/openhcl_boot/src/arch/x86_64/address_space.rs b/openhcl/openhcl_boot/src/arch/x86_64/address_space.rs index 8a444405c5..1bd32d9535 100644 --- a/openhcl/openhcl_boot/src/arch/x86_64/address_space.rs +++ b/openhcl/openhcl_boot/src/arch/x86_64/address_space.rs @@ -25,21 +25,21 @@ use zerocopy::FromBytes; use zerocopy::IntoBytes; use zerocopy::KnownLayout; -const X64_PTE_PRESENT: u64 = 1; -const X64_PTE_READ_WRITE: u64 = 1 << 1; -const X64_PTE_ACCESSED: u64 = 1 << 5; +pub const X64_PTE_PRESENT: u64 = 1; +pub const X64_PTE_READ_WRITE: u64 = 1 << 1; +pub const X64_PTE_ACCESSED: u64 = 1 << 5; const X64_PTE_DIRTY: u64 = 1 << 6; const X64_PTE_LARGE_PAGE: u64 = 1 << 7; -const X64_PTE_CONFIDENTIAL: u64 = 1 << 51; +pub const X64_PTE_CONFIDENTIAL: u64 = 1 << 51; -const PAGE_TABLE_ENTRY_COUNT: usize = 512; +pub const PAGE_TABLE_ENTRY_COUNT: usize = 512; -const X64_PAGE_SHIFT: u64 = 12; -const X64_PTE_BITS: u64 = 9; +pub const X64_PAGE_SHIFT: u64 = 12; +pub const X64_PTE_BITS: u64 = 9; #[derive(Debug, IntoBytes, KnownLayout, FromBytes)] #[repr(transparent)] -struct PageTableEntry { +pub struct PageTableEntry { entry: AtomicU64, } #[derive(Debug, Copy, Clone)] @@ -117,7 +117,7 @@ impl PageTableEntry { #[repr(C)] #[derive(Debug, IntoBytes, KnownLayout, FromBytes)] -struct PageTable { +pub struct PageTable { entries: [PageTableEntry; PAGE_TABLE_ENTRY_COUNT], } diff --git a/openhcl/openhcl_boot/src/arch/x86_64/mod.rs b/openhcl/openhcl_boot/src/arch/x86_64/mod.rs index e24c4b0eb9..9fcb07790b 100644 --- a/openhcl/openhcl_boot/src/arch/x86_64/mod.rs +++ b/openhcl/openhcl_boot/src/arch/x86_64/mod.rs @@ -14,6 +14,7 @@ mod vp; mod vsm; use crate::host_params::shim_params::IsolationType; +use crate::host_params::shim_params::ShimParams; pub use address_space::TdxHypercallPage; pub use memory::setup_vtl2_memory; pub use memory::verify_imported_regions_hash; @@ -40,6 +41,18 @@ pub fn physical_address_bits(isolation: IsolationType) -> u8 { } } +pub fn initialize(p: &ShimParams) { + if p.isolation_type == IsolationType::Snp { + snp::Ghcb::initialize(); + } +} + +pub fn uninitialize(p: &ShimParams) { + if p.isolation_type == IsolationType::Snp { + snp::Ghcb::uninitialize(); + } +} + // Entry point. #[cfg(minimal_rt)] core::arch::global_asm! { diff --git a/openhcl/openhcl_boot/src/arch/x86_64/snp.rs b/openhcl/openhcl_boot/src/arch/x86_64/snp.rs index ebdfb607b3..983f7cd3d2 100644 --- a/openhcl/openhcl_boot/src/arch/x86_64/snp.rs +++ b/openhcl/openhcl_boot/src/arch/x86_64/snp.rs @@ -4,12 +4,38 @@ //! SNP support for the bootshim. use super::address_space::LocalMap; +use super::address_space::PAGE_TABLE_ENTRY_COUNT; +use super::address_space::X64_PAGE_SHIFT; +use super::address_space::X64_PTE_ACCESSED; +use super::address_space::X64_PTE_PRESENT; +use super::address_space::X64_PTE_READ_WRITE; +use crate::arch::x86_64::address_space::X64_PTE_CONFIDENTIAL; +use crate::single_threaded::SingleThreaded; +use bitfield_struct::bitfield; use core::arch::asm; +use core::cell::Cell; +use core::cell::UnsafeCell; +use core::sync::atomic::Ordering; +use core::sync::atomic::compiler_fence; +use core::sync::atomic::fence; +use hvdef::HvRegisterValue; +use hvdef::HvX64RegisterName; +use hvdef::hypercall::HvInputVtl; +use hvdef::hypercall::HypercallOutput; use memory_range::MemoryRange; use minimal_rt::arch::msr::read_msr; use minimal_rt::arch::msr::write_msr; +use x86defs::X64_PAGE_SIZE; use x86defs::X86X_AMD_MSR_GHCB; use x86defs::snp::GhcbInfo; +use x86defs::snp::GhcbMsr; +use x86defs::snp::GhcbProtocolVersion; +use x86defs::snp::GhcbUsage; +use x86defs::snp::SevExitCode; +use x86defs::snp::SevIoAccessInfo; +use zerocopy::IntoBytes; + +static GHCB_PREVIOUS: SingleThreaded> = SingleThreaded(Cell::new(0)); pub struct Ghcb; @@ -32,25 +58,567 @@ pub enum AcceptGpaError { Unknown, } -impl Ghcb { +struct GhcbCall { + extra_data: u64, + page_number: u64, + info: GhcbInfo, +} + +// The memory mapping bits likely don't belong to this module, but +// no centralized facility seems to exist for them yet. + +/// 4-level virtual address. The number of bits used in the VA +/// ought to be requested through CPUID. Here it is "hardcoded" +/// to 48 bits, which is the most common case. +#[bitfield(u64)] +struct VirtAddr4Level { + /// Offset inside the page. + #[bits(12)] + offset: usize, + /// PT index. + #[bits(9)] + pt_index: usize, + /// PD index. + #[bits(9)] + pd_index: usize, + /// PDP index. + #[bits(9)] + pdp_index: usize, + /// PML4 index. + #[bits(9)] + pml4_index: usize, + /// Reserved bits. + #[bits(16)] + reserved: usize, +} + +impl VirtAddr4Level { + const fn canonicalize(&self) -> VirtAddr4Level { + // If PML4 is greater than 255, make it upper-half canonical + // by sign extending the PML4 index. + Self::from_bits((self.into_bits().wrapping_shl(16) as i64).wrapping_shr(16) as u64) + } +} + +/// Page table. +#[repr(C, align(4096))] +struct PageTable { + entries: [u64; PAGE_TABLE_ENTRY_COUNT], +} + +// Would be great to allocate this pages dynamically as otherwise they go +// into the IGVM file and require measurement through the PSP. + +/// PDP table to map the GHCB +static PDP_TABLE: SingleThreaded> = + SingleThreaded(UnsafeCell::new(PageTable { + entries: [0; PAGE_TABLE_ENTRY_COUNT], + })); + +/// PD table to map the GHCB +static PD_TABLE: SingleThreaded> = + SingleThreaded(UnsafeCell::new(PageTable { + entries: [0; PAGE_TABLE_ENTRY_COUNT], + })); + +/// Page table to map the GHCB +static PAGE_TABLE: SingleThreaded> = + SingleThreaded(UnsafeCell::new(PageTable { + entries: [0; PAGE_TABLE_ENTRY_COUNT], + })); + +const PML4_INDEX: usize = 0x1d0; // upper half mapping +const PDP_INDEX: usize = 0; +const PD_INDEX: usize = 0; +const PT_INDEX: usize = 0; +const GHCB_GVA: VirtAddr4Level = VirtAddr4Level::new() + .with_pt_index(PT_INDEX) + .with_pd_index(PD_INDEX) + .with_pdp_index(PDP_INDEX) + .with_pml4_index(PML4_INDEX) + .canonicalize(); + +fn get_cr3() -> u64 { + let mut cr3: u64; + + // SAFETY: No access to the memory. + unsafe { + asm!("mov {0}, cr3", out(reg) cr3, options(nostack)); + } + cr3 +} + +fn cache_lines_flush_page(addr: u64) { + const FLUSH_SIZE: u64 = 64; // NOTE: hardcoded cache line size. + let start = addr & !(X64_PAGE_SIZE - 1); + let end = start + X64_PAGE_SIZE; + + // Make sure there are no pending writes on the cache lines. + fence(Ordering::SeqCst); + + for addr in (start..end).step_by(FLUSH_SIZE as usize) { + // SAFETY: No concurrency issues. + unsafe { + asm!("clflush [{0}]", in(reg) addr, options(nostack)); + } + } +} + +fn flush_tlb() { + fence(Ordering::SeqCst); + // NOTE: no flush for the global pages. + // SAFETY: No concurrency issues. + unsafe { + asm!("mov cr3, {0}", in(reg) get_cr3(), options(nostack)); + } + compiler_fence(Ordering::SeqCst); +} + +fn page_table(pfn: u64) -> &'static mut [u64] { + // SAFETY: The next page address must be set, identical mapping. + unsafe { + core::slice::from_raw_parts_mut((pfn << X64_PAGE_SHIFT) as *mut u64, PAGE_TABLE_ENTRY_COUNT) + } +} + +fn pte_for_pfn(pfn: u64, confidential: bool) -> u64 { + let common = X64_PTE_PRESENT | X64_PTE_ACCESSED | X64_PTE_READ_WRITE | (pfn << X64_PAGE_SHIFT); + if confidential { + common | X64_PTE_CONFIDENTIAL + } else { + common + } +} + +/// GHCB page access. The GHCB page is statically allocated and +/// initialized. The GHCB page might be accessed and modified +/// concurrently by the (malicious) host, and the atomic accesses +/// mitigate the possibility of torn reads/writes. +mod ghcb_access { + use super::GHCB_GVA; + use crate::PageAlign; + use crate::arch::x86_64::address_space::X64_PAGE_SHIFT; + use crate::zeroed; + use core::mem::offset_of; + use core::sync::atomic::AtomicU8; + use core::sync::atomic::AtomicU16; + use core::sync::atomic::AtomicU32; + use core::sync::atomic::AtomicU64; + use core::sync::atomic::Ordering; + use x86defs::snp::GHCB_PAGE_HV_HYPERCALL_DATA_SIZE; + use x86defs::snp::GhcbPage; + use x86defs::snp::GhcbPageHvHypercall; + use x86defs::snp::GhcbProtocolVersion; + use x86defs::snp::GhcbSaveArea; + use x86defs::snp::GhcbUsage; + + /// The GHCB page itself. Must not be *ever* accessed directly + /// using the static. It might be unaccepted at any time, and + /// the VA below is mapped with the C-bit set. + /// + /// The declaration is just a means to get the page statically + /// allocated and aligned. + static GHCB: PageAlign<[u8; size_of::()]> = zeroed(); + + pub fn page_number() -> u64 { + // Identical mapping, the GVA is the same as the GPA. + let gva = GHCB.0.as_ptr() as u64; + gva >> X64_PAGE_SHIFT + } + /// # Safety /// - /// Regardless of the content of the GHCB page or MSR, this instruction should not be able - /// to cause memory safety issues. - fn sev_vmgexit() { + /// The caller must ensure that the GHCB page is properly mapped + /// and can be accessed in a memory-safe manner. + unsafe fn ghcb_data() -> &'static mut [T] { + // SAFETY: The GHCB page is statically allocated and initialized. + // It is either mapped by the time of access, or the code won't + // be executed at all due to the hardware fault. + unsafe { + core::slice::from_raw_parts_mut( + GHCB_GVA.into_bits() as *mut T, + size_of::() / size_of::(), + ) + } + } + + macro_rules! ghcb_field_set { + ($field:ident, $type:ty, $val:expr) => {{ + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_data::<$type>() }; + let pos = offset_of!(GhcbPage, $field) / size_of::<$type>(); + ghcb_data[pos].store($val, Ordering::SeqCst); + }}; + } + + macro_rules! ghcb_save_field_set { + ($field:ident, $type:ty, $func:ident, $val:expr) => {{ + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_data::<$type>() }; + // Save area is at the beginning of the GHCB page. + let pos = offset_of!(GhcbSaveArea, $field) / size_of::<$type>(); + ghcb_data[pos].$func($val, Ordering::SeqCst); + }}; + } + + macro_rules! ghcb_save_field_get { + ($field:ident, $type:ty) => {{ + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_data::<$type>() }; + // Save area is at the beginning of the GHCB page. + let pos = offset_of!(GhcbSaveArea, $field) / size_of::<$type>(); + ghcb_data[pos].load(Ordering::SeqCst) + }}; + } + + pub fn clear_page() { + // SAFETY: Atomic access to the GHCB page. + unsafe { ghcb_data::() } + .iter() + .for_each(|x| x.store(0, Ordering::SeqCst)); + } + + pub fn clear_bitmaps() { + ghcb_save_field_set!(valid_bitmap0, AtomicU64, store, 0); + ghcb_save_field_set!(valid_bitmap1, AtomicU64, store, 0); + } + + macro_rules! ghcb_save_set_valid_bitmap0 { + ($save_field:ident) => {{ + let mask = 1u64 << (offset_of!(GhcbSaveArea, $save_field) / 8); + ghcb_save_field_set!(valid_bitmap0, AtomicU64, fetch_or, mask); + }}; + } + + macro_rules! ghcb_save_set_valid_bitmap1 { + ($save_field:ident) => {{ + let mask = 1u64 << (offset_of!(GhcbSaveArea, $save_field) / 8 - 64); + ghcb_save_field_set!(valid_bitmap1, AtomicU64, fetch_or, mask); + }}; + } + + macro_rules! ghcb_save_assert_valid_bitmap0 { + ($save_field:ident) => {{ + let mask = 1u64 << (offset_of!(GhcbSaveArea, $save_field) / 8); + assert_eq!(ghcb_save_field_get!(valid_bitmap0, AtomicU64) & mask, mask); + }}; + } + + macro_rules! ghcb_save_assert_valid_bitmap1 { + ($save_field:ident) => {{ + let mask = 1u64 << (offset_of!(GhcbSaveArea, $save_field) / 8 - 64); + assert_eq!(ghcb_save_field_get!(valid_bitmap1, AtomicU64) & mask, mask); + }}; + } + + pub fn set_usage(usage: GhcbUsage) { + ghcb_field_set!(ghcb_usage, AtomicU32, usage.into_bits()); + } + + pub fn set_protocol_version(version: GhcbProtocolVersion) { + ghcb_field_set!(protocol_version, AtomicU16, version.into_bits()); + } + + pub fn set_sw_exit_code(code: u64) { + ghcb_save_field_set!(sw_exit_code, AtomicU64, store, code); + ghcb_save_set_valid_bitmap1!(sw_exit_code); + } + + pub fn set_sw_exit_info1(info: u64) { + ghcb_save_field_set!(sw_exit_info1, AtomicU64, store, info); + ghcb_save_set_valid_bitmap1!(sw_exit_info1); + } + + pub fn sw_exit_info1() -> u64 { + ghcb_save_assert_valid_bitmap1!(sw_exit_info1); + ghcb_save_field_get!(sw_exit_info1, AtomicU64) + } + + pub fn set_sw_exit_info2(info: u64) { + ghcb_save_field_set!(sw_exit_info2, AtomicU64, store, info); + ghcb_save_set_valid_bitmap1!(sw_exit_info2); + } + + pub fn set_rax(rax: u64) { + ghcb_save_field_set!(rax, AtomicU64, store, rax); + ghcb_save_set_valid_bitmap0!(rax); + } + + pub fn rax() -> u64 { + ghcb_save_assert_valid_bitmap0!(rax); + ghcb_save_field_get!(rax, AtomicU64) + } + + pub fn set_rcx(rcx: u64) { + ghcb_save_field_set!(rcx, AtomicU64, store, rcx); + ghcb_save_set_valid_bitmap1!(rcx); + } + + pub fn set_rdx(rdx: u64) { + ghcb_save_field_set!(rdx, AtomicU64, store, rdx); + ghcb_save_set_valid_bitmap1!(rdx); + } + + pub fn rdx() -> u64 { + ghcb_save_assert_valid_bitmap1!(rdx); + ghcb_save_field_get!(rdx, AtomicU64) + } + + /// # Safety + /// + /// The caller must ensure that the GHCB page is properly mapped + /// and can be accessed in a memory-safe manner. + unsafe fn ghcb_hv_hypercall() -> &'static mut [T] { + // SAFETY: The GHCB page is statically allocated and initialized. + // It is either mapped by the time of access, or the code won't + // be executed at all due to the hardware fault. + unsafe { + core::slice::from_raw_parts_mut( + GHCB_GVA.into_bits() as *mut T, + size_of::() / size_of::(), + ) + } + } + + macro_rules! ghcb_hv_hypercall_field_set { + ($field:ident, $type:ty, $val:expr) => {{ + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_hv_hypercall::<$type>() }; + let pos = offset_of!(GhcbPageHvHypercall, $field) / size_of::<$type>(); + ghcb_data[pos].store($val, Ordering::SeqCst); + }}; + } + + macro_rules! ghcb_hv_hypercall_field_get { + ($field:ident, $type:ty) => {{ + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_hv_hypercall::<$type>() }; + let pos = offset_of!(GhcbPageHvHypercall, $field) / size_of::<$type>(); + ghcb_data[pos].load(Ordering::SeqCst) + }}; + } + + pub fn set_hypercall_data(data: &[u8], start: usize) { + // SAFETY: Atomic access to the GHCB page. + let ghcb_data = unsafe { ghcb_hv_hypercall::() }; + assert!(data.len() <= GHCB_PAGE_HV_HYPERCALL_DATA_SIZE); + + ghcb_data[start..start + data.len()] + .iter() + .zip(data.iter()) + .for_each(|(x, y)| x.store(*y, Ordering::SeqCst)); + } + + pub fn set_hypercall_input(input: u64) { + ghcb_hv_hypercall_field_set!(io, AtomicU64, input); + } + + pub fn hypercall_output() -> u64 { + ghcb_hv_hypercall_field_get!(io, AtomicU64) + } + + pub fn set_hypercall_output_gpa(gpa: u64) { + ghcb_hv_hypercall_field_set!(output_gpa, AtomicU64, gpa); + } +} + +#[allow(dead_code)] +enum IoAccessSize { + Byte = 1, + Word = 2, + Dword = 4, +} + +impl Ghcb { + pub fn initialize() { + // Make sure page alignment. + assert!((PAGE_TABLE.get() as u64) & (X64_PAGE_SIZE - 1) == 0); + assert!((PD_TABLE.get() as u64) & (X64_PAGE_SIZE - 1) == 0); + assert!((PDP_TABLE.get() as u64) & (X64_PAGE_SIZE - 1) == 0); + + // Map the GHCB page in the guest as non-confidential. + + let page_root = get_cr3() & !(X64_PAGE_SIZE - 1); + let pml4table = page_table(page_root >> X64_PAGE_SHIFT); + assert!(pml4table[PML4_INDEX] & X64_PTE_PRESENT == 0); + + // Running in identical mapping. + let pdp_table_pfn = (PDP_TABLE.get() as u64) >> X64_PAGE_SHIFT; + let pd_table_pfn = (PD_TABLE.get() as u64) >> X64_PAGE_SHIFT; + let page_table_pfn = (PAGE_TABLE.get() as u64) >> X64_PAGE_SHIFT; + let page_number = ghcb_access::page_number(); + + let pdp_table = page_table(pdp_table_pfn); + let pd_table = page_table(pd_table_pfn); + let page_table = page_table(page_table_pfn); + + pml4table[PML4_INDEX] = pte_for_pfn(pdp_table_pfn, true); + pdp_table[PDP_INDEX] = pte_for_pfn(pd_table_pfn, true); + pd_table[PD_INDEX] = pte_for_pfn(page_table_pfn, true); + page_table[PT_INDEX] = pte_for_pfn(page_number, true); + + flush_tlb(); + // Evict the page from the cache before changing the encrypted state. + cache_lines_flush_page(GHCB_GVA.into_bits()); + + // Unaccept the page, invalidates page state. + pvalidate(page_number, GHCB_GVA.into_bits(), false, false).expect("memory unaccept"); + // Issue VMG exit to request the hypervisor to update the page state to host visible in RMP. + let resp = Ghcb::ghcb_call(GhcbCall { + info: GhcbInfo::PAGE_STATE_CHANGE, + extra_data: x86defs::snp::GHCB_DATA_PAGE_STATE_SHARED, + page_number, + }); + assert!(resp.into_bits() == GhcbInfo::PAGE_STATE_UPDATED.0); + + // Map the page as non-confidential by updating the PTE. + page_table[PT_INDEX] = pte_for_pfn(page_number, false); + flush_tlb(); + // Evict the page from the cache before changing the encrypted state. + cache_lines_flush_page(GHCB_GVA.into_bits()); + + // Flipping the C-bit makes the contents of the GHCB page scrambled, + // zero it out. + ghcb_access::clear_page(); + ghcb_access::set_protocol_version(GhcbProtocolVersion::V2); + + // Register the GHCB page with the hypervisor. + + let resp = Self::ghcb_call(GhcbCall { + extra_data: 0, + page_number: ghcb_access::page_number(), + info: GhcbInfo::REGISTER_REQUEST, + }); + assert!( + resp.info() == GhcbInfo::REGISTER_RESPONSE.0 + && resp.extra_data() == 0 + && resp.pfn() == ghcb_access::page_number(), + "GhcbInfo::REGISTER_RESPONSE returned msr value {resp:x?}" + ); + + // Register to issue Hyper-V hypercalls. + let guest_os_id = hvdef::hypercall::HvGuestOsMicrosoft::new().with_os_id(1); + assert!(Self::set_msr( + hvdef::HV_X64_MSR_GUEST_OS_ID, + guest_os_id.into() + )); + // and make sure it is set as expected. + assert!( + Self::get_msr(hvdef::HV_X64_MSR_GUEST_OS_ID).expect("GHCB: Failed to set guest OS ID") + == guest_os_id.into() + ); + Self::set_register(HvX64RegisterName::GuestOsId, guest_os_id.into_bits().into()) + .expect("failed to set guest OS ID"); + + // SAFETY: Always safe to read the GHCB MSR, no concurrency issues. + GHCB_PREVIOUS.replace(unsafe { read_msr(X86X_AMD_MSR_GHCB) }); + } + + pub fn uninitialize() { + // Unregister from issuing Hyper-V hypercalls. + let guest_os_id = hvdef::hypercall::HvGuestOsMicrosoft::new(); + Self::set_register(HvX64RegisterName::GuestOsId, guest_os_id.into_bits().into()) + .expect("failed to set guest OS ID"); + assert!(Self::set_msr( + hvdef::HV_X64_MSR_GUEST_OS_ID, + guest_os_id.into() + )); + // and make sure it is set as expected. + assert!( + Self::get_msr(hvdef::HV_X64_MSR_GUEST_OS_ID).expect("GHCB: Failed to set guest OS ID") + == guest_os_id.into() + ); + + // Tell the hypervisor that the GHCB page is at GPA 0 now. + // That'll make it to unmap the overlay page and let the `pvalidate` + // below to succeed. + // + // Soon after this, the GHCB page will be mapped by the kernel at the + // GPA of its chhosing. The temporarily mapping at GPA 0 poses no + // security risk as that page does not contain any sensitive data + // in the IGVM file. + // + // Once support for unpamming the GHCB page from the latest SEV-ES + // specification is added, this will be removed in favor of the standard + // unmap operation. + let resp = Self::ghcb_call(GhcbCall { + extra_data: 0, + page_number: 0, + info: GhcbInfo::REGISTER_REQUEST, + }); + assert!( + resp.info() == GhcbInfo::REGISTER_RESPONSE.0 + && resp.extra_data() == 0 + && resp.pfn() == 0, + "GhcbInfo::REGISTER_RESPONSE returned msr value {resp:x?}" + ); + + // Map the GHCB page in the guest as confidential and accept it again + // to return to the original state. + + // Evict the page from the cache before changing the encrypted state. + cache_lines_flush_page(GHCB_GVA.into_bits()); + + // Update the page table entry to make it confidential. + // Running in identical mapping. + let page_table_pfn = (PAGE_TABLE.get() as u64) >> X64_PAGE_SHIFT; + let page_table = page_table(page_table_pfn); + let page_number = ghcb_access::page_number(); + + page_table[PT_INDEX] |= X64_PTE_CONFIDENTIAL; + + // Issue VMG exit to request the hypervisor to update the page state to private in RMP. + let resp = Ghcb::ghcb_call(GhcbCall { + info: GhcbInfo::PAGE_STATE_CHANGE, + extra_data: x86defs::snp::GHCB_DATA_PAGE_STATE_PRIVATE, + page_number, + }); + assert!(resp.into_bits() == GhcbInfo::PAGE_STATE_UPDATED.0); + + // Accept the page, invalidates page state. + pvalidate(page_number, GHCB_GVA.into_bits(), false, true).expect("memory accept"); + + // Needed here, not before. + flush_tlb(); + + ghcb_access::clear_page(); + + // SAFETY: Always safe to write the GHCB MSR, no concurrency issues. + unsafe { write_msr(X86X_AMD_MSR_GHCB, GHCB_PREVIOUS.get()) }; + } + + #[inline(always)] + fn vmg_exit() { // SAFETY: Using the `vmgexit` instruction forces an exit to the hypervisor but doesn't // directly change program state. unsafe { - asm! {r#" - rep vmmcall - "# - } + asm!("rep vmmcall", options(nomem, nostack)); } } + /// Perform the GHCB call + fn ghcb_call(call_data: GhcbCall) -> GhcbMsr { + let GhcbCall { + info, + extra_data, + page_number, + } = call_data; + let ghcb_control = GhcbMsr::new() + .with_pfn(page_number) + .with_info(info.0) + .with_extra_data(extra_data); + + GhcbMsr::from_bits( + // SAFETY: Writing and reding known good value to/from the GHCB MSR, following the GHCB protocol. + unsafe { + write_msr(X86X_AMD_MSR_GHCB, ghcb_control.into_bits()); + Self::vmg_exit(); + read_msr(X86X_AMD_MSR_GHCB) + }, + ) + } + pub fn change_page_visibility(range: MemoryRange, host_visible: bool) { - // SAFETY: Always safe to read the GHCB MSR. - let previous_value = unsafe { read_msr(X86X_AMD_MSR_GHCB) }; for page_number in range.start_4k_gpn()..range.end_4k_gpn() { let extra_data = if host_visible { x86defs::snp::GHCB_DATA_PAGE_STATE_SHARED @@ -58,26 +626,169 @@ impl Ghcb { x86defs::snp::GHCB_DATA_PAGE_STATE_PRIVATE }; - let val = (extra_data << 52) | (page_number << 12) | GhcbInfo::PAGE_STATE_CHANGE.0; - - // SAFETY: Writing known good value to the GHCB MSR. - let val = unsafe { - write_msr(X86X_AMD_MSR_GHCB, val); - Self::sev_vmgexit(); - read_msr(X86X_AMD_MSR_GHCB) - }; + let resp = Self::ghcb_call(GhcbCall { + info: GhcbInfo::PAGE_STATE_CHANGE, + extra_data, + page_number, + }); // High 32 bits are status and should be 0 (HV_STATUS_SUCCESS), Low 32 bits should be // GHCB_INFO_PAGE_STATE_UPDATED. Assert if otherwise. assert!( - val == GhcbInfo::PAGE_STATE_UPDATED.0, - "GhcbInfo::PAGE_STATE_UPDATED returned msr value {val}" + resp.into_bits() == GhcbInfo::PAGE_STATE_UPDATED.0, + "GhcbInfo::PAGE_STATE_UPDATED returned msr value {resp:x?}" ); } + } + + #[must_use] + fn read_io_port(port: u16, access_size: IoAccessSize) -> Option { + ghcb_access::set_usage(GhcbUsage::BASE); + ghcb_access::set_protocol_version(GhcbProtocolVersion::V2); + ghcb_access::clear_bitmaps(); + + let io_exit_info = SevIoAccessInfo::new() + .with_port(port) + .with_read_access(true); + let io_exit_info = match access_size { + IoAccessSize::Byte => io_exit_info.with_access_size8(true), + IoAccessSize::Word => io_exit_info.with_access_size16(true), + IoAccessSize::Dword => io_exit_info.with_access_size32(true), + }; + + ghcb_access::set_sw_exit_code(SevExitCode::IOIO.0); + ghcb_access::set_sw_exit_info1(io_exit_info.into_bits().into()); + ghcb_access::set_sw_exit_info2(0); + + Self::ghcb_call(GhcbCall { + info: GhcbInfo::NORMAL, + extra_data: 0, + page_number: ghcb_access::page_number(), + }); + ghcb_access::set_usage(GhcbUsage::INVALID); + + if ghcb_access::sw_exit_info1() != 0 { + None + } else { + Some(ghcb_access::rax() as u32) + } + } + + #[must_use] + fn write_io_port(port: u16, access_size: IoAccessSize, data: u32) -> bool { + ghcb_access::set_usage(GhcbUsage::BASE); + ghcb_access::set_protocol_version(GhcbProtocolVersion::V2); + ghcb_access::clear_bitmaps(); + + let io_exit_info = SevIoAccessInfo::new() + .with_port(port) + .with_read_access(false); + let io_exit_info = match access_size { + IoAccessSize::Byte => io_exit_info.with_access_size8(true), + IoAccessSize::Word => io_exit_info.with_access_size16(true), + IoAccessSize::Dword => io_exit_info.with_access_size32(true), + }; + + ghcb_access::set_sw_exit_code(SevExitCode::IOIO.0); + ghcb_access::set_sw_exit_info1(io_exit_info.into_bits().into()); + ghcb_access::set_sw_exit_info2(0); + + ghcb_access::set_rax(data as u64); + + Self::ghcb_call(GhcbCall { + info: GhcbInfo::NORMAL, + extra_data: 0, + page_number: ghcb_access::page_number(), + }); + ghcb_access::set_usage(GhcbUsage::INVALID); + + ghcb_access::sw_exit_info1() == 0 + } + + #[must_use] + pub fn set_msr(msr_index: u32, value: u64) -> bool { + ghcb_access::set_usage(GhcbUsage::BASE); + ghcb_access::set_protocol_version(GhcbProtocolVersion::V2); + ghcb_access::clear_bitmaps(); + + ghcb_access::set_sw_exit_code(SevExitCode::MSR.0); + ghcb_access::set_sw_exit_info1(1); + ghcb_access::set_sw_exit_info2(0); + + ghcb_access::set_rcx(msr_index as u64); + ghcb_access::set_rax(value as u32 as u64); + ghcb_access::set_rdx((value >> 32) as u32 as u64); + + Self::ghcb_call(GhcbCall { + info: GhcbInfo::NORMAL, + extra_data: 0, + page_number: ghcb_access::page_number(), + }); + ghcb_access::set_usage(GhcbUsage::INVALID); + + ghcb_access::sw_exit_info1() == 0 + } + + #[must_use] + pub fn get_msr(msr_index: u32) -> Option { + ghcb_access::set_usage(GhcbUsage::BASE); + ghcb_access::set_protocol_version(GhcbProtocolVersion::V2); + ghcb_access::clear_bitmaps(); + + ghcb_access::set_sw_exit_code(SevExitCode::MSR.0); + ghcb_access::set_sw_exit_info1(0); + ghcb_access::set_sw_exit_info2(0); + + ghcb_access::set_rcx(msr_index as u64); + + Self::ghcb_call(GhcbCall { + info: GhcbInfo::NORMAL, + extra_data: 0, + page_number: ghcb_access::page_number(), + }); + ghcb_access::set_usage(GhcbUsage::INVALID); + + if ghcb_access::sw_exit_info1() != 0 { + None + } else { + Some(ghcb_access::rax() | (ghcb_access::rdx() << 32)) + } + } + + pub fn set_register( + name: HvX64RegisterName, + value: HvRegisterValue, + ) -> Result<(), hvdef::HvError> { + let header = hvdef::hypercall::GetSetVpRegisters { + partition_id: hvdef::HV_PARTITION_ID_SELF, + vp_index: hvdef::HV_VP_INDEX_SELF, + target_vtl: HvInputVtl::CURRENT_VTL, + rsvd: [0; 3], + }; + let reg_assoc = hvdef::hypercall::HvRegisterAssoc { + name: name.into(), + pad: Default::default(), + value, + }; + let control = hvdef::hypercall::Control::new() + .with_code(hvdef::HypercallCode::HvCallSetVpRegisters.0) + .with_rep_count(1); + + ghcb_access::set_usage(GhcbUsage::HYPERCALL); + ghcb_access::set_hypercall_data(header.as_bytes(), 0); + ghcb_access::set_hypercall_data(reg_assoc.as_bytes(), size_of_val(&header)); + ghcb_access::set_hypercall_input(control.into_bits()); + ghcb_access::set_hypercall_output_gpa(0); + + Self::ghcb_call(GhcbCall { + info: GhcbInfo::NORMAL, + extra_data: 0, + page_number: ghcb_access::page_number(), + }); + ghcb_access::set_usage(GhcbUsage::INVALID); - // SAFETY: Restoring previous GHCB value is safe. - unsafe { write_msr(X86X_AMD_MSR_GHCB, previous_value) }; + HypercallOutput::from_bits(ghcb_access::hypercall_output()).result() } } @@ -137,7 +848,7 @@ pub fn set_page_acceptance( range: MemoryRange, validate: bool, ) -> Result<(), AcceptGpaError> { - let pages_per_large_page = x86defs::X64_LARGE_PAGE_SIZE / hvdef::HV_PAGE_SIZE; + let pages_per_large_page = x86defs::X64_LARGE_PAGE_SIZE / X64_PAGE_SIZE; let mut page_count = range.page_count_4k(); let mut page_base = range.start_4k_gpn(); @@ -177,3 +888,18 @@ pub fn set_page_acceptance( Ok(()) } + +/// GHCB based io port access. +pub struct SnpIoAccess; + +impl minimal_rt::arch::IoAccess for SnpIoAccess { + unsafe fn inb(&self, port: u16) -> u8 { + // Best effort + Ghcb::read_io_port(port, IoAccessSize::Byte).unwrap_or(!0) as u8 + } + + unsafe fn outb(&self, port: u16, data: u8) { + // Best effort + let _ = Ghcb::write_io_port(port, IoAccessSize::Byte, data as u32); + } +} diff --git a/openhcl/openhcl_boot/src/boot_logger.rs b/openhcl/openhcl_boot/src/boot_logger.rs index 5a080588c1..1280b5c46e 100644 --- a/openhcl/openhcl_boot/src/boot_logger.rs +++ b/openhcl/openhcl_boot/src/boot_logger.rs @@ -8,6 +8,8 @@ //! or any guest code is executed, and therefore it can not leak anything //! sensitive. +#[cfg(target_arch = "x86_64")] +use crate::arch::snp::SnpIoAccess; #[cfg(target_arch = "x86_64")] use crate::arch::tdx::TdxIoAccess; use crate::host_params::shim_params::IsolationType; @@ -28,6 +30,8 @@ enum Logger { Serial(Serial), #[cfg(target_arch = "x86_64")] TdxSerial(Serial), + #[cfg(target_arch = "x86_64")] + SnpSerial(Serial), None, } @@ -37,6 +41,8 @@ impl Logger { Logger::Serial(serial) => serial.write_str(s), #[cfg(target_arch = "x86_64")] Logger::TdxSerial(serial) => serial.write_str(s), + #[cfg(target_arch = "x86_64")] + Logger::SnpSerial(serial) => serial.write_str(s), Logger::None => Ok(()), } } @@ -88,6 +94,8 @@ pub fn boot_logger_runtime_init(isolation_type: IsolationType, com3_serial_avail (IsolationType::None, true) => Logger::Serial(Serial::init()), #[cfg(target_arch = "x86_64")] (IsolationType::Tdx, true) => Logger::TdxSerial(Serial::init(TdxIoAccess)), + #[cfg(target_arch = "x86_64")] + (IsolationType::Snp, true) => Logger::SnpSerial(Serial::init(SnpIoAccess)), _ => Logger::None, }; diff --git a/openhcl/openhcl_boot/src/main.rs b/openhcl/openhcl_boot/src/main.rs index a728b7aae5..786f8a0640 100644 --- a/openhcl/openhcl_boot/src/main.rs +++ b/openhcl/openhcl_boot/src/main.rs @@ -556,6 +556,8 @@ fn shim_main(shim_params_raw_offset: isize) -> ! { enable_enlightened_panic(); } + arch::initialize(&p); + // Enable the in-memory log. boot_logger_memory_init(p.log_buffer); @@ -775,6 +777,8 @@ fn shim_main(shim_params_raw_offset: isize) -> ! { log::info!("uninitializing hypercalls, about to jump to kernel"); hvcall().uninitialize(); + log::info!("uninitializing arch, about to jump to the kernel"); + arch::uninitialize(&p); cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { diff --git a/vm/x86/x86defs/src/lib.rs b/vm/x86/x86defs/src/lib.rs index c97ac8e6be..a0b65d8fbc 100644 --- a/vm/x86/x86defs/src/lib.rs +++ b/vm/x86/x86defs/src/lib.rs @@ -447,6 +447,7 @@ pub struct PageFaultErrorCode { _unused: u32, } +pub const X64_PAGE_SIZE: u64 = 0x1000; pub const X64_LARGE_PAGE_SIZE: u64 = 0x200000; #[bitfield(u64)] diff --git a/vm/x86/x86defs/src/snp.rs b/vm/x86/x86defs/src/snp.rs index f389541dfb..a783819377 100644 --- a/vm/x86/x86defs/src/snp.rs +++ b/vm/x86/x86defs/src/snp.rs @@ -4,6 +4,7 @@ //! AMD SEV-SNP specific definitions. use crate::ApicRegisterValue; +use crate::X64_PAGE_SIZE; use bitfield_struct::bitfield; use static_assertions::const_assert_eq; use zerocopy::FromBytes; @@ -675,13 +676,113 @@ pub const GHCB_DATA_PAGE_STATE_MASK: u64 = 0x00F; pub const GHCB_DATA_PAGE_STATE_LARGE_PAGE: u64 = 0x010; open_enum::open_enum! { + #[derive(FromBytes, IntoBytes)] pub enum GhcbUsage: u32 { BASE = 0, HYPERCALL = 1, VTL_RETURN = 2, + INVALID = !0, } } +impl GhcbUsage { + pub const fn into_bits(self) -> u32 { + self.0 + } + + pub const fn from_bits(bits: u32) -> Self { + Self(bits) + } +} + +open_enum::open_enum! { + #[derive(FromBytes, IntoBytes)] + pub enum GhcbProtocolVersion: u16 { + V1 = 1, + V2 = 2, + } +} + +impl GhcbProtocolVersion { + pub const fn into_bits(self) -> u16 { + self.0 + } + + pub const fn from_bits(bits: u16) -> Self { + Self(bits) + } +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, IntoBytes, FromBytes)] +pub struct GhcbSaveArea { + pub reserved_0x0: [u8; 203], + pub cpl: u8, + pub reserved_0xcc: [u8; 116], + pub xss: u64, + pub reserved_0x148: [u8; 24], + pub dr7: u64, + pub reserved_0x168: [u8; 16], + pub rip: u64, + pub reserved_0x180: [u8; 88], + pub rsp: u64, + pub reserved_0x1e0: [u8; 24], + pub rax: u64, + pub reserved_0x200: [u8; 264], + pub rcx: u64, + pub rdx: u64, + pub rbx: u64, + pub reserved_0x320: [u8; 8], + pub rbp: u64, + pub rsi: u64, + pub rdi: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, + pub reserved_0x380: [u8; 16], + pub sw_exit_code: u64, + pub sw_exit_info1: u64, + pub sw_exit_info2: u64, + pub sw_scratch: u64, + pub reserved_0x3b0: [u8; 56], + pub xcr0: u64, + pub valid_bitmap0: u64, + pub valid_bitmap1: u64, + pub x87_state_gpa: u64, +} + +#[repr(C, align(4096))] +#[derive(Debug, Copy, Clone, IntoBytes, FromBytes)] +pub struct GhcbPage { + pub save: GhcbSaveArea, + pub reserved_save: [u8; 2048 - size_of::()], + pub shared_buffer: [u8; 2032], + pub reserved_0xff0: [u8; 10], + pub protocol_version: GhcbProtocolVersion, + pub ghcb_usage: GhcbUsage, +} + +const _: () = assert!(size_of::() == X64_PAGE_SIZE as usize); + +pub const GHCB_PAGE_HV_HYPERCALL_DATA_SIZE: usize = 4072; + +/// GHCB layout for the secure enlightened Hyper-V hypercalls. +#[repr(C, align(4096))] +#[derive(Debug, Copy, Clone, IntoBytes, FromBytes)] +pub struct GhcbPageHvHypercall { + pub data: [u8; GHCB_PAGE_HV_HYPERCALL_DATA_SIZE], + pub output_gpa: u64, + pub io: u64, + pub reserved: u64, +} + +const _: () = assert!(size_of::() == X64_PAGE_SIZE as usize); + /// Struct representing GHCB hypercall parameters. These are located at the GHCB /// page starting at [`GHCB_PAGE_HYPERCALL_PARAMETERS_OFFSET`]. #[repr(C)]