mvp vm attestation

Cargo.toml

@@ -95,6 +95,12 @@ sled-agent-client = { git = "https://github.com/oxidecomputer/omicron", branch =
 crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "a945a32ba9e1f2098ce3a8963765f1894f37110b" }
 crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "a945a32ba9e1f2098ce3a8963765f1894f37110b" }
 
+# TODO: pin these to git SHAs
+# Attestation
+#dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", branch = "jhendricks/update-sled-agent-types-versions", features = ["sled-agent"] }
+dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", features = ["sled-agent"] }
+vm-attest = { git = "https://github.com/oxidecomputer/vm-attest", rev = "a7c2a341866e359a3126aaaa67823ec5097000cd", default-features = false }
+
 # External dependencies
 anyhow = "1.0"
 async-trait = "0.1.88"
@@ -163,6 +169,7 @@ serde_arrays = "0.1"
 serde_derive = "1.0"
 serde_json = "1.0"
 serde_test = "1.0.138"
+sha2 = "0.10.9"
 slog = "2.7"
 slog-async = "2.8"
 slog-bunyan = "2.4.0"
@@ -201,3 +208,6 @@ zerocopy = "0.8.25"
 # [patch."https://github.com/oxidecomputer/crucible"]
 # crucible = { path = "../crucible/upstairs" }
 # crucible-client-types = { path = "../crucible/crucible-client-types" }
+
+#[patch."https://github.com/oxidecomputer/dice-util"]
+#dice-verifier = { path = "/home/jordan/src/dice-util/verifier", features = ["sled-agent"] }

bin/propolis-server/Cargo.toml

@@ -69,6 +69,7 @@ rgb_frame.workspace = true
 rfb = { workspace = true, features = ["tungstenite"] }
 uuid.workspace = true
 usdt.workspace = true
+vm-attest.workspace = true
 base64.workspace = true
 schemars = { workspace = true, features = ["chrono", "uuid1"] }
 

bin/propolis-server/src/lib/initializer.rs

@@ -4,7 +4,6 @@
 
 use std::convert::TryInto;
 use std::fs::File;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::num::{NonZeroU8, NonZeroUsize};
 use std::os::unix::fs::FileTypeExt;
 use std::sync::Arc;
@@ -25,6 +24,10 @@ use crucible_client_types::VolumeConstructionRequest;
 pub use nexus_client::Client as NexusClient;
 use oximeter::types::ProducerRegistry;
 use oximeter_instruments::kstat::KstatSampler;
+use propolis::attestation;
+use propolis::attestation::server::AttestationServerConfig;
+use propolis::attestation::server::AttestationSock;
+use propolis::attestation::server::AttestationSockInit;
 use propolis::block;
 use propolis::chardev::{self, BlockingSource, Source};
 use propolis::common::{Lifecycle, GB, MB, PAGE_SIZE};
@@ -105,6 +108,9 @@ pub enum MachineInitError {
     #[error("failed to specialize CPUID for vcpu {0}")]
     CpuidSpecializationFailed(i32, #[source] propolis::cpuid::SpecializeError),
 
+    #[error("failed to start attestation server")]
+    AttestationServer(#[source] std::io::Error),
+
     #[cfg(feature = "falcon")]
     #[error("softnpu p9 device missing")]
     SoftNpuP9Missing,
@@ -478,25 +484,23 @@ impl MachineInitializer<'_> {
         Ok(())
     }
 
-    pub fn initialize_vsock(
+    pub async fn initialize_vsock(
         &mut self,
         chipset: &RegisteredChipset,
-    ) -> Result<(), MachineInitError> {
+        attest_cfg: Option<AttestationServerConfig>,
+    ) -> Result<
+        (Option<AttestationSock>, Option<AttestationSockInit>),
+        MachineInitError,
+    > {
         use propolis::vsock::proxy::VsockPortMapping;
 
-        // OANA Port 605 - VM Attestation RFD 605
-        const ATTESTATION_PORT: u16 = 605;
-        const ATTESTATION_ADDR: SocketAddr = SocketAddr::new(
-            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
-            ATTESTATION_PORT,
-        );
-
+        // TODO: early return if none?
         if let Some(vsock) = &self.spec.vsock {
             let bdf: pci::Bdf = vsock.spec.pci_path.into();
 
             let mappings = vec![VsockPortMapping::new(
-                ATTESTATION_PORT.into(),
-                ATTESTATION_ADDR,
+                attestation::ATTESTATION_PORT.into(),
+                attestation::ATTESTATION_ADDR,
             )];
 
             let guest_cid = GuestCid::try_from(vsock.spec.guest_cid)
@@ -516,9 +520,20 @@ impl MachineInitializer<'_> {
 
             self.devices.insert(vsock.id.clone(), device.clone());
             chipset.pci_attach(bdf, device);
+
+            // Spawn attestation server that will go over the vsock
+            if let Some(cfg) = attest_cfg {
+                let (attest, attest_init) = AttestationSock::new(
+                    self.log.new(slog::o!("component" => "attestation-server")),
+                    cfg.sled_agent_addr,
+                )
+                .await
+                .map_err(MachineInitError::AttestationServer)?;
+                return Ok((Some(attest), Some(attest_init)));
+            }
         }
 
-        Ok(())
+        Ok((None, None))
     }
 
     async fn create_storage_backend_from_spec(
@@ -672,6 +687,85 @@ impl MachineInitializer<'_> {
         }
     }
 
+    /// Collect the necessary information out of the VM under construction into the provided
+    /// `AttestationSocketInit`. This is expected to populate `attest_init` with information so the
+    /// caller can spawn off `AttestationSockInit::run`.
+    pub fn prepare_rot_initializer(
+        &self,
+        attest_init: &mut AttestationSockInit,
+    ) {
+        let uuid = self.properties.id;
+
+        attest_init.instance_uuid = Some(uuid);
+
+        // The first boot entry is a key into `self.spec.disks`, which is how we'll get to a
+        // Crucible volume backing this boot option.
+        //
+        // TODO: remove this, but for reference:
+        // > if let Some(spec) = self.spec.disks.et(&boot_entry.device_id)
+        let boot_disk_entry = self.spec.boot_settings.as_ref()
+            .and_then(|settings| {
+                if settings.order.len() >= 2 {
+                    // In a rack we only configure propolis-server with zero or one boot disks.
+                    // It's possible to provide a fuller list, and in the future the product may
+                    // actually expose such a capability. At that time, we'll need to have a
+                    // reckoning for what "boot disk measurement" from the RoT actually means; it
+                    // probably "should" be "the measurement of the disk that EDK2 decided to boot
+                    // into", but that communication to and from the guest is a little more
+                    // complicated than we want or need to build out today.
+                    //
+                    // Since as the system exists we either have no specific boot disk (and don't
+                    // know where the guest is expected to end up), or one boot disk (and can
+                    // determine which disk to collect a measurement of before even running guest
+                    // firmware), we encode this expectation up front. If the product has changed
+                    // such that this assert is reached, "that's exciting!" and "sorry for crashing
+                    // your Propolis".
+                    panic!("Unsupported VM RoT configuration: more than one boot disk");
+                }
+
+                settings.order.first()
+            });
+
+        if let Some(boot_entry) = boot_disk_entry {
+            let disk_entry = self.spec.disks.get(&boot_entry.device_id)
+                .expect("TODO: crosscheck against boot config stuff: boot entry is valid");
+
+            let backend_id = match &disk_entry.device_spec {
+                spec::StorageDevice::Virtio(disk) => &disk.backend_id,
+                spec::StorageDevice::Nvme(disk) => &disk.backend_id,
+            };
+
+            let volume = match self.block_backends.get(backend_id) {
+                Some(block_backend) => {
+                    let crucible_backend = match block_backend
+                        .as_any()
+                        .downcast_ref::<block::CrucibleBackend>(
+                    ) {
+                        Some(backend) => backend,
+                        None => {
+                            // Probably fine, just not handled right now.
+                            slog::error!(
+                                self.log,
+                                "boot disk is not a Crucible volume"
+                            );
+                            return;
+                        }
+                    };
+                    crucible_backend.clone_volume()
+                }
+                None => {
+                    slog::error!(
+                        self.log,
+                        "boot disk does not name a block backend?!"
+                    );
+                    return;
+                }
+            };
+
+            attest_init.volume_ref = Some(volume);
+        }
+    }
+
     /// Initializes the storage devices and backends listed in this
     /// initializer's instance spec.
     ///

bin/propolis-server/src/lib/server.rs

@@ -36,6 +36,7 @@ use internal_dns_resolver::{ResolveError, Resolver};
 use internal_dns_types::names::ServiceName;
 pub use nexus_client::Client as NexusClient;
 use oximeter::types::ProducerRegistry;
+use propolis::attestation::server::AttestationServerConfig;
 use propolis_api_types::disk::{
     InstanceVCRReplace, SnapshotRequestPathParams, VCRRequestPathParams,
     VolumeStatus, VolumeStatusPathParams,
@@ -95,6 +96,9 @@ pub struct StaticConfig {
     /// The configuration to use when setting up this server's Oximeter
     /// endpoint.
     metrics: Option<MetricsEndpointConfig>,
+
+    /// TODO: comment
+    attest_config: Option<AttestationServerConfig>,
 }
 
 /// Context accessible from HTTP callbacks.
@@ -113,6 +117,7 @@ impl DropshotEndpointContext {
         use_reservoir: bool,
         log: slog::Logger,
         metric_config: Option<MetricsEndpointConfig>,
+        attest_config: Option<AttestationServerConfig>,
     ) -> Self {
         let vnc_server = VncServer::new(log.clone());
         Self {
@@ -121,6 +126,7 @@ impl DropshotEndpointContext {
                 bootrom_version,
                 use_reservoir,
                 metrics: metric_config,
+                attest_config,
             },
             vnc_server,
             vm: crate::vm::Vm::new(&log),
@@ -245,6 +251,7 @@ impl PropolisServerApi for PropolisServerImpl {
             nexus_client,
             vnc_server: server_context.vnc_server.clone(),
             local_server_addr: rqctx.server.local_addr,
+            attest_config: server_context.static_config.attest_config,
         };
 
         let vm_init = match init {

bin/propolis-server/src/lib/vm/ensure.rs

@@ -563,7 +563,8 @@ async fn initialize_vm_objects(
         &properties,
     ))?;
     init.initialize_network_devices(&chipset).await?;
-    init.initialize_vsock(&chipset)?;
+    let (tcp_attest, attest_init) =
+        init.initialize_vsock(&chipset, options.attest_config).await?;
 
     #[cfg(feature = "failure-injection")]
     init.initialize_test_devices();
@@ -578,6 +579,16 @@ async fn initialize_vm_objects(
         .initialize_storage_devices(&chipset, options.nexus_client.clone())
         .await?;
 
+    // If we have a VM RoT, that RoT needs to be able to collect some
+    // information about the guest before it can be actually usable. That
+    // information collection can - at the moment - happen entirely
+    // asynchronously. So, prepare the RoT initialization if necessary, then
+    // spawn it off to run independently.
+    if let Some(mut attest_init) = attest_init {
+        init.prepare_rot_initializer(&mut attest_init);
+        tokio::spawn(attest_init.run());
+    }
+
     let ramfb =
         init.initialize_fwcfg(spec.board.cpus, &options.bootrom_version)?;
 
@@ -642,6 +653,7 @@ async fn initialize_vm_objects(
         com1,
         framebuffer: Some(ramfb),
         ps2ctrl,
+        tcp_attest,
     };
 
     // Another really terrible hack. As we've found in Propolis#1008, brk()

bin/propolis-server/src/lib/vm/mod.rs

@@ -100,6 +100,7 @@ use state_publisher::StatePublisher;
 use tokio::sync::{oneshot, watch, RwLock, RwLockReadGuard};
 
 use crate::{server::MetricsEndpointConfig, spec::Spec, vnc::VncServer};
+use propolis::attestation::server::AttestationServerConfig;
 
 mod active;
 pub(crate) mod ensure;
@@ -309,6 +310,8 @@ pub(super) struct EnsureOptions {
     /// The address of this Propolis process, used by the live migration
     /// protocol to transfer serial console connections.
     pub(super) local_server_addr: SocketAddr,
+
+    pub(super) attest_config: Option<AttestationServerConfig>,
 }
 
 impl Vm {

bin/propolis-server/src/lib/vm/objects.rs

@@ -13,6 +13,7 @@ use std::{
 
 use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt};
 use propolis::{
+    attestation,
     hw::{ps2::ctrl::PS2Ctrl, qemu::ramfb::RamFb, uart::LpcUart},
     vmm::VmmHdl,
     Machine,
@@ -51,6 +52,7 @@ pub(super) struct InputVmObjects {
     pub com1: Arc<Serial<LpcUart>>,
     pub framebuffer: Option<Arc<RamFb>>,
     pub ps2ctrl: Arc<PS2Ctrl>,
+    pub tcp_attest: Option<attestation::server::AttestationSock>,
 }
 
 /// The collection of objects and state that make up a Propolis instance.
@@ -86,6 +88,13 @@ pub(crate) struct VmObjectsLocked {
 
     /// A handle to the VM's PS/2 controller.
     ps2ctrl: Arc<PS2Ctrl>,
+
+    /// Attestation server.
+    //
+    // This is held here only to keep the attestation server *somewhere*, but
+    // it's never used after being spawned.
+    #[allow(dead_code)]
+    tcp_attest: Option<attestation::server::AttestationSock>,
 }
 
 impl VmObjects {
@@ -126,6 +135,7 @@ impl VmObjectsLocked {
             com1: input.com1,
             framebuffer: input.framebuffer,
             ps2ctrl: input.ps2ctrl,
+            tcp_attest: input.tcp_attest,
         }
     }