diff --git a/Guide/src/dev_guide/tests/perf.md b/Guide/src/dev_guide/tests/perf.md index 21def8cc7c..3f6375280b 100644 --- a/Guide/src/dev_guide/tests/perf.md +++ b/Guide/src/dev_guide/tests/perf.md @@ -130,6 +130,27 @@ By default a RAM-backed disk is used to isolate virtio/storvsc overhead without host filesystem noise. Pass `--data-disk` with a path on fast storage (e.g., NVMe) for end-to-end latency measurements. +### Virtio-fs + +Measures virtio-fs file I/O throughput (MiB/s) and IOPS using fio in a linux_direct VM with an erofs tool image and a host-backed virtio-fs mount. Includes both single-thread and parallel (4-job) random I/O workloads to exercise multi-queue behavior: + +```bash +burette run --test virtio-fs -o virtiofs.json + +# Custom test file size (default 512 MiB) +burette run --test virtio-fs --virtiofs-file-size-mib 1024 -o virtiofs.json +``` + +Reported metrics: + +- `fio_virtiofs_seq_read_bw` / `fio_virtiofs_seq_write_bw` — sequential bandwidth (MiB/s, 128k blocks) +- `fio_virtiofs_rand_read_bw` / `fio_virtiofs_rand_write_bw` — random bandwidth (MiB/s, 4k blocks) +- `fio_virtiofs_rand_read_iops` / `fio_virtiofs_rand_write_iops` — random IOPS (4k blocks) +- `fio_virtiofs_rand_read_par4_bw` / `fio_virtiofs_rand_write_par4_bw` — parallel random bandwidth (4 jobs) +- `fio_virtiofs_rand_read_par4_iops` / `fio_virtiofs_rand_write_par4_iops` — parallel random IOPS (4 jobs) + +Uses `--direct=0` (FUSE does not support O_DIRECT) with explicit page cache invalidation before each workload to ensure I/O hits the FUSE path. + ## Comparing Reports ```bash diff --git a/petri/burette/src/tests/common.rs b/petri/burette/src/tests/common.rs new file mode 100644 index 0000000000..895973a376 --- /dev/null +++ b/petri/burette/src/tests/common.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Shared helpers used by multiple performance tests. + +use petri_artifacts_common::tags::MachineArch; + +/// Build the default firmware (linux_direct) for the host architecture. +pub fn build_firmware(resolver: &petri::ArtifactResolver<'_>) -> petri::Firmware { + petri::Firmware::linux_direct(resolver, MachineArch::host()) +} + +/// Resolve the petritools erofs image for the host architecture. +pub fn require_petritools_erofs( + resolver: &petri::ArtifactResolver<'_>, +) -> petri_artifacts_core::ResolvedArtifact { + use petri_artifacts_vmm_test::artifacts::petritools::*; + match MachineArch::host() { + MachineArch::X86_64 => resolver.require(PETRITOOLS_EROFS_X64).erase(), + MachineArch::Aarch64 => resolver.require(PETRITOOLS_EROFS_AARCH64).erase(), + } +} diff --git a/petri/burette/src/tests/disk_io.rs b/petri/burette/src/tests/disk_io.rs index 58115823d9..d9652f893f 100644 --- a/petri/burette/src/tests/disk_io.rs +++ b/petri/burette/src/tests/disk_io.rs @@ -12,6 +12,7 @@ //! Supports both virtio-blk and storvsc (synthetic SCSI) disk backends. use crate::report::MetricResult; +use crate::tests::common; use anyhow::Context as _; use petri::pipette::cmd; use petri_artifacts_common::tags::MachineArch; @@ -59,30 +60,16 @@ pub struct DiskIoTestState { disk_device: String, } -fn build_firmware(resolver: &petri::ArtifactResolver<'_>) -> petri::Firmware { - petri::Firmware::linux_direct(resolver, MachineArch::host()) -} - -fn require_petritools_erofs( - resolver: &petri::ArtifactResolver<'_>, -) -> petri_artifacts_core::ResolvedArtifact { - use petri_artifacts_vmm_test::artifacts::petritools::*; - match MachineArch::host() { - MachineArch::X86_64 => resolver.require(PETRITOOLS_EROFS_X64).erase(), - MachineArch::Aarch64 => resolver.require(PETRITOOLS_EROFS_AARCH64).erase(), - } -} - /// Register artifacts needed by the disk I/O test. pub fn register_artifacts(resolver: &petri::ArtifactResolver<'_>) { - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); petri::PetriVmArtifacts::::new( resolver, firmware, MachineArch::host(), true, ); - require_petritools_erofs(resolver); + common::require_petritools_erofs(resolver); } /// GUID for the data disk SCSI controller (used for storvsc backend). @@ -135,7 +122,7 @@ impl crate::harness::WarmPerfTest for DiskIoTest { ); } - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); let artifacts = petri::PetriVmArtifacts::::new( resolver, @@ -154,7 +141,7 @@ impl crate::harness::WarmPerfTest for DiskIoTest { }; // Open the perf rootfs erofs image for the virtio-blk device. - let erofs_path = require_petritools_erofs(resolver); + let erofs_path = common::require_petritools_erofs(resolver); let erofs_file = fs_err::File::open(&erofs_path)?; let mut builder = petri::PetriVmBuilder::minimal(params, artifacts, driver)? diff --git a/petri/burette/src/tests/mod.rs b/petri/burette/src/tests/mod.rs index 8c6d14c2a4..76f4b852a9 100644 --- a/petri/burette/src/tests/mod.rs +++ b/petri/burette/src/tests/mod.rs @@ -4,6 +4,7 @@ //! Performance tests for OpenVMM. pub mod boot_time; +pub mod common; pub mod disk_io; pub mod memory; pub mod network; diff --git a/petri/burette/src/tests/network.rs b/petri/burette/src/tests/network.rs index b3a2bd717d..b9734b1b96 100644 --- a/petri/burette/src/tests/network.rs +++ b/petri/burette/src/tests/network.rs @@ -17,6 +17,7 @@ //! the NIC frontend, selected via the `--nic` flag. use crate::report::MetricResult; +use crate::tests::common; use anyhow::Context as _; use petri::pipette::cmd; @@ -74,30 +75,16 @@ pub struct NetworkTestState { driver: pal_async::DefaultDriver, } -fn build_firmware(resolver: &petri::ArtifactResolver<'_>) -> petri::Firmware { - petri::Firmware::linux_direct(resolver, MachineArch::host()) -} - -fn require_petritools_erofs( - resolver: &petri::ArtifactResolver<'_>, -) -> petri_artifacts_core::ResolvedArtifact { - use petri_artifacts_vmm_test::artifacts::petritools::*; - match MachineArch::host() { - MachineArch::X86_64 => resolver.require(PETRITOOLS_EROFS_X64).erase(), - MachineArch::Aarch64 => resolver.require(PETRITOOLS_EROFS_AARCH64).erase(), - } -} - /// Register artifacts needed by the network test. pub fn register_artifacts(resolver: &petri::ArtifactResolver<'_>) { - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); petri::PetriVmArtifacts::::new( resolver, firmware, MachineArch::host(), true, ); - require_petritools_erofs(resolver); + common::require_petritools_erofs(resolver); } impl crate::harness::WarmPerfTest for NetworkTest { @@ -182,7 +169,7 @@ impl crate::harness::WarmPerfTest for NetworkTest { None }; - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); let artifacts = petri::PetriVmArtifacts::::new( resolver, firmware, @@ -204,7 +191,7 @@ impl crate::harness::WarmPerfTest for NetworkTest { }; // Open the perf rootfs erofs image for the virtio-blk device. - let erofs_path = require_petritools_erofs(resolver); + let erofs_path = common::require_petritools_erofs(resolver); let erofs_file = fs_err::File::open(&erofs_path)?; let mut builder = petri::PetriVmBuilder::minimal(params, artifacts, driver)? diff --git a/petri/burette/src/tests/virtio_fs.rs b/petri/burette/src/tests/virtio_fs.rs index 54745daa0d..1bcd8ae318 100644 --- a/petri/burette/src/tests/virtio_fs.rs +++ b/petri/burette/src/tests/virtio_fs.rs @@ -22,6 +22,7 @@ //! reads/writes `/tmp/vfs/test.dat`. use crate::report::MetricResult; +use crate::tests::common; use anyhow::Context as _; use petri::pipette::cmd; use petri_artifacts_common::tags::MachineArch; @@ -67,30 +68,16 @@ pub struct VirtioFsTestState { _vfs_root: tempfile::TempDir, } -fn build_firmware(resolver: &petri::ArtifactResolver<'_>) -> petri::Firmware { - petri::Firmware::linux_direct(resolver, MachineArch::host()) -} - -fn require_petritools_erofs( - resolver: &petri::ArtifactResolver<'_>, -) -> petri_artifacts_core::ResolvedArtifact { - use petri_artifacts_vmm_test::artifacts::petritools::*; - match MachineArch::host() { - MachineArch::X86_64 => resolver.require(PETRITOOLS_EROFS_X64).erase(), - MachineArch::Aarch64 => resolver.require(PETRITOOLS_EROFS_AARCH64).erase(), - } -} - /// Register artifacts needed by the virtio-fs test. pub fn register_artifacts(resolver: &petri::ArtifactResolver<'_>) { - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); petri::PetriVmArtifacts::::new( resolver, firmware, MachineArch::host(), true, ); - require_petritools_erofs(resolver); + common::require_petritools_erofs(resolver); } impl crate::harness::WarmPerfTest for VirtioFsTest { @@ -114,7 +101,7 @@ impl crate::harness::WarmPerfTest for VirtioFsTest { "file_size_mib must be greater than 0" ); - let firmware = build_firmware(resolver); + let firmware = common::build_firmware(resolver); let artifacts = petri::PetriVmArtifacts::::new( resolver, @@ -133,7 +120,7 @@ impl crate::harness::WarmPerfTest for VirtioFsTest { }; // Open the perf rootfs erofs image for the virtio-blk device (carries fio). - let erofs_path = require_petritools_erofs(resolver); + let erofs_path = common::require_petritools_erofs(resolver); let erofs_file = fs_err::File::open(&erofs_path)?; // Host directory backing the virtio-fs mount. @@ -254,15 +241,18 @@ impl crate::harness::WarmPerfTest for VirtioFsTest { // cold against the FUSE path rather than warming the page cache). // Sequential tests use 128k blocks to exercise zero-copy and // max-pages; random tests use 4k blocks for IOPS measurement. - let fio_jobs: &[(&str, &str, &str)] = &[ - // (fio_rw_mode, primary_field, block_size) - ("read", "read", "128k"), - ("write", "write", "128k"), - ("randread", "read", "4k"), - ("randwrite", "write", "4k"), + let fio_jobs: &[(&str, &str, &str, u32)] = &[ + // (fio_rw_mode, primary_field, block_size, numjobs) + ("read", "read", "128k", 1), + ("write", "write", "128k", 1), + ("randread", "read", "4k", 1), + ("randwrite", "write", "4k", 1), + // Parallel I/O tests (numjobs=4) to exercise multi-queue. + ("randread", "read", "4k", 4), + ("randwrite", "write", "4k", 4), ]; - for &(rw_mode, field, bs) in fio_jobs { + for &(rw_mode, field, bs, numjobs) in fio_jobs { let is_random = rw_mode.starts_with("rand"); let phase = if is_random { rw_mode.strip_prefix("rand").unwrap() @@ -270,21 +260,30 @@ impl crate::harness::WarmPerfTest for VirtioFsTest { rw_mode }; let prefix = if is_random { "rand" } else { "seq" }; + let par_suffix = if numjobs > 1 { + format!("_par{numjobs}") + } else { + String::new() + }; - let perf_label = format!("fio_virtiofs_{prefix}_{phase}"); + // Drop guest page caches before starting the perf recorder so + // the trace captures only the fio workload. + drop_guest_caches(&state.agent).await?; + + let perf_label = format!("fio_virtiofs_{prefix}_{phase}{par_suffix}"); recorder.start(&perf_label)?; - let json = run_fio_job(&state.agent, rw_mode, bs, size_mib) + let json = run_fio_job(&state.agent, rw_mode, bs, size_mib, numjobs) .await - .with_context(|| format!("fio {rw_mode} failed"))?; + .with_context(|| format!("fio {rw_mode} numjobs={numjobs} failed"))?; recorder.stop()?; - let bw_name = format!("fio_virtiofs_{prefix}_{phase}_bw"); + let bw_name = format!("fio_virtiofs_{prefix}_{phase}{par_suffix}_bw"); metrics.push(parse_fio_bw(&json, &bw_name, field)?); if is_random { - let iops_name = format!("fio_virtiofs_{prefix}_{phase}_iops"); + let iops_name = format!("fio_virtiofs_{prefix}_{phase}{par_suffix}_iops"); metrics.push(parse_fio_iops(&json, &iops_name, field)?); } } @@ -299,11 +298,23 @@ impl crate::harness::WarmPerfTest for VirtioFsTest { } } +/// Flush dirty pages and drop guest page caches so reads exercise the full +/// FUSE request path rather than being served from guest RAM. +async fn drop_guest_caches(agent: &petri::pipette::PipetteClient) -> anyhow::Result<()> { + let sh = agent.unix_shell(); + let script = "sync && echo 3 > /proc/sys/vm/drop_caches"; + cmd!(sh, "sh -c {script}") + .read() + .await + .context("failed to sync and drop guest page caches")?; + Ok(()) +} + /// Run a single fio job against the virtio-fs test file and return the raw /// JSON output. /// -/// Before each job we invalidate the guest page cache so that reads exercise -/// the full FUSE request path rather than being served from guest RAM. +/// The caller is responsible for dropping guest page caches before calling +/// this function (see `drop_guest_caches`). /// `ramp_time=0` ensures the measurement window starts cold; the harness's /// warmup iteration handles VM-level warm-up. `--end_fsync=1` flushes /// buffered writes through FUSE before fio reports results. @@ -317,24 +328,16 @@ async fn run_fio_job( rw_mode: &str, bs: &str, size_mib: u64, + numjobs: u32, ) -> anyhow::Result { - // Flush dirty pages then drop guest page caches so reads go through - // the FUSE path. `sync` first ensures writeback is complete, making - // cache state deterministic between fio jobs. - let drop_sh = agent.unix_shell(); - let drop_script = "sync; echo 3 > /proc/sys/vm/drop_caches"; - cmd!(drop_sh, "sh -c {drop_script}") - .read() - .await - .context("failed to drop guest page caches")?; - let mut sh = agent.unix_shell(); sh.chroot("/perf"); let size_arg = format!("{size_mib}M"); - let output: String = cmd!(sh, "fio --name=test --filename=/tmp/vfs/test.dat --rw={rw_mode} --bs={bs} --ioengine=io_uring --direct=0 --runtime=10 --time_based=1 --ramp_time=0 --iodepth=32 --numjobs=1 --size={size_arg} --invalidate=1 --end_fsync=1 --output-format=json") + let numjobs_arg = numjobs.to_string(); + let output: String = cmd!(sh, "fio --name=test --filename=/tmp/vfs/test.dat --rw={rw_mode} --bs={bs} --ioengine=io_uring --direct=0 --runtime=10 --ramp_time=0 --iodepth=32 --numjobs={numjobs_arg} --size={size_arg} --invalidate=1 --end_fsync=1 --output-format=json") .read() .await - .with_context(|| format!("fio {rw_mode} on virtio-fs failed"))?; + .with_context(|| format!("fio {rw_mode} numjobs={numjobs} on virtio-fs failed"))?; Ok(output) } @@ -343,12 +346,15 @@ async fn run_fio_job( fn parse_fio_bw(json: &str, metric_name: &str, field: &str) -> anyhow::Result { let v: serde_json::Value = serde_json::from_str(json).context("failed to parse fio JSON")?; - let bw_bytes = v["jobs"][0][field]["bw_bytes"].as_f64().with_context(|| { - tracing::error!(json = %json, "failed to find {field}.bw_bytes in fio output"); - format!("missing {field}.bw_bytes in fio output for {metric_name}") - })?; + let jobs = v["jobs"].as_array().context("missing jobs array")?; + let mut total_bw: f64 = 0.0; + for job in jobs { + total_bw += job[field]["bw_bytes"] + .as_f64() + .with_context(|| format!("missing {field}.bw_bytes in fio output for {metric_name}"))?; + } - let mib_s = bw_bytes / (1024.0 * 1024.0); + let mib_s = total_bw / (1024.0 * 1024.0); Ok(MetricResult { name: metric_name.to_string(), unit: "MiB/s".to_string(), @@ -360,11 +366,15 @@ fn parse_fio_bw(json: &str, metric_name: &str, field: &str) -> anyhow::Result anyhow::Result { let v: serde_json::Value = serde_json::from_str(json).context("failed to parse fio JSON")?; - let iops = v["jobs"][0][field]["iops"].as_f64().with_context(|| { - tracing::error!(json = %json, "failed to find {field}.iops in fio output"); - format!("missing {field}.iops in fio output for {metric_name}") - })?; + let jobs = v["jobs"].as_array().context("missing jobs array")?; + let mut total_iops: f64 = 0.0; + for job in jobs { + total_iops += job[field]["iops"] + .as_f64() + .with_context(|| format!("missing {field}.iops in fio output for {metric_name}"))?; + } + let iops = total_iops; Ok(MetricResult { name: metric_name.to_string(), unit: "IOPS".to_string(),