Stream verifier scheduler output, quiet VMM diagnostics, fix summary glyph

Cargo.toml

@@ -9,7 +9,7 @@ repository = "https://github.com/likewhatevs/ktstr"
 
 [package]
 name = "ktstr"
-version = "0.28.0"
+version = "0.29.0"
 edition.workspace = true
 rust-version.workspace = true
 license.workspace = true
@@ -154,7 +154,7 @@ virtio-queue = "0.17"
 vm-superio = "0.8.1"
 zerocopy = { version = "0.8.48", features = ["derive"] }
 linkme = "0.3"
-ktstr-macros = { version = "0.28.0", path = "ktstr-macros" }
+ktstr-macros = { version = "0.29.0", path = "ktstr-macros" }
 goblin = { version = "0.10.5", features = ["elf64"] }
 # Memory-map /proc/self/exe in the coverage-instrumentation detection
 # probe (`is_coverage_instrumented_binary` / `find_symbol_vaddrs` in

doc/guide/src/features.md

@@ -120,7 +120,7 @@ scx_p2dq:
  kernel        p2dq_dequeue  p2dq_dispatch  p2dq_enqueue  p2dq_exit  p2dq_exit_task  p2dq_init  p2dq_init_task  p2dq_running  p2dq_select_cpu  ...
  kernel_local  5             1159..2130     2026..5118    25         419             2121       27601           609           801..887         ...
 
-verifier summary: 8 ✅  <span class="t-red">4 ❌</span>  0 🇽
+verifier summary: 8 ✅  <span class="t-red">4 ❌</span>  0 ❎
  topology   scx_bpfland  scx_lavd  scx_p2dq
  odd-3llc   ✅           <span class="t-red">❌</span>        ✅
  smt-2llc   ✅           <span class="t-red">❌</span>        ✅

doc/guide/src/recipes/test-new-scheduler.md

@@ -131,7 +131,7 @@ ktstr_sched:
  kernel      ktstr_dispatch  ktstr_dump  ktstr_dump_cpu  ktstr_dump_task  ktstr_enqueue  ktstr_exit  ktstr_exit_task  ktstr_init  ktstr_init_task  ktstr_select_cp  ktstr_yield
  kernel_7_0  102             81          13              70               74             25          419              2296        <span class="t-yel">29077</span>            39               8
 
-<span class="t-grn">verifier summary: 4 ✅  0 ❌  0 🇽</span>
+<span class="t-grn">verifier summary: 4 ✅  0 ❌  0 ❎</span>
  topology   ktstr_sched
  odd-3llc   ✅
  smt-2llc   ✅

doc/guide/src/running-tests/verifier.md

@@ -73,7 +73,7 @@ scx_p2dq:
  kernel        p2dq_dequeue  p2dq_dispatch  p2dq_enqueue  p2dq_exit  p2dq_exit_task  p2dq_init  p2dq_init_task  p2dq_running  p2dq_select_cpu  p2dq_set_cpumas  p2dq_stopping  p2dq_update_idl
  kernel_local  5             1159..2130     2026..5118    25         419             2121       27601           609           801..887         149              853            723
 
-verifier summary: 8 ✅  <span class="t-red">4 ❌</span>  0 🇽
+verifier summary: 8 ✅  <span class="t-red">4 ❌</span>  0 ❎
  topology   scx_bpfland  scx_lavd  scx_p2dq
  odd-3llc   ✅           <span class="t-red">❌</span>        ✅
  smt-2llc   ✅           <span class="t-red">❌</span>        ✅
@@ -101,10 +101,9 @@ A cell in the `verified_insns` table shows a single number when the
 count is flat across topologies, `lo..hi` when it varies, and `-`
 when that program reported no stats on that kernel. In the grid, ✅
 means the scheduler verified, attached, and dispatched on every
-kernel that ran the cell; ❌ means it failed on every kernel; 🇽 means
-mixed results across kernels (the 🇽 glyph renders inconsistently in
-some terminal fonts — the failing-combinations list below the grid is
-the authoritative record).
+kernel that ran the cell; ❌ means it failed on every kernel; ❎ means
+mixed results across kernels (the failing-combinations list below the
+grid is the authoritative record).
 
 > [!NOTE]
 > For a scheduler outside your workspace, either declare it with
@@ -135,7 +134,7 @@ kernel list: range expanded to 3 kernel(s): 6.12.95, 6.13.12, 6.14.11
 With multiple kernels resolved, each cell runs against its own, the
 `verified_insns` table grows one row per kernel, and the pass/fail
 grid folds kernels into each cell (✅ passed on every kernel, ❌
-failed on every kernel, 🇽 mixed — the failing-combinations list is
+failed on every kernel, ❎ mixed — the failing-combinations list is
 authoritative):
 
 <!-- captured: cargo ktstr verifier --kernel 7.0.14-tarball-x86_64-kcabd40422 --kernel local-8cd2b47-x86_64-kcabd40422 --scheduler ktstr_sched --test kaslr_axis_e2e tiny-1llc tiny-2llc | ktstr 0.23.0 | kernels 7.0.14 + v7.1-patched -->
@@ -146,14 +145,14 @@ authoritative):
  kernel_7_0_14        102             81          13              70               74             25          419              2296        29077            39               8
  kernel_local_8cd2b4  102             81          13              70               74             25          419              2296        29077            39               8
 
-verifier summary: 2 ✅  0 ❌  0 🇽
+verifier summary: 2 ✅  0 ❌  0 ❎
  topology   ktstr_sched
  tiny-1llc  ✅
  tiny-2llc  ✅</pre></div>
 
 Flat rows across kernels are the boring, reassuring case — the same
 BPF verified identically on both. A kfunc or verifier change between
-kernels shows up as diverging counts, or as a 🇽 cell.
+kernels shows up as diverging counts, or as a ❎ cell.
 
 ## What a cell checks
 
@@ -229,7 +228,7 @@ Global function ktstr_dispatch() doesn't return scalar. Only those are supported
 <span class="t-red">R1 invalid mem access 'scalar'</span>
 processed 186 insns (limit 1000000) max_states_per_insn 0 total_states 7 peak_states 7 mark_read 0
 ...
-verifier summary: 0 ✅  1 ❌  0 🇽
+verifier summary: 0 ✅  1 ❌  0 ❎
  topology   ktstr_broken
  tiny-1llc  ❌
 

ktstr-macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ktstr-macros"
-version = "0.28.0"
+version = "0.29.0"
 edition.workspace = true
 rust-version.workspace = true
 license.workspace = true

src/lib.rs

@@ -1373,6 +1373,27 @@ pub const KTSTR_LOCK_DIR_ENV: &str = "KTSTR_LOCK_DIR";
 /// arch: x86_64 and aarch64); both readers identical.
 pub const KTSTR_VERBOSE_ENV: &str = "KTSTR_VERBOSE";
 
+/// Name of the environment variable that enables the VMM's verbose
+/// host-side diagnostics (per-vCPU affinity-mask lines, the BSP
+/// run-loop trace, and the `CLEANUP:` teardown timings). Value-typed:
+/// only the exact string `"1"` enables; unset / empty / any other
+/// value is disabled. Read once via
+/// `crate::vmm::debug_logging_enabled` (crate-private), which
+/// OR-combines it with [`RUNNER_DEBUG_ENV`]. Default-off keeps these
+/// lines out of normal
+/// CI logs where they otherwise bury the scheduler failure under
+/// investigation.
+pub const KTSTR_DEBUG_ENV: &str = "KTSTR_DEBUG";
+
+/// Name of the environment variable GitHub Actions exports (set to
+/// `"1"`) when a job is re-run with "Enable debug logging". Not a
+/// `KTSTR_*` variable, but honoured alongside [`KTSTR_DEBUG_ENV`] by
+/// `crate::vmm::debug_logging_enabled` (crate-private) so a failed CI
+/// job re-run in debug mode gets the full VMM diagnostics without any
+/// ktstr-specific
+/// knob. Value-typed: only `"1"` enables.
+pub const RUNNER_DEBUG_ENV: &str = "RUNNER_DEBUG";
+
 /// Name of the environment variable that bypasses LLC resource
 /// locks at scenario setup (test_support::dispatch / cargo-ktstr
 /// shell). Set by the `--bypass-llc-locks` CLI flag.

src/snapshots/ktstr__verifier__tests__snapshot_format_verifier_output_with_log.snap

@@ -11,6 +11,8 @@ llc+steal --- verifier stats ---
   processed=42  states=8/10
 
 llc+steal --- scheduler log ---
+-- BEGIN PROG LOAD LOG --
 func#0 @0
 0: R1=ctx() R10=fp0
 processed 42 insns (limit 1000000) max_states_per_insn 1 total_states 10 peak_states 8 mark_read 5
+-- END PROG LOAD LOG --

src/test_support/dispatch.rs

@@ -2216,6 +2216,14 @@ fn run_verifier_cell_inner(
         Ok(result) => {
             let output = crate::verifier::format_verifier_output("verifier", &result, raw);
             print!("{output}");
+            // Propagate the scheduler's captured stderr to the test's
+            // real stderr (best-effort — whatever streamed before a
+            // watchdog kill). Emitted BEFORE the `... FAILED:` verdict
+            // line below so the verdict stays the last thing on stderr.
+            let sched_stderr = crate::verifier::format_verifier_stderr("verifier", &result, raw);
+            if !sched_stderr.is_empty() {
+                eprint!("{sched_stderr}");
+            }
             // PASS requires verify + attach (sched_ext `enabled`) +
             // dispatch (the injected workload made progress).
             // `cell_verdict` names the first failing gate (timed_out →

src/test_support/eval/mod.rs

@@ -1440,11 +1440,16 @@ fn run_ktstr_test_inner_impl(
                 // remaining verdict-bearing variants in this arm
                 // (TestResult, Exit, SchedExit, ScenarioStart,
                 // ScenarioPause, ScenarioResume, Stdout, SchedLog,
-                // Lifecycle, ExecExit, Dmesg, ProbeOutput, SnapshotReply,
-                // Crash) are consumed by other walkers further down the
-                // pipeline (parse_assert_result_from_drain, bulk_exit
+                // SchedStdout, SchedStderr, Lifecycle, ExecExit, Dmesg,
+                // ProbeOutput, SnapshotReply, Crash) are consumed by
+                // other walkers further down the pipeline
+                // (parse_assert_result_from_drain, bulk_exit
                 // lookup in collect_results, lifecycle classifier,
-                // sched_log concatenator, etc.). No per-entry side effect
+                // sched_log concatenator, etc.). The live scheduler
+                // SchedStdout/SchedStderr streams are consumed only by
+                // the verifier cell path (`concat_sched_std*_chunks`);
+                // for a normal test VM they are diagnostic no-ops here.
+                // No per-entry side effect
                 // here. (Stderr is NOT in this arm — it has its own arm
                 // below that streams to host stderr.) WprofTrace +
                 // WprofTraceChunk are no-ops here: the #[cfg(feature="wprof")]
@@ -1468,6 +1473,8 @@ fn run_ktstr_test_inner_impl(
                     | crate::vmm::wire::MsgType::ScenarioResume
                     | crate::vmm::wire::MsgType::Stdout
                     | crate::vmm::wire::MsgType::SchedLog
+                    | crate::vmm::wire::MsgType::SchedStdout
+                    | crate::vmm::wire::MsgType::SchedStderr
                     | crate::vmm::wire::MsgType::Lifecycle
                     | crate::vmm::wire::MsgType::ExecExit
                     | crate::vmm::wire::MsgType::Dmesg

src/tests.rs

@@ -406,8 +406,8 @@ fn ktstr_env_constants_are_their_literals() {
     use super::{
         KTSTR_BUDGET_SECS_ENV, KTSTR_BUSYBOX_PATH_ENV, KTSTR_BYPASS_LLC_LOCKS_ENV,
         KTSTR_CACHE_DIR_ENV, KTSTR_CARGO_TEST_MODE_ENV, KTSTR_CGROUP_WALK_ROOT_ENV,
-        KTSTR_CONTENTION_BYPASS_ENV, KTSTR_CPU_CAP_ENV, KTSTR_GHA_CACHE_ENV, KTSTR_GUEST_INIT_ENV,
-        KTSTR_HOST_CGROUP_PARENT_ENV, KTSTR_JEMALLOC_ALLOC_WORKER_BINARY_ENV,
+        KTSTR_CONTENTION_BYPASS_ENV, KTSTR_CPU_CAP_ENV, KTSTR_DEBUG_ENV, KTSTR_GHA_CACHE_ENV,
+        KTSTR_GUEST_INIT_ENV, KTSTR_HOST_CGROUP_PARENT_ENV, KTSTR_JEMALLOC_ALLOC_WORKER_BINARY_ENV,
         KTSTR_JEMALLOC_PROBE_BINARY_ENV, KTSTR_KERNEL_ENV, KTSTR_KERNEL_LIST_ENV,
         KTSTR_KERNEL_PARALLELISM_ENV, KTSTR_LOCK_DIR_ENV, KTSTR_LOG_PASSES_ENV,
         KTSTR_NO_PERF_MODE_ENV, KTSTR_NO_SKIP_MODE_ENV, KTSTR_ORCHESTRATED_ENV,
@@ -429,6 +429,8 @@ fn ktstr_env_constants_are_their_literals() {
     assert_eq!(KTSTR_CACHE_DIR_ENV, "KTSTR_CACHE_DIR");
     assert_eq!(KTSTR_LOCK_DIR_ENV, "KTSTR_LOCK_DIR");
     assert_eq!(KTSTR_VERBOSE_ENV, "KTSTR_VERBOSE");
+    assert_eq!(KTSTR_DEBUG_ENV, "KTSTR_DEBUG");
+    assert_eq!(super::RUNNER_DEBUG_ENV, "RUNNER_DEBUG");
     assert_eq!(KTSTR_BYPASS_LLC_LOCKS_ENV, "KTSTR_BYPASS_LLC_LOCKS");
     assert_eq!(KTSTR_CPU_CAP_ENV, "KTSTR_CPU_CAP");
     assert_eq!(KTSTR_CONTENTION_BYPASS_ENV, "KTSTR_CONTENTION_BYPASS");