From 07cbcf658b7ac7140ee403ad322f631735c63836 Mon Sep 17 00:00:00 2001 From: Matt Suiche Date: Thu, 22 Oct 2026 15:43:38 -0700 Subject: [PATCH] preparation: tolerate optional restored/rejected candidate fields on results RecordedPreparationSession's tuning report sums result.restored_candidates and result.rejected_candidates, but the flashinfer PreparationResult those results come from provides neither (its __init__ signature: plans, coverage, benchmark_calls, benchmark_closers, cache_hits, benchmarked_candidates, elapsed_seconds, compilation, parent_cute_compilations, parent_triton_compilations, overlapped_benchmark_samples, program_counts). Every multi-rank preparation crashes with AttributeError: 'PreparationResult' object has no attribute 'restored_candidates'. Did you mean 'benchmarked_candidates'? the moment the report is built - observed live on a 4x DGX Spark TP4 ring export, ~70 min into b12x autotune, killing the whole rank group. The fields are report-only; read them with getattr and default 0 so a flashinfer that starts publishing them is picked up unchanged, and one that does not (current) reports 0 instead of aborting the preparation. Validated: 4-rank export on 4x DGX Spark (GB10) proceeds past the report build that previously crashed the group. --- ignition/preparation/tuning.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ignition/preparation/tuning.py b/ignition/preparation/tuning.py index dbb8d194..fd236e37 100644 --- a/ignition/preparation/tuning.py +++ b/ignition/preparation/tuning.py @@ -357,8 +357,10 @@ def update(status): "race_budget_bytes": self.race_budget, "headroom_bytes": self.headroom_bytes, "cache_hits": sum(result.cache_hits for result in results), "benchmarked_candidates": sum(result.benchmarked_candidates for result in results), - "restored_candidates": sum(result.restored_candidates for result in results), - "rejected_candidates": sum(result.rejected_candidates for result in results), + "restored_candidates": sum(getattr(result, "restored_candidates", 0) + for result in results), + "rejected_candidates": sum(getattr(result, "rejected_candidates", 0) + for result in results), "elapsed_seconds": sum(result.elapsed_seconds for result in results), "parent_cute_compilations": sum(result.parent_cute_compilations for result in results), "parent_triton_compilations": sum(result.parent_triton_compilations for result in results),