diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 2198628bc..ffdcfb3d5 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -108,6 +108,10 @@ jobs: run: | coverage run $COV_ARGS -m pytest mpisppy/tests/test_generic_cylinders.py -v + - name: Test xhat from file + run: | + coverage run $COV_ARGS -m pytest mpisppy/tests/test_xhat_from_file.py -v + - name: Test docs run: | cd ./doc/ diff --git a/doc/src/index.rst b/doc/src/index.rst index 701a8067d..448ee83b5 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -53,6 +53,7 @@ MPI is used. properbundles.rst pickling.rst + xhat_from_file.rst smps.rst agnostic.rst generic_admm.rst diff --git a/doc/src/xhat_from_file.rst b/doc/src/xhat_from_file.rst new file mode 100644 index 000000000..eac061c53 --- /dev/null +++ b/doc/src/xhat_from_file.rst @@ -0,0 +1,141 @@ +.. _xhat_from_file: + +Supplying an Initial Xhat from a File +===================================== + +Every xhat spoke (``xhatlooper``, ``xhatshufflelooper``, +``xhatspecific``, ``xhatxbar``) will optionally read a first-stage +solution ``xhat`` from a ``.npy`` file, evaluate it across all +scenarios once, and report the resulting inner bound — **before** its +normal exploration loop starts. + +When This is Useful +------------------- + +- **Warm start from a prior run on a similar instance.** If you have + already solved a related instance (same first-stage structure, same + nonant order), the previous run's xhat is often a good starting + candidate for the current run. Feed it in and the xhatter reports + that inner bound immediately instead of waiting for normal + exploration to stumble onto something comparable. This is the + most common use case. +- **User-supplied heuristic candidate.** If your domain knowledge or + a hand-computed heuristic gives you a promising ``xhat``, supplying + it as the first thing the xhatter evaluates often shortens the time + to a useful inner bound. +- **Testing infeasibility-driven features.** Combined with + the xhat feasibility-cuts feature (PR #671), you can write a known-infeasible + ``xhat`` to a ``.npy`` file and hand it in; the feasibility-cut + path then fires end-to-end, letting you verify that the same xhat + is not revisited on the next iteration. + +Enabling the Feature +-------------------- + +``generic_cylinders`` exposes a single string flag: + +.. code-block:: bash + + --xhat-from-file + +where ```` points at a ``.npy`` file whose contents is a +one-dimensional numpy array holding the first-stage values **in the +same order as the problem's root-node nonant list**. Order-sensitive +— the ordinary pyomo iteration order over +``scenario._mpisppy_node_list[0].nonant_vardata_list`` for any local +scenario. (If you generated the file from a previous mpi-sppy run, +the order matches automatically.) + +The flag is off by default; the feature is only active when the flag +is supplied. + +File Format +----------- + +``.npy`` only, via the existing ``mpisppy.confidence_intervals.ciutils.read_xhat`` +helper. That is the canonical mpi-sppy xhat on-disk format: the +MMW confidence-interval code already uses it +(``--mmw-xhat-input-file-name``), and several examples write xhats +this way (``ciutils.write_xhat``). + +To produce a compatible file from a script: + +.. code-block:: python + + import numpy as np + xhat_values = [1.0, 0.0, 1.0] # in nonant order + np.save("my_xhat.npy", np.array(xhat_values, dtype=float)) + +Example +------- + +.. code-block:: bash + + python -m mpisppy.generic_cylinders \ + --module-name my_model \ + --num-scens 10 \ + --solver-name gurobi \ + --max-iterations 50 \ + --default-rho 1.0 \ + --lagrangian --xhatshuffle \ + --xhat-from-file prior_run_xhat.npy + +Each rank reads the file once, the xhat spoke evaluates it, and the +resulting inner bound (if finite) is sent to the hub before the +spoke starts its normal shuffle loop. + +Scope and Limitations +--------------------- + +**Two-stage only.** V1 supports two-stage problems only, matching +``ciutils.read_xhat``. Multi-stage is planned as a follow-up; for +now, enabling the flag on a multi-stage run raises + +.. code-block:: text + + RuntimeError: --xhat-from-file is two-stage only; multi-stage + support is planned as a follow-up. + +**Length must match.** The file's vector length must equal the +problem's root-node nonant count. A mismatch raises at spoke startup +with an error naming both counts — no silent truncation or padding. + +**Missing file is a hard error.** The path must exist when the spoke +starts; a missing file is not silently treated as "feature off". + +Interaction with ``--*-try-jensens-first`` +------------------------------------------ + +Both ``--xhat-from-file`` and Jensen's ``--*-try-jensens-first`` (see +the Jensen's-bound docs) contribute a single candidate xhat before +the xhatter's normal loop. They can be used together. The explicit +file-supplied candidate is evaluated **first**, then Jensen's, then +the normal exploration. ``update_if_improving`` keeps whichever is +best, so correctness does not depend on order; the ordering is a +predictability and log-readability choice. + +Interaction with ``--xhat-feasibility-cuts-count`` +-------------------------------------------------- + +If both flags are on and the file-supplied xhat turns out to be +infeasible in some scenario, the xhatter's infeasibility path fires +and a feasibility cut is emitted (see the xhat feasibility-cuts feature (PR #671) +for the mechanics). The hub installs the cut into every scenario, +and the same xhat is not revisited. This combination is the +recommended way to exercise feasibility cuts in an end-to-end test: +hand in a known-infeasible binary vector via ``--xhat-from-file`` +with ``--xhat-feasibility-cuts-count=1`` and watch the cut land. + +Follow-up Milestones +-------------------- + +- Multi-stage support (per-node xhat file or a multi-node format). +- Additional file formats (CSV, JSON) if a concrete use case appears. + +See Also +-------- + +- :ref:`Spokes` — overview of the xhat spokes. +- the xhat feasibility-cuts feature (PR #671) — the companion feature for + non-complete-recourse problems. +- ``doc/xhat_from_file_design.md`` — the design document. diff --git a/doc/xhat_from_file_design.md b/doc/xhat_from_file_design.md new file mode 100644 index 000000000..47c60a69a --- /dev/null +++ b/doc/xhat_from_file_design.md @@ -0,0 +1,249 @@ +# Design: Supply an Initial Xhat from a File + +**Status:** Draft — up for discussion. Nothing is implemented yet. +**Author:** dlw (captured with Claude Code assistance) +**Last updated:** 2026-04-23 + +## Motivation + +Two independent uses call for the same mechanism: + +1. **Warm start / user-supplied candidate.** A user often has a first-stage + solution they want to try first — from a prior run, a hand-computed + heuristic, a rolling-horizon neighbor, or a solution transferred from + a different but related instance. Today, the only way to inject an + `xhat` into a running cylinder system is to modify code; there is no + CLI surface for it. +2. **Testing infeasibility-driven features.** The new xhat feasibility + cuts (PR #671, issue #601) install a cut when an xhatter finds an + infeasible `xhat`. End-to-end testing that path currently requires + engineering a scenario that happens to be infeasible for *some* + `xhat` the xhatter will naturally propose. A supply-your-own-`xhat` + flag makes it trivial: write the known-infeasible binary vector to + a file and hand it in. + +The same file-read path serves both use cases — so ship it once. + +## Non-goals + +- Persistent xhat swapping mid-run. The file is read once at spoke + startup; the xhatter evaluates it, reports the inner bound (or + infeasibility), and then continues with its normal exploration. No + re-read mechanism. +- A cross-language data exchange format. We use what mpi-sppy already + uses. +- Multi-stage. V1 is two-stage-only (matches the current `.npy` + reader's scope); multi-stage is a named follow-up. + +## Related work in the tree + +- **`mpisppy/confidence_intervals/ciutils.py`** has + `read_xhat(path, num_stages=2)` and `write_xhat(xhat, path, + num_stages=2)`. Both only handle `num_stages=2` today; both use + `numpy.save`/`numpy.load` on the `xhat['ROOT']` vector. The MMW + path uses these via `--mmw-xhat-input-file-name`. +- **Jensen's bound design** (PR #657, + `doc/jensens_bound_design.md`) adds an opt-in `--*-try-jensens-first` + flag to each xhatter. On startup, the xhatter builds an average + scenario, solves it, uses the first-stage solution as an xhat, + evaluates it across all scenarios, and reports the resulting inner + bound — then continues with normal iteration. Our new feature + wants to do exactly the same thing with a file-supplied xhat + instead of an average-scenario-derived one. **Precedence with + Jensen's is the main new design decision; see §4.** +- **`Xhat_Eval.evaluate(nonant_cache)`** + (`mpisppy/utils/xhat_eval.py::Xhat_Eval.evaluate`) already does the + right thing: fix nonants to the supplied values, `solve_loop` across + local scenarios, compute the expected objective. Whatever mechanism + loads the file just needs to pack the values into a `nonant_cache` + and call `evaluate`. + +## Proposed architecture + +Single flag on the xhat spokes, read once at startup: + +``` +--xhat-from-file +``` + +### Format + +`.npy` only, via the existing `ciutils.read_xhat` helper. + +- It is already the canonical mpi-sppy xhat on-disk format (MMW uses + it; examples write it). +- It is already restricted to two-stage, which aligns with our V1 + non-goal. +- Extending to more formats (CSV, JSON) is a follow-up, only if a real + use case appears. Each format adds surface area and edge cases + (column order, header presence, numeric precision); none are worth + paying for speculatively. + +### Where the read happens + +At the start of each xhat spoke's `main()`, immediately **after** +`self.xhat_prep()` (which sets up `Xhat_Eval`) and **before** the +spoke's normal iteration loop. Mirrors the Jensen's hook point +verbatim so the two features share the same pre-loop slot. + +Concretely, add a small helper to `XhatBase` (the extension; not the +cylinder base) that: + +1. Reads the `.npy` file via `ciutils.read_xhat`. +2. Packs the values into a `nonant_cache` shaped like + `s._mpisppy_data.nonant_indices` order. Same packing logic as + Jensen's mixin §5.2 (`_pack_nonant_cache`). +3. Calls `self.opt.evaluate(nonant_cache)` to get an `Eobj` (or + `None` on infeasibility). +4. If finite: `self.update_if_improving(Eobj)` → send inner bound + and best-xhat to the hub as the first inner-bound report. +5. If `None`: xhat was infeasible. The xhatter's normal infeasibility + handling takes over (including the `--xhat-feasibility-cuts-count` + emission if that feature is enabled), and the spoke continues into + its regular loop. + +The helper lives on `XhatBase` so every xhatter (`xhatlooper`, +`xhatshufflelooper`, `xhatspecific`, `xhatxbar`) picks it up for +free. The spokes themselves stay untouched beyond the one-liner +invocation at the top of `main()`. + +### CLI + cfg plumbing + +- New `cfg.xhat_from_file_args()` method registering the flag with + default `None` (feature off). +- Added to `mpisppy/generic/parsing.py::parse_args` alongside the + other `xhatXxx_args()` calls. +- `shared_options` in `cfg_vanilla` carries the path string through + to every xhat spoke's options dict, parallel to how we're + propagating `xhat_feasibility_cuts_count`. +- Startup-time validation: if the flag is set, the file must exist + and be readable. Hard-fail with a clear error message at `xhat_prep` + time (not later when we try to load). + +## Precedence with Jensen's `--*-try-jensens-first` + +Both features contribute a candidate xhat to evaluate once, before the +spoke's main loop. Both use `Xhat_Eval.evaluate(nonant_cache)` and +`update_if_improving(Eobj)`. They do not conflict mechanically — but +we need a rule for ordering if both are set. + +**Rule:** *file-supplied xhat first, then Jensen's, then the spoke's +normal loop.* Rationale: + +- The file is an *explicit* user hint — the user went to the trouble + of writing an `.npy`. If both are on, they probably want to see + what the explicit candidate does before Jensen's. +- Jensen's is a cheap-to-compute heuristic first pick; a natural + fallback when the file-supplied one is infeasible. +- `update_if_improving` keeps whichever is better, so correctness + is invariant to order; only the ordering of the first `send_bound` + report differs. Predictable ordering matters for test stability + and log readability. + +Implementation: the two features contribute candidates to the front +of an (internal, trivial) list: + +``` +candidates = [] +if xhat_from_file_path is not None: + candidates.append(("from-file", load_nonant_cache_from_file())) +if jensens_enabled: + candidates.append(("jensens", solve_ev_and_pack())) +for label, nc in candidates: + Eobj = self.opt.evaluate(nc) + if Eobj is not None: + self.update_if_improving(Eobj) + # infeasibility falls through; feas-cut path (if on) fires +``` + +No mixin surgery needed; both features just agree to share the +"candidates" list if we want to refactor later. For V1, each feature +writes its own snippet at the top of `main()` and the doc commits to +the above order. + +## Multi-stage + +Deferred, same as for Jensen's and for `ciutils.read_xhat` itself. +For V1, the startup check raises if `num_stages != 2`, with a pointer +to this design doc's follow-up section. Mechanically, multi-stage +needs either (a) a per-node `.npy` or (b) a format that carries node +keys — both are future work. The feature is still useful in its +two-stage form, and shipping that now does not foreclose the +multi-stage extension. + +## Interaction with `--xhat-feasibility-cuts-count` + +This is the test-motivated case. When both flags are on: + +1. Spoke reads the file, evaluates. +2. `Xhat_Eval.evaluate` detects infeasibility (or not). On + infeasibility, `_try_one` → `_maybe_emit_feasibility_cut` fires + (the code landed in PR #671), sending a no-good row through + `Field.XHAT_FEASIBILITY_CUT`. +3. The hub's `XhatFeasibilityCutExtension` installs the cut into + every scenario. +4. On the next xhatter iteration, the same `xhat` is now excluded + by the installed cut. + +Step 4 gives us the end-to-end assertion the current +`test_xhat_feasibility_cuts.py` unit tests cannot make: *the +infeasible xhat is not revisited*. A new integration test: + +- Writes a known-infeasible binary vector to an `.npy`. +- Runs `generic_cylinders` with `--xhat-from-file` + + `--xhat-feasibility-cuts-count 1` + a minimal binary-first-stage + model that is infeasible at that vector but feasible elsewhere. +- Asserts (via run output or a post-run probe) that a cut got + installed. + +This is the testing payoff that motivated the feature. It does not +drive the design — but it shapes what "first-milestone scope" means +below. + +## Open questions + +- **Should the file-read be rank-zero-only plus bcast, or should every + rank read independently?** `ciutils.read_xhat` is called per-rank + today; with a small shared file system this is fine, but on + clusters with flaky shared storage, rank-0-read + `bcast` is safer. + V1: mirror the existing `ciutils` behavior (per-rank read). Revisit + if a user reports trouble. +- **What happens when the file's vector length doesn't match the + problem's root-node nonant count?** Hard-fail at load time with a + clear message. No truncation, no padding. +- **Should the inner bound reported from the file xhat update the + `best_solution_cache`?** Yes — `update_if_improving` does this by + default (`spoke.py:173-190`). The file-supplied xhat is a real + candidate and should be reported like any other. + +## First-milestone scope + +A discrete, reviewable PR delivering: + +1. `cfg.xhat_from_file_args()` registering `--xhat-from-file`. +2. `XhatBase` helper that loads the file, packs the nonant cache, + evaluates, and updates if improving. +3. Each of the four xhat spokes calls that helper at the top of + `main()` (right after `xhat_prep`, before Jensen's if present). +4. Hard-fail paths: missing file, nonant-length mismatch, multi-stage. +5. `cfg_vanilla.shared_options` propagates the path. +6. Wired into `mpisppy/generic/parsing.py` so `generic_cylinders` + accepts the flag. +7. Tests: + - Unit: loader returns the right vector; wrong-length raises. + - Integration (no MPI): mock `Xhat_Eval.evaluate` and assert the + helper calls it with the file-loaded vector. + - Integration (the testing-PR-671 payoff): `run_all.py` entry + that runs the USAR `wheel_spinner` (binary first-stage) with a + pre-computed `.npy` file and `--xhat-feasibility-cuts-count=1`, + then asserts the run completes — and if we can arrange a + known-infeasible xhat, that a feasibility cut fired. +8. User-facing `doc/src/xhat_from_file.rst` wired into + `doc/src/index.rst`. Includes the "use this to test feasibility + cuts" recipe. + +Follow-ups (not in V1): + +- Multi-stage support. +- CSV/JSON support. +- Re-read mid-run (unlikely to ever be worth it). diff --git a/mpisppy/cylinders/xhatbase.py b/mpisppy/cylinders/xhatbase.py index 175e29db2..1e62ba1c7 100644 --- a/mpisppy/cylinders/xhatbase.py +++ b/mpisppy/cylinders/xhatbase.py @@ -8,6 +8,9 @@ ############################################################################### import abc +import os +import math + import mpisppy.cylinders.spoke as spoke from mpisppy.utils.xhat_eval import Xhat_Eval @@ -16,15 +19,15 @@ class XhatInnerBoundBase(spoke.InnerBoundNonantSpoke): @abc.abstractmethod def xhat_extension(self): raise NotImplementedError - + def xhat_prep(self): ## for later - self.verbose = self.opt.options["verbose"] # typing aid + self.verbose = self.opt.options["verbose"] # typing aid if not isinstance(self.opt, Xhat_Eval): raise RuntimeError(f"{self.__class__.__name__} must be used with Xhat_Eval.") - + xhatter = self.xhat_extension() ### begin iter0 stuff @@ -44,7 +47,62 @@ def xhat_prep(self): xhatter.post_iter0() if self.opt.extensions is not None: self.opt.extobject.post_iter0() # for an extension - + self.opt._save_nonants() # make the cache + # Optional: try an xhat loaded from a file before the normal + # xhatter main loop. See doc/src/xhat_from_file.rst. + self._try_file_xhat() + return xhatter + + def _try_file_xhat(self): + """Evaluate a file-supplied xhat once, before the main loop. + + Gated on ``options['xhat_from_file']`` being a path. Two-stage + only for V1 (matches ``ciutils.read_xhat``). Hard-fails on + missing file, length mismatch, or multi-stage. Restores + nonants afterwards so the spoke's main loop sees clean state. + """ + path = self.opt.options.get("xhat_from_file", None) + if not path: + return + # Lazy import to avoid any startup-time coupling and to keep + # numpy out of the non-feature path. + from mpisppy.confidence_intervals import ciutils + + if self.opt.multistage: + raise RuntimeError( + "--xhat-from-file is two-stage only; multi-stage support " + "is planned as a follow-up. See " + "doc/xhat_from_file_design.md." + ) + if not os.path.exists(path): + raise RuntimeError( + f"--xhat-from-file={path!r} does not exist." + ) + + nonant_cache = ciutils.read_xhat(path, num_stages=2) + # Length check against the root-node nonant count of an + # arbitrary local scenario (all local scenarios share the + # same nonant count by PH invariant). + any_s = next(iter(self.opt.local_scenarios.values())) + expected = len(any_s._mpisppy_data.nonant_indices) + got = len(nonant_cache["ROOT"]) + if got != expected: + raise RuntimeError( + f"--xhat-from-file vector length {got} does not match the " + f"problem's root-node nonant count {expected} (file={path!r})." + ) + + if self.cylinder_rank == 0: + print(f"[xhat-from-file] evaluating {path!r} " + f"({expected} nonants)") + Eobj = self.opt.evaluate(nonant_cache) + # Restore nonants so the main loop starts from clean state. + self.opt._restore_nonants() + if Eobj is not None and math.isfinite(Eobj): + self.update_if_improving(Eobj) + elif self.cylinder_rank == 0: + print(f"[xhat-from-file] candidate gave Eobj={Eobj!r}; not " + f"updating inner bound") diff --git a/mpisppy/generic/parsing.py b/mpisppy/generic/parsing.py index 781489917..b13201541 100644 --- a/mpisppy/generic/parsing.py +++ b/mpisppy/generic/parsing.py @@ -135,6 +135,7 @@ def parse_args(m): cfg.subgradient_bounder_args() cfg.xhatshuffle_args() cfg.xhatxbar_args() + cfg.xhat_from_file_args() cfg.norm_rho_args() cfg.primal_dual_rho_args() cfg.converger_args() diff --git a/mpisppy/tests/test_xhat_from_file.py b/mpisppy/tests/test_xhat_from_file.py new file mode 100644 index 000000000..9403ff25c --- /dev/null +++ b/mpisppy/tests/test_xhat_from_file.py @@ -0,0 +1,294 @@ +############################################################################### +# mpi-sppy: MPI-based Stochastic Programming in PYthon +# +# Copyright (c) 2024, Lawrence Livermore National Security, LLC, Alliance for +# Sustainable Energy, LLC, The Regents of the University of California, et al. +# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md for +# full copyright and license information. +############################################################################### +"""Unit tests for ``XhatInnerBoundBase._try_file_xhat``. + +Covers: off-by-default, missing file, multi-stage reject, length +mismatch, happy path (finite ``Eobj`` → ``update_if_improving``), +and infeasibility-style result (non-finite ``Eobj`` → no update, nonants +restored). + +The tests stub out ``self.opt`` and the spoke's bound-update method so +that no MPI, solver, or full cylinder wiring is needed. +""" + +import math +import os +import tempfile +import unittest + +import numpy as np +import pyomo.environ as pyo + +import mpisppy.utils.sputils as sputils +from mpisppy.cylinders.xhatbase import XhatInnerBoundBase +from mpisppy.spbase import SPBase + + +def _binary_scenario(sname, **_): + m = pyo.ConcreteModel(name=sname) + m.x = pyo.Var([0, 1, 2], domain=pyo.Binary) + m.fsc = pyo.Expression(expr=m.x[0] + m.x[1] + m.x[2]) + m.obj = pyo.Objective(expr=m.x[0] + m.x[1] + m.x[2]) + sputils.attach_root_node(m, m.fsc, [m.x]) + m._mpisppy_probability = "uniform" + return m + + +def _make_sp(num=2, options=None): + opts = {"toc": False, "verbose": False} + if options is not None: + opts.update(options) + return SPBase( + options=opts, + all_scenario_names=[f"scen{i}" for i in range(num)], + scenario_creator=_binary_scenario, + ) + + +class _FakeXhatEval: + """Just enough ``Xhat_Eval`` surface for ``_try_file_xhat``.""" + + def __init__(self, sp, evaluate_result): + self.local_scenarios = sp.local_scenarios + self.options = sp.options + self.multistage = sp.multistage + self._evaluate_result = evaluate_result + self.evaluate_calls = [] + self.restore_calls = 0 + + def evaluate(self, nonant_cache): + self.evaluate_calls.append(nonant_cache) + return self._evaluate_result + + def _restore_nonants(self): + self.restore_calls += 1 + + +def _make_helper(sp, evaluate_result=None): + """Build an ``XhatInnerBoundBase``-shaped object without going through + its __init__ (which needs the whole spoke stack).""" + h = XhatInnerBoundBase.__new__(XhatInnerBoundBase) + h.opt = _FakeXhatEval(sp, evaluate_result) + h.cylinder_rank = 0 + h.updates = [] + + def _fake_update(eobj): + h.updates.append(eobj) + return True + h.update_if_improving = _fake_update + return h + + +def _write_npy(values): + fd, path = tempfile.mkstemp(suffix=".npy") + os.close(fd) + np.save(path, np.array(values, dtype=float)) + return path + + +class TestFileXhatDisabled(unittest.TestCase): + + def test_off_by_default_is_noop(self): + sp = _make_sp() + helper = _make_helper(sp, evaluate_result=10.0) + helper._try_file_xhat() + self.assertEqual(helper.opt.evaluate_calls, []) + self.assertEqual(helper.updates, []) + + def test_empty_string_path_is_noop(self): + sp = _make_sp(options={"xhat_from_file": ""}) + helper = _make_helper(sp, evaluate_result=10.0) + helper._try_file_xhat() + self.assertEqual(helper.opt.evaluate_calls, []) + + +class TestFileXhatHardFails(unittest.TestCase): + + def test_missing_file_raises(self): + sp = _make_sp(options={"xhat_from_file": "/tmp/does_not_exist_xhat.npy"}) + helper = _make_helper(sp, evaluate_result=10.0) + with self.assertRaises(RuntimeError) as cm: + helper._try_file_xhat() + self.assertIn("does not exist", str(cm.exception)) + + def test_length_mismatch_raises(self): + # Problem has 3 binary nonants; file has 2. + path = _write_npy([1.0, 0.0]) + try: + sp = _make_sp(options={"xhat_from_file": path}) + helper = _make_helper(sp, evaluate_result=10.0) + with self.assertRaises(RuntimeError) as cm: + helper._try_file_xhat() + msg = str(cm.exception) + self.assertIn("length", msg) + self.assertIn("2", msg) + self.assertIn("3", msg) + finally: + os.remove(path) + + def test_multistage_raises(self): + path = _write_npy([1.0, 0.0, 1.0]) + try: + sp = _make_sp(options={"xhat_from_file": path}) + helper = _make_helper(sp, evaluate_result=10.0) + helper.opt.multistage = True # force the multi-stage path + with self.assertRaises(RuntimeError) as cm: + helper._try_file_xhat() + self.assertIn("two-stage", str(cm.exception)) + finally: + os.remove(path) + + +class TestFileXhatHappyPath(unittest.TestCase): + + def test_finite_eobj_updates_and_restores(self): + path = _write_npy([1.0, 0.0, 1.0]) + try: + sp = _make_sp(options={"xhat_from_file": path}) + helper = _make_helper(sp, evaluate_result=42.0) + helper._try_file_xhat() + # evaluate was called with {'ROOT': [1,0,1]} + self.assertEqual(len(helper.opt.evaluate_calls), 1) + nc = helper.opt.evaluate_calls[0] + self.assertIn("ROOT", nc) + np.testing.assert_array_equal(nc["ROOT"], [1.0, 0.0, 1.0]) + # update_if_improving received Eobj + self.assertEqual(helper.updates, [42.0]) + # nonants were restored for the subsequent main loop + self.assertEqual(helper.opt.restore_calls, 1) + finally: + os.remove(path) + + def test_infeasible_like_eobj_does_not_update_but_still_restores(self): + """When evaluate reports a non-finite objective (some scenario + was infeasible), the helper must NOT call update_if_improving — + but it must still restore nonants.""" + path = _write_npy([1.0, 0.0, 1.0]) + try: + sp = _make_sp(options={"xhat_from_file": path}) + helper = _make_helper(sp, evaluate_result=float("inf")) + helper._try_file_xhat() + self.assertEqual(len(helper.opt.evaluate_calls), 1) + self.assertEqual(helper.updates, []) + self.assertEqual(helper.opt.restore_calls, 1) + finally: + os.remove(path) + + def test_none_eobj_does_not_update(self): + """Evaluate returning None is the 'no expected value' case — + skip the update, still restore.""" + path = _write_npy([1.0, 0.0, 1.0]) + try: + sp = _make_sp(options={"xhat_from_file": path}) + helper = _make_helper(sp, evaluate_result=None) + helper._try_file_xhat() + self.assertEqual(helper.updates, []) + self.assertEqual(helper.opt.restore_calls, 1) + finally: + os.remove(path) + + +class TestMathIsfinite(unittest.TestCase): + """Sanity check that math.isfinite is the predicate we intend: + inf is not finite, large real is finite.""" + + def test_finite_positive(self): + self.assertTrue(math.isfinite(1e18)) + + def test_infinity_not_finite(self): + self.assertFalse(math.isfinite(float("inf"))) + + def test_nan_not_finite(self): + self.assertFalse(math.isfinite(float("nan"))) + + +class TestConfigArgRegistration(unittest.TestCase): + def test_xhat_from_file_args_registers_with_default_none(self): + from mpisppy.utils import config as cfgmod + cfg = cfgmod.Config() + cfg.xhat_from_file_args() + self.assertIn("xhat_from_file", cfg) + self.assertIsNone(cfg.get("xhat_from_file")) + + def test_generic_parsing_registers_the_flag(self): + """Covers the cfg.xhat_from_file_args() call inside + mpisppy.generic.parsing.parse_args.""" + import sys + import types + import mpisppy.generic.parsing as parsing + + stub = types.ModuleType("__stub_model_for_parse_args__") + def inparser_adder(cfg): + cfg.add_to_config("num_scens", "stub", int, default=1) + stub.inparser_adder = inparser_adder + + saved_argv = sys.argv + sys.argv = ["prog"] + try: + cfg = parsing.parse_args(stub) + finally: + sys.argv = saved_argv + + self.assertIn("xhat_from_file", cfg) + + +class TestXhatPrepCallSite(unittest.TestCase): + """Covers the ``self._try_file_xhat()`` call inside ``xhat_prep``. + + We stub ``self.opt`` with the minimum that xhat_prep touches and + leave ``options['xhat_from_file']`` unset, so the real + ``_try_file_xhat`` early-returns and the spoke's xhat_prep flow + exercises the call site without needing an npy file.""" + + def test_xhat_prep_invokes_try_file_xhat(self): + import mpisppy.cylinders.xhatbase as cxb + from mpisppy.utils.xhat_eval import Xhat_Eval + + sp = _make_sp() + + class _StubExt: + def pre_iter0(self): pass + def post_iter0(self): pass + + class _StubXhatEval(Xhat_Eval): + def __init__(self2): + # Skip Xhat_Eval.__init__; set the attributes used by + # xhat_prep and _try_file_xhat's early-return branch. + self2.options = {"verbose": False, "xhat_from_file": None} + self2.local_scenarios = sp.local_scenarios + self2.extensions = None + self2.multistage = sp.multistage + self2.E1 = 1.0 + self2.E1_tolerance = 1e-5 + def _save_original_nonants(self2): pass + def _lazy_create_solvers(self2): pass + def _update_E1(self2): pass + def _save_nonants(self2): pass + + spoke_obj = cxb.XhatInnerBoundBase.__new__(cxb.XhatInnerBoundBase) + spoke_obj.opt = _StubXhatEval() + spoke_obj.cylinder_rank = 0 + spoke_obj.xhat_extension = lambda: _StubExt() + + calls = {"n": 0} + original_try = cxb.XhatInnerBoundBase._try_file_xhat + + def _spy(self): + calls["n"] += 1 + original_try(self) # exercises the "path is None → return" branch + cxb.XhatInnerBoundBase._try_file_xhat = _spy + try: + spoke_obj.xhat_prep() + finally: + cxb.XhatInnerBoundBase._try_file_xhat = original_try + self.assertEqual(calls["n"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/mpisppy/utils/cfg_vanilla.py b/mpisppy/utils/cfg_vanilla.py index 3c571a63a..1e1e13dd9 100644 --- a/mpisppy/utils/cfg_vanilla.py +++ b/mpisppy/utils/cfg_vanilla.py @@ -46,6 +46,9 @@ def shared_options(cfg): "user_warmstart" : cfg.user_warmstart, "turn_off_names_check" : cfg.turn_off_names_check or cfg.get("scenarios_per_bundle") is not None, + # Optional initial xhat candidate file (.npy); None disables. + # Consumed by XhatInnerBoundBase._try_file_xhat. + "xhat_from_file" : cfg.get("xhat_from_file", None), } if _hasit(cfg, "solver_options"): odict = sputils.option_string_to_dict(cfg.solver_options) diff --git a/mpisppy/utils/config.py b/mpisppy/utils/config.py index c334d5332..4bdb8d291 100644 --- a/mpisppy/utils/config.py +++ b/mpisppy/utils/config.py @@ -891,6 +891,21 @@ def xhatlshaped_args(self): domain=bool, default=False) + def xhat_from_file_args(self): + # Supply an initial xhat candidate from a .npy file. Every xhat + # spoke (xhatlooper, xhatshufflelooper, xhatspecific, xhatxbar) + # that descends from XhatInnerBoundBase will evaluate it once, + # before its normal exploration loop. Two-stage only today + # (matches ciutils.read_xhat). See + # doc/src/xhat_from_file.rst. + self.add_to_config("xhat_from_file", + description="Path to a .npy file holding an initial " + "first-stage xhat vector to evaluate " + "before normal xhatter exploration. " + "Two-stage only. Default None (off).", + domain=str, + default=None) + def wtracker_args(self): self.add_to_config('wtracker', diff --git a/run_coverage.bash b/run_coverage.bash index b28a87737..f8c52a5eb 100755 --- a/run_coverage.bash +++ b/run_coverage.bash @@ -97,6 +97,9 @@ run_phase "test_smps (serial)" \ run_phase "test_generic_cylinders (serial)" \ coverage run --rcfile=.coveragerc -m pytest mpisppy/tests/test_generic_cylinders.py -v +run_phase "test_xhat_from_file (serial)" \ + coverage run --rcfile=.coveragerc -m pytest mpisppy/tests/test_xhat_from_file.py -v + run_phase "test_conf_int_farmer (spawns mpiexec)" \ coverage run --rcfile=.coveragerc mpisppy/tests/test_conf_int_farmer.py