Solver options: design doc + phase 1 (layered representation, dormant)

.github/workflows/test_pr_and_main.yml

@@ -100,6 +100,10 @@ jobs:
         run: |
           coverage run $COV_ARGS -m pytest mpisppy/tests/test_component_map_usage.py
 
+      - name: Test solver options layers
+        run: |
+          coverage run $COV_ARGS -m pytest mpisppy/tests/test_solver_options_layers.py -v
+
       - name: Test nonant name validation
         run: |
           coverage run $COV_ARGS -m pytest mpisppy/tests/test_nonant_validation.py -v

mpisppy/phbase.py

@@ -284,6 +284,10 @@ def __init__(
         self.iter0_solver_options = options["iter0_solver_options"]
         self.iterk_solver_options = options["iterk_solver_options"]
         self.current_solver_options = self.iter0_solver_options
+        # Phase-1 groundwork (doc/designs/solver_options_redesign.md
+        # §6.4 phase 1). Layer list is stored but not yet read by the
+        # solve path; iter0/iterk dicts above continue to drive solves.
+        self.solver_options_layers = options.get("solver_options_layers", [])
 
         # flags to complete the invariant
         self.convobject = None  # PH converger

mpisppy/tests/test_solver_options_layers.py

@@ -0,0 +1,218 @@
+###############################################################################
+# mpi-sppy: MPI-based Stochastic Programming in PYthon
+#
+# Copyright (c) 2024, Lawrence Livermore National Security, LLC, Alliance for
+# Sustainable Energy, LLC, The Regents of the University of California, et al.
+# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md for
+# full copyright and license information.
+###############################################################################
+# Phase-1 tests for solver_options_layers — see
+# doc/designs/solver_options_redesign.md §6.4. The contract this file
+# pins is: the new layered representation folds (per-iteration) to
+# dicts identical to the legacy iter0_solver_options /
+# iterk_solver_options dicts produced by shared_options and
+# apply_solver_specs. Phase 1 is a pure-refactor groundwork step;
+# every later phase will lean on this equivalence.
+
+import copy
+import unittest
+
+from mpisppy.utils import config
+from mpisppy.utils.cfg_vanilla import shared_options, apply_solver_specs
+from mpisppy.utils.sputils import (
+    fold_solver_options_layers,
+    solver_options_layer,
+)
+
+
+def _bare_cfg():
+    cfg = config.Config()
+    cfg.popular_args()
+    cfg.add_mipgap_specs()
+    return cfg
+
+
+def _spoke_cfg(spoke_name):
+    cfg = _bare_cfg()
+    cfg.add_solver_specs(prefix=spoke_name)
+    cfg.add_mipgap_specs(prefix=spoke_name)
+    return cfg
+
+
+class TestSharedOptionsLayers(unittest.TestCase):
+
+    def test_empty_cfg_yields_empty_layers(self):
+        cfg = _bare_cfg()
+        sh = shared_options(cfg)
+        self.assertEqual(sh["solver_options_layers"], [])
+        self.assertEqual(
+            fold_solver_options_layers(sh["solver_options_layers"], 0), {})
+        self.assertEqual(
+            fold_solver_options_layers(sh["solver_options_layers"], 1), {})
+
+    def test_solver_options_string_is_default_layer(self):
+        cfg = _bare_cfg()
+        cfg.solver_options = "mipgap=0.01 logfile=run.log"
+        sh = shared_options(cfg)
+        layers = sh["solver_options_layers"]
+        expected = {"mipgap": 0.01, "logfile": "run.log"}
+        self.assertEqual(fold_solver_options_layers(layers, 0), expected)
+        self.assertEqual(fold_solver_options_layers(layers, 1), expected)
+        self.assertEqual(fold_solver_options_layers(layers, 7), expected)
+
+    def test_iter0_iterk_mipgap_yield_predicate_layers(self):
+        cfg = _bare_cfg()
+        cfg.iter0_mipgap = 0.01
+        cfg.iterk_mipgap = 0.02
+        sh = shared_options(cfg)
+        layers = sh["solver_options_layers"]
+        self.assertEqual(
+            fold_solver_options_layers(layers, 0), {"mipgap": 0.01})
+        self.assertEqual(
+            fold_solver_options_layers(layers, 1), {"mipgap": 0.02})
+        self.assertEqual(
+            fold_solver_options_layers(layers, 5), {"mipgap": 0.02})
+
+    def test_max_solver_threads_overrides_solver_options_threads(self):
+        # Mirrors today's behavior at cfg_vanilla.py:83-85: the global
+        # thread cap overwrites whatever 'threads' the user wrote inline.
+        cfg = _bare_cfg()
+        cfg.solver_options = "mipgap=0.01 threads=2"
+        cfg.max_solver_threads = 4
+        sh = shared_options(cfg)
+        layers = sh["solver_options_layers"]
+        folded = fold_solver_options_layers(layers, 0)
+        self.assertEqual(folded["mipgap"], 0.01)
+        self.assertEqual(folded["threads"], 4)
+
+    def test_combined_fold_equals_legacy_iter0_iterk_dicts(self):
+        # The regression contract for phase 1: under any cfg
+        # combination, fold(layers, 0) must match iter0_solver_options
+        # and fold(layers, k>=1) must match iterk_solver_options.
+        cfg = _bare_cfg()
+        cfg.solver_options = "logfile=run.log threads=2"
+        cfg.max_solver_threads = 4
+        cfg.iter0_mipgap = 0.01
+        cfg.iterk_mipgap = 0.02
+        sh = shared_options(cfg)
+        layers = sh["solver_options_layers"]
+        self.assertEqual(
+            fold_solver_options_layers(layers, 0),
+            sh["iter0_solver_options"],
+        )
+        self.assertEqual(
+            fold_solver_options_layers(layers, 1),
+            sh["iterk_solver_options"],
+        )
+        self.assertEqual(
+            fold_solver_options_layers(layers, 7),
+            sh["iterk_solver_options"],
+        )
+
+
+class TestApplySolverSpecsLayers(unittest.TestCase):
+
+    def _spoke_dict_from(self, sh):
+        # apply_solver_specs operates on spoke["opt_kwargs"]["options"];
+        # it expects a deepcopy of shared_options output.
+        return {"opt_kwargs": {"options": copy.deepcopy(sh)}}
+
+    def test_per_spoke_solver_options_replace_layers(self):
+        # Phase 1 mirrors today's replace-not-overlay semantics in
+        # apply_solver_specs (cfg_vanilla.py:119-120). Phase 5 will
+        # change this to overlay; phase-1 tests pin the legacy contract.
+        cfg = _spoke_cfg("lagrangian")
+        cfg.solver_options = "logfile=run.log"
+        cfg.lagrangian_solver_options = "mipgap=0.001"
+        sh = shared_options(cfg)
+        spoke = self._spoke_dict_from(sh)
+        apply_solver_specs("lagrangian", spoke, cfg)
+        opts = spoke["opt_kwargs"]["options"]
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 0),
+            opts["iter0_solver_options"],
+        )
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 1),
+            opts["iterk_solver_options"],
+        )
+        self.assertEqual(opts["iter0_solver_options"], {"mipgap": 0.001})
+
+    def test_per_spoke_iter0_iterk_mipgap_layer_predicate(self):
+        cfg = _spoke_cfg("lagrangian")
+        cfg.solver_options = "logfile=run.log"
+        cfg.lagrangian_iter0_mipgap = 0.01
+        cfg.lagrangian_iterk_mipgap = 0.02
+        sh = shared_options(cfg)
+        spoke = self._spoke_dict_from(sh)
+        apply_solver_specs("lagrangian", spoke, cfg)
+        opts = spoke["opt_kwargs"]["options"]
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 0),
+            opts["iter0_solver_options"],
+        )
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 1),
+            opts["iterk_solver_options"],
+        )
+
+    def test_per_spoke_threads_reapplied(self):
+        # apply_solver_specs re-applies max_solver_threads at the end
+        # (cfg_vanilla.py:127-129). The layered version must produce
+        # the same final folded dict for both predicates.
+        cfg = _spoke_cfg("lagrangian")
+        cfg.solver_options = "mipgap=0.01"
+        cfg.lagrangian_solver_options = "presolve=1"
+        cfg.max_solver_threads = 8
+        sh = shared_options(cfg)
+        spoke = self._spoke_dict_from(sh)
+        apply_solver_specs("lagrangian", spoke, cfg)
+        opts = spoke["opt_kwargs"]["options"]
+        self.assertEqual(opts["iter0_solver_options"].get("threads"), 8)
+        self.assertEqual(opts["iterk_solver_options"].get("threads"), 8)
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 0),
+            opts["iter0_solver_options"],
+        )
+        self.assertEqual(
+            fold_solver_options_layers(opts["solver_options_layers"], 1),
+            opts["iterk_solver_options"],
+        )
+
+
+class TestPredicateValidation(unittest.TestCase):
+
+    def test_valid_predicates_accepted(self):
+        for when in ("default", "iter0", "iterk",
+                     ("after_iter", 0), ("after_iter", 5)):
+            solver_options_layer(when, {"mipgap": 0.01})
+
+    def test_unknown_string_predicate_rejected(self):
+        with self.assertRaises(ValueError):
+            solver_options_layer("sometimes", {})
+
+    def test_after_iter_negative_rejected(self):
+        with self.assertRaises(ValueError):
+            solver_options_layer(("after_iter", -1), {})
+
+    def test_after_iter_non_int_rejected(self):
+        with self.assertRaises(ValueError):
+            solver_options_layer(("after_iter", 1.5), {})
+        with self.assertRaises(ValueError):
+            solver_options_layer(("after_iter", "5"), {})
+
+    def test_after_iter_bool_rejected(self):
+        # bool is an int subclass — guard against (after_iter, True)
+        with self.assertRaises(ValueError):
+            solver_options_layer(("after_iter", True), {})
+
+    def test_fold_rejects_invalid_predicate(self):
+        # Hand-built layer (bypasses solver_options_layer) is still validated
+        # by fold time.
+        bad_layers = [{"when": "huh", "options": {}}]
+        with self.assertRaises(ValueError):
+            fold_solver_options_layers(bad_layers, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

mpisppy/utils/cfg_vanilla.py

@@ -62,6 +62,11 @@ def shared_options(cfg, is_hub=False):
         "display_convergence_detail": cfg.display_convergence_detail,
         "iter0_solver_options": dict(),
         "iterk_solver_options": dict(),
+        # Phase-1 groundwork (see doc/designs/solver_options_redesign.md
+        # §5.2 / §6.4). The layer list is built in parallel with the
+        # legacy iter0/iterk dicts and folds to the same values; no
+        # consumer reads from it yet.
+        "solver_options_layers": [],
         "tee-rank0-solves": cfg.tee_rank0_solves,
         "trace_prefix" : cfg.trace_prefix,
         "presolve" : cfg.presolve,
@@ -79,14 +84,25 @@ def shared_options(cfg, is_hub=False):
         odict = sputils.option_string_to_dict(cfg.solver_options)
         shoptions["iter0_solver_options"] = odict
         shoptions["iterk_solver_options"] = copy.deepcopy(odict)
-    # note that specific options usch as mipgap will override        
+        shoptions["solver_options_layers"].append(
+            sputils.solver_options_layer("default", odict))
+    # note that specific options such as mipgap will override
     if _hasit(cfg, "max_solver_threads"):
         shoptions["iter0_solver_options"]["threads"] = cfg.max_solver_threads
         shoptions["iterk_solver_options"]["threads"] = cfg.max_solver_threads
+        shoptions["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "default", {"threads": cfg.max_solver_threads}))
     if _hasit(cfg, "iter0_mipgap"):
         shoptions["iter0_solver_options"]["mipgap"] = cfg.iter0_mipgap
+        shoptions["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "iter0", {"mipgap": cfg.iter0_mipgap}))
     if _hasit(cfg, "iterk_mipgap"):
         shoptions["iterk_solver_options"]["mipgap"] = cfg.iterk_mipgap
+        shoptions["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "iterk", {"mipgap": cfg.iterk_mipgap}))
     if _hasit(cfg, "reduced_costs"):
         shoptions["rc_bound_tol"] = cfg.rc_bound_tol
     if _hasit(cfg, "solver_log_dir"):
@@ -113,21 +129,39 @@ def shared_options(cfg, is_hub=False):
 
 def apply_solver_specs(name, spoke, cfg):
     options = spoke["opt_kwargs"]["options"]
+    # Phase-1 groundwork: keep the legacy iter0/iterk dict mutations and
+    # mirror them onto solver_options_layers (§5.2 / §6.4 phase 1).
+    # Layer list is dormant; no consumer reads it yet. Phase 5 will
+    # change replace-style to overlay; phase 1 keeps the legacy contract.
+    options.setdefault("solver_options_layers", [])
     if _hasit(cfg, name+"_solver_name"):
         options["solver_name"] = cfg.get(name+"_solver_name")
     if _hasit(cfg, name+"_solver_options"):
         odict = sputils.option_string_to_dict(cfg.get(name+"_solver_options"))
         options["iter0_solver_options"] = odict
         options["iterk_solver_options"] = copy.deepcopy(odict)
+        # Mirror replace semantics for layers.
+        options["solver_options_layers"] = [
+            sputils.solver_options_layer("default", odict)
+        ]
     if _hasit(cfg, name+"_iter0_mipgap"):
         options["iter0_solver_options"]["mipgap"] = cfg.get(name+"_iter0_mipgap")
+        options["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "iter0", {"mipgap": cfg.get(name+"_iter0_mipgap")}))
     if _hasit(cfg, name+"_iterk_mipgap"):
         options["iterk_solver_options"]["mipgap"] = cfg.get(name+"_iterk_mipgap")
+        options["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "iterk", {"mipgap": cfg.get(name+"_iterk_mipgap")}))
     # re-apply max_solver_threads since we may have over-written the
     # iter*_solver_options above.
     if _hasit(cfg, "max_solver_threads"):
         options["iter0_solver_options"]["threads"] = cfg.max_solver_threads
         options["iterk_solver_options"]["threads"] = cfg.max_solver_threads
+        options["solver_options_layers"].append(
+            sputils.solver_options_layer(
+                "default", {"threads": cfg.max_solver_threads}))
 
 def add_multistage_options(cylinder_dict,all_nodenames,branching_factors):
     cylinder_dict = copy.deepcopy(cylinder_dict)

mpisppy/utils/sputils.py

@@ -710,6 +710,79 @@ def option_dict_to_string(odict):
     return ostr
 
 
+# Solver-options layered representation. See
+# doc/designs/solver_options_redesign.md §5.2.
+#
+# A layer is a plain dict {"when": <predicate>, "options": <dict>}.
+# Predicates:
+#   "default"             — applies at every iteration
+#   "iter0"               — only at iteration 0
+#   "iterk"               — iterations k >= 1
+#   ("after_iter", N)     — iterations k >= N (N is int)
+
+
+def _validate_when(when):
+    """Raise ValueError if *when* is not a recognized layer predicate."""
+    if when in ("default", "iter0", "iterk"):
+        return
+    if isinstance(when, tuple) and len(when) == 2 and when[0] == "after_iter":
+        N = when[1]
+        # bool is a subclass of int; reject it explicitly
+        if isinstance(N, bool) or not isinstance(N, int) or N < 0:
+            raise ValueError(
+                f"after_iter predicate requires a non-negative int, got {N!r}")
+        return
+    raise ValueError(f"Unknown solver-options layer predicate: {when!r}")
+
+
+def solver_options_layer(when, options):
+    """Build a single solver-options layer.
+
+    Args:
+        when: predicate, one of "default", "iter0", "iterk", or
+            ("after_iter", N) with N a non-negative int.
+        options (dict): the options to fold in when the predicate matches.
+
+    Returns:
+        dict with keys "when" and "options".
+    """
+    _validate_when(when)
+    return {"when": when, "options": dict(options)}
+
+
+def _layer_matches(when, k):
+    _validate_when(when)
+    if when == "default":
+        return True
+    if when == "iter0":
+        return k == 0
+    if when == "iterk":
+        return k >= 1
+    # only ("after_iter", N) remains, validated above
+    return k >= when[1]
+
+
+def fold_solver_options_layers(layers, k):
+    """Fold a list of solver-options layers into one dict for iteration k.
+
+    Walks layers in list order, picks layers whose predicate matches k,
+    and flat-dict-unions their options into a running dict (last write
+    wins per key). See doc/designs/solver_options_redesign.md §5.4.
+
+    Args:
+        layers (list): list of layers as built by solver_options_layer.
+        k (int): iteration number (0 for iter0).
+
+    Returns:
+        dict: the merged options for iteration k.
+    """
+    folded = {}
+    for layer in layers:
+        if _layer_matches(layer["when"], k):
+            folded.update(layer["options"])
+    return folded
+
+
 ################################################################################
 # Various utilities related to scenario rank maps (some may not be in use)