Merge pull request #79 from CurtisTechSolutions/fix/gym-checkpoint-opponent-identity

fix(training): bind Gym checkpoints to opponent settings

Author: nosam1998

examples/gym-checkpoints.md

@@ -0,0 +1,16 @@
+# Resuming single-seat training
+
+`GameEnv.get_state()` version 3 binds the learner seat, game/configuration,
+built-in opponent name and level, native engine state, environment RNG, and
+curriculum. A restore into an environment with a different built-in opponent
+fails atomically. Versions 1 and 2 remain loadable with the default random
+opponent only, because those formats did not identify an opponent.
+
+Python callbacks, including `ChatPolicy`, use the `python` identity marker.
+The trainer must restore the matching model, optimizer, policy RNG and remote
+model configuration itself. Environment checkpoints do not contain weights,
+API keys, HTTP clients, or external model state. Gym space sampling RNGs are
+also caller-owned. Use your trainer's seeded RNG for reproducible action sampling.
+
+Checkpoints contain private engine and curriculum data. Save them in trusted
+training storage; policy inputs should use only seat-visible observations.

python/gamesforai/gym_env.py

@@ -189,7 +189,8 @@ def clone(self) -> GameEnv:
     def get_state(self) -> dict[str, Any]:
         """Trusted checkpoint containing private engine state and environment RNG."""
         return {
-            "version": 2,
+            "version": 3,
+            "opponent": self.opponent if isinstance(self.opponent, str) else "python",
             "seat": self.seat,
             "native": json.loads(self.native.get_state()),
             "rng": copy.deepcopy(self.np_random.bit_generator.state),
@@ -200,11 +201,26 @@ def get_state(self) -> dict[str, Any]:
 
     def set_state(self, state: dict[str, Any]) -> None:
         """Atomically restore this wrapper; external policy state is not included."""
+        if not isinstance(state, dict) or type(state.get("version")) is not int:
+            raise ValueError("incompatible environment checkpoint")
+        version = state["version"]
         required = {"version", "seat", "native", "rng", "rng_seed", "needs_reset"}
-        if state.get("version") == 2:
+        if version in (2, 3):
             required.add("curriculum")
-        if set(state) != required or state["version"] not in (1, 2) or state["seat"] != self.seat:
+        if version == 3:
+            required.add("opponent")
+        if (
+            set(state) != required or version not in (1, 2, 3)
+            or type(state["seat"]) is not int or state["seat"] != self.seat
+            or type(state["rng_seed"]) is not int or state["rng_seed"] < -1
+        ):
             raise ValueError("incompatible environment checkpoint")
+        identity = self.opponent if isinstance(self.opponent, str) else "python"
+        # Old checkpoints did not bind an opponent; their original default was random.
+        if (version == 3 and state["opponent"] != identity) or (
+            version < 3 and identity != "random"
+        ):
+            raise ValueError("checkpoint opponent differs from this environment")
         if not isinstance(state["needs_reset"], bool):
             raise ValueError("invalid checkpoint episode flag")
         candidate = self.native.clone()

python/tests/test_curriculum.py

@@ -57,6 +57,7 @@ def test_older_gym_checkpoints_remain_loadable():
     snapshot = env.get_state()
     snapshot["version"] = 1
     del snapshot["curriculum"]
+    del snapshot["opponent"]
     env.set_state(snapshot)
     assert env.get_state()["native"] == snapshot["native"]
 

python/tests/test_gym_checkpoint.py

@@ -0,0 +1,79 @@
+import copy
+import json
+
+import numpy as np
+import pytest
+
+from gamesforai import make
+
+
+@pytest.mark.parametrize("opponent", ["random", "minimax:1", "minimax:3", "mcts:2"])
+def test_resume_repeats_policy_decisions_and_rng(opponent):
+    env = make("connect4", opponent=opponent)
+    env.reset(seed=87)
+    restored = make("connect4", opponent=opponent)
+    restored.set_state(json.loads(json.dumps(env.get_state())))
+    for _ in range(4):
+        action = int(np.flatnonzero(env.action_masks())[0])
+        first, second = env.step(action), restored.step(action)
+        np.testing.assert_array_equal(first[0], second[0])
+        assert first[1:4] == second[1:4]
+        assert env.get_state() == restored.get_state()
+        if first[2] or first[3]:
+            break
+
+
+@pytest.mark.parametrize("opponent", ["random", "minimax:2", "mcts:3"])
+def test_wrong_opponent_is_rejected_without_state_changes(opponent):
+    original = make("connect4", opponent="minimax:3")
+    original.reset(seed=3)
+    other = make("connect4", opponent=opponent)
+    other.reset(seed=5)
+    before = other.get_state()
+    with pytest.raises(ValueError, match="opponent"):
+        other.set_state(original.get_state())
+    assert other.get_state() == before
+
+
+def test_legacy_checkpoints_bind_only_to_random_opponent():
+    random = make("connect4")
+    random.reset(seed=81)
+    legacy = random.get_state()
+    del legacy["opponent"]
+    for version in (2, 1):
+        legacy["version"] = version
+        if version == 1:
+            del legacy["curriculum"]
+        random.set_state(legacy)
+        search = make("connect4", opponent="minimax:1")
+        with pytest.raises(ValueError, match="opponent"):
+            search.set_state(legacy)
+
+
+@pytest.mark.parametrize("field,value", [
+    ("version", True), ("seat", False), ("rng_seed", True), ("rng_seed", -2),
+])
+def test_malformed_metadata_is_atomic(field, value):
+    env = make("connect4")
+    env.reset(seed=4)
+    before = env.get_state()
+    invalid = copy.deepcopy(before)
+    invalid[field] = value
+    with pytest.raises(ValueError):
+        env.set_state(invalid)
+    assert env.get_state() == before
+
+
+def test_python_policy_state_remains_explicitly_caller_owned():
+    def policy(obs, info):
+        return int(np.flatnonzero(info["action_mask"])[0])
+
+    env = make("connect4", opponent=policy)
+    env.reset(seed=2)
+    state = env.get_state()
+    assert state["opponent"] == "python"
+    restored = make("connect4", opponent=policy)
+    restored.set_state(state)
+    assert restored.get_state() == state
+    with pytest.raises(ValueError, match="opponent"):
+        make("connect4").set_state(state)