Merge pull request #87 from CurtisTechSolutions/feat/python-model-ratings

feat(training): rate local model snapshots with Glicko-2

Author: nosam1998

Cargo.lock

@@ -16,6 +16,7 @@ extension-module = ["pyo3/extension-module"]
 [dependencies]
 gfa-core.workspace = true
 gfa-opponents.workspace = true
+gfa-service.workspace = true
 gfa-engine-uci.workspace = true
 gfa-games.workspace = true
 pyo3.workspace = true

crates/apps/gfa-py/Cargo.toml

@@ -1,4 +1,5 @@
 //! Native Python training bridge. Typed engine states stay in Rust between steps.
+mod ratings;
 mod uci;
 mod vector;
 
@@ -247,6 +248,8 @@ fn position_set_data(name: &str) -> PyResult<(String, String)> {
 /// GamesForAI's native engine bridge.
 #[pymodule]
 mod _native {
+    #[pymodule_export]
+    use super::ratings::{rating_pair_json, rating_period_json, rating_validate_json};
     #[pymodule_export]
     use super::{games, position_set_data, uci::NativeUciPool, vector::NativeVectorEnv, NativeEnv};
 }

crates/apps/gfa-py/src/lib.rs

@@ -0,0 +1,64 @@
+use crate::error;
+use gfa_service::ratings::{rate_pair, RatedResult, Rating};
+use pyo3::prelude::*;
+
+fn parse(text: &str) -> PyResult<Rating> {
+    if text.len() > 4096 {
+        return Err(error("rating payload exceeds 4096 bytes"));
+    }
+    let rating: Rating = serde_json::from_str(text).map_err(error)?;
+    rating.validate().map_err(error)
+}
+
+/// Validate a serialized rating without updating its uncertainty.
+#[pyfunction]
+pub(crate) fn rating_validate_json(rating: &str) -> PyResult<()> {
+    parse(rating).map(|_| ())
+}
+
+/// Apply the same pure rating period used by the service domain.
+#[pyfunction]
+#[pyo3(signature = (rating, results, tau=0.5))]
+pub(crate) fn rating_period_json(
+    py: Python<'_>,
+    rating: &str,
+    results: &str,
+    tau: f64,
+) -> PyResult<String> {
+    if results.len() > 1024 * 1024 {
+        return Err(error("rating results exceed 1 MiB"));
+    }
+    let rating = parse(rating)?;
+    let inputs: Vec<(Rating, f64)> = serde_json::from_str(results).map_err(error)?;
+    py.detach(|| {
+        let results: Vec<_> = inputs
+            .into_iter()
+            .map(|(opponent, score)| RatedResult { opponent, score })
+            .collect();
+        let updated = rating.update_period(&results, tau).map_err(error)?;
+        serde_json::to_string(&updated).map_err(error)
+    })
+}
+
+/// Rate both seats from the original values, optionally preserving calibrated anchors.
+#[pyfunction]
+#[pyo3(signature = (left, right, score, fixed_left=false, fixed_right=false, tau=0.5))]
+pub(crate) fn rating_pair_json(
+    left: &str,
+    right: &str,
+    score: f64,
+    fixed_left: bool,
+    fixed_right: bool,
+    tau: f64,
+) -> PyResult<String> {
+    let pair = rate_pair(
+        parse(left)?,
+        parse(right)?,
+        score,
+        fixed_left,
+        fixed_right,
+        tau,
+    )
+    .map_err(error)?;
+    serde_json::to_string(&pair).map_err(error)
+}

crates/apps/gfa-py/src/ratings.rs

@@ -0,0 +1,25 @@
+# Rating model snapshots
+
+```python
+from gamesforai.ratings import Rating, rate_pair
+
+model_v1, model_v2 = Rating(), Rating()
+model_v1, model_v2 = rate_pair(model_v1, model_v2, 1.0)  # v1 won
+print(model_v1.rating, model_v1.interval95)
+checkpoint = model_v1.to_dict()
+restored = Rating(**checkpoint)
+```
+
+These Python functions call the same Rust Glicko-2 implementation as the service
+domain. Use stable model snapshot identifiers and separate rating pools for
+each game, rules configuration and assistance policy. Start uncalibrated
+players with the default prior. Fixed anchors require measured calibration.
+
+Use score 1/0.5/0 for win/draw/loss. Exclude truncated and failed games instead
+of treating them as draws. For a whole rating period, pass each player's
+results with opponents' pre-period ratings to `Rating.update`; do not substitute
+ratings that were already updated during that period. Calling `update()` with
+no games represents one explicit inactive period and increases uncertainty.
+
+Rating dictionaries are portable checkpoint data. This local API does not
+write a server leaderboard or manufacture calibrated engine ratings.

examples/model-ratings.md

@@ -0,0 +1,47 @@
+"""Shared native Glicko-2 calculations for local model leagues."""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+
+from ._native import rating_pair_json, rating_period_json, rating_validate_json
+
+
+def _dump(value):
+    return json.dumps(value, allow_nan=False)
+
+
+@dataclass(frozen=True)
+class Rating:
+    rating: float = 1500.0
+    rd: float = 350.0
+    volatility: float = 0.06
+    games_played: int = 0
+
+    def __post_init__(self):
+        rating_validate_json(_dump(asdict(self)))
+
+    @property
+    def interval95(self):
+        """Approximate rating interval; this is not a win-rate confidence bound."""
+        return (self.rating - 2 * self.rd, self.rating + 2 * self.rd)
+
+    def update(self, results=(), *, tau=0.5):
+        """Apply one period from (pre-period opponent Rating, score) pairs."""
+        payload = [(asdict(opponent), score) for opponent, score in results]
+        return Rating(**json.loads(rating_period_json(_dump(asdict(self)), _dump(payload), tau)))
+
+    def to_dict(self):
+        return asdict(self)
+
+
+def rate_pair(left, right, score, *, fixed_left=False, fixed_right=False, tau=0.5):
+    """Return both new ratings without mutating either pre-match rating."""
+    if isinstance(score, bool):
+        raise ValueError("score must be numeric 0, 0.5 or 1")
+    if type(fixed_left) is not bool or type(fixed_right) is not bool:
+        raise ValueError("fixed anchor flags must be boolean")
+    result = json.loads(rating_pair_json(
+        _dump(asdict(left)), _dump(asdict(right)), score, fixed_left, fixed_right, tau
+    ))
+    return tuple(Rating(**rating) for rating in result)

python/gamesforai/ratings.py

@@ -0,0 +1,51 @@
+import json
+
+import pytest
+
+from gamesforai.ratings import Rating, rate_pair
+
+
+def test_native_rating_reference_and_json_checkpoint():
+    current = Rating(rd=200)
+    updated = current.update([(Rating(rating=1400, rd=30), 1),
+                              (Rating(rating=1550, rd=100), 0),
+                              (Rating(rating=1700, rd=300), 0)])
+    assert updated.rating == pytest.approx(1464.06, abs=0.01)
+    assert updated.rd == pytest.approx(151.52, abs=0.01)
+    assert updated.games_played == 3
+    assert Rating(**json.loads(json.dumps(updated.to_dict()))) == updated
+    assert current.rd == 200
+
+
+def test_pair_symmetry_anchor_and_inactive_period():
+    original = Rating()
+    winner, loser = rate_pair(original, original, 1)
+    assert winner.rating + loser.rating == pytest.approx(3000)
+    assert winner.games_played == loser.games_played == 1
+    idle = winner.update()
+    assert idle.rating == winner.rating and idle.rd > winner.rd
+    anchor = Rating(rating=1800, rd=0)
+    learner, fixed = rate_pair(original, anchor, 1, fixed_right=True)
+    assert learner.rating > winner.rating
+    assert fixed.rating == anchor.rating and fixed.rd == 0
+    assert original.interval95 == (800, 2200)
+
+
+@pytest.mark.parametrize("kwargs", [
+    {"rd": -1}, {"volatility": 0}, {"rating": float("nan")},
+    {"games_played": -1}, {"games_played": True},
+])
+def test_invalid_rating_inputs(kwargs):
+    with pytest.raises(ValueError):
+        Rating(**kwargs)
+
+
+def test_invalid_results_and_anchor_flags():
+    original = Rating()
+    for score in (-1, 0.25, 2, float("inf")):
+        with pytest.raises(ValueError):
+            rate_pair(original, original, score)
+    with pytest.raises(ValueError):
+        rate_pair(original, original, 1, fixed_left=1)
+    with pytest.raises(ValueError):
+        original.update([(original, 1)], tau=0)