Skip to content

Commit 7508406

Browse files
nosam1998nosam1998
authored
feat(web): add saved models and evaluation setup (#104)
1 parent 2161ffe commit 7508406

9 files changed

Lines changed: 1553 additions & 7 deletions

File tree

‎tests/browser/models.spec.mjs‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { expect, test } from "@playwright/test";
2+
import { readFile } from "node:fs/promises";
3+
4+
async function download(page, button) {
5+
const pending = page.waitForEvent("download");
6+
await page.getByRole("button", { name: button, exact: true }).click();
7+
const file = await pending;
8+
const output = test.info().outputPath(file.suggestedFilename());
9+
await file.saveAs(output);
10+
return { name: file.suggestedFilename(), text: await readFile(output, "utf8") };
11+
}
12+
async function open(page) {
13+
await page.goto("/#/models");
14+
await expect(page.getByRole("heading", { name: "Your models", exact: true })).toBeVisible();
15+
}
16+
17+
test("Python model configuration saves, reloads and exports a runnable evaluation", async ({ page }) => {
18+
await open(page);
19+
await page.getByLabel("Snapshot name", { exact: true }).fill("connect4-policy-v1");
20+
await page.getByLabel("Python factory", { exact: true }).fill("my_agent:build_policy");
21+
await page.getByLabel("Factory options (JSON)", { exact: true }).fill('{"checkpoint":"weights/v1.pt","device":"cpu"}');
22+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
23+
await expect(page.getByRole("status").filter({ hasText: "Configuration saved in this browser." })).toBeVisible();
24+
const exported = await download(page, "Download agent.json");
25+
expect(exported.name).toBe("agent.json");
26+
expect(JSON.parse(exported.text)).toEqual({ id: "connect4-policy-v1", type: "python", factory: "my_agent:build_policy", params: { checkpoint: "weights/v1.pt", device: "cpu" } });
27+
await page.reload();
28+
await page.getByRole("button", { name: "Load connect4-policy-v1", exact: true }).click();
29+
await expect(page.getByLabel("Factory options (JSON)", { exact: true })).toHaveValue(/weights\/v1.pt/);
30+
await page.getByLabel("Evaluation game", { exact: true }).selectOption("tictactoe");
31+
await expect(page.getByLabel("Evaluation game", { exact: true }).locator("option")).toHaveCount(3);
32+
await page.getByLabel("Evaluation opponent", { exact: true }).selectOption("minimax");
33+
await page.getByLabel("Opponent level", { exact: true }).selectOption("3");
34+
await page.getByLabel("Games per opponent", { exact: true }).fill("20");
35+
await page.getByLabel("Save training trajectories", { exact: true }).check();
36+
await page.getByLabel("Dataset format", { exact: true }).selectOption("parquet");
37+
await expect(page.getByTestId("evaluation-command")).toHaveText("gfa tournament --game tictactoe --agents agent.json --opponents minimax:3 --games 20 --seed 42 --report tournament-report.json --trajectories training-run-001 --format parquet");
38+
await page.getByText("Try a single episode first", { exact: true }).click();
39+
const script = await download(page, "Download episode script");
40+
expect(script.name).toBe("try-model.py");
41+
expect(script.text).toContain('env = make("tictactoe", opponent="random")');
42+
expect(script.text).toContain('load_document("agent.json")');
43+
await page.getByRole("button", { name: "Remove connect4-policy-v1", exact: true }).click();
44+
await page.reload();
45+
await expect(page.getByRole("article", { name: "connect4-policy-v1", exact: true })).toHaveCount(0);
46+
});
47+
48+
test("HTTP model exports environment-based credentials without contacting the endpoint", async ({ page }) => {
49+
const external = [];
50+
page.on("request", request => { if (new URL(request.url()).hostname === "model.example") external.push(request.url()); });
51+
await open(page);
52+
await page.getByLabel("Snapshot name", { exact: true }).fill("http-model-v2");
53+
await page.getByRole("radio", { name: /HTTP model/ }).check();
54+
await page.getByLabel("API root", { exact: true }).fill("https://model.example/v1/");
55+
await page.getByLabel("Model identifier", { exact: true }).fill("my-model");
56+
await page.getByLabel("API key environment variable (optional)", { exact: true }).fill("MY_MODEL_KEY");
57+
await page.getByText("Response settings", { exact: true }).click();
58+
await page.getByLabel("Use structured output", { exact: true }).uncheck();
59+
await page.getByLabel("Output tokens", { exact: true }).fill("512");
60+
await page.getByLabel("Temperature", { exact: true }).fill("0.3");
61+
const file = await download(page, "Download agent.json");
62+
expect(JSON.parse(file.text)).toEqual({ id: "http-model-v2", type: "chat", base_url: "https://model.example/v1", model: "my-model", api_key_env: "MY_MODEL_KEY", timeout: 60, max_tokens: 512, temperature: 0.3, structured: false });
63+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
64+
expect(external).toEqual([]);
65+
expect(await page.evaluate(() => localStorage.getItem("gamesforai.models.v1"))).not.toContain('"api_key":');
66+
});
67+
68+
test("invalid settings are explained and cannot produce a run configuration", async ({ page }) => {
69+
await open(page);
70+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
71+
await expect(page.getByRole("alert")).toContainText("Give this snapshot a name");
72+
await page.getByLabel("Snapshot name", { exact: true }).fill("test-policy");
73+
await page.getByLabel("Factory options (JSON)", { exact: true }).fill('{"seed":99}');
74+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
75+
await expect(page.getByRole("alert")).toContainText("Remove seed and seat");
76+
await expect(page.getByRole("button", { name: "Download agent.json", exact: true })).toBeDisabled();
77+
await page.getByRole("radio", { name: /HTTP model/ }).check();
78+
await page.getByLabel("API root", { exact: true }).fill("https://user:secret@model.example/v1");
79+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
80+
await expect(page.getByRole("alert")).toContainText("without credentials");
81+
await page.getByLabel("API root", { exact: true }).fill("http://localhost:8000/v1/chat/completions");
82+
await expect(page.getByRole("alert")).toContainText("Use the API root ending in /v1");
83+
await page.getByLabel("API root", { exact: true }).fill("http://localhost:8000/v1");
84+
await page.getByLabel("Model identifier", { exact: true }).fill("local-model");
85+
await page.getByLabel("Games per opponent", { exact: true }).fill("3");
86+
await expect(page.getByRole("alert")).toContainText("even number of games");
87+
await expect(page.getByTestId("evaluation-command")).toHaveCount(0);
88+
});
89+
90+
test("mobile model setup supports keyboard, download and unavailable browser storage", async ({ page, context }) => {
91+
await page.setViewportSize({ width: 360, height: 800 });
92+
await context.addInitScript(() => {
93+
Storage.prototype.getItem = () => { throw new Error("blocked"); };
94+
Storage.prototype.setItem = () => { throw new Error("blocked"); };
95+
});
96+
await open(page);
97+
await page.getByRole("button", { name: "New", exact: true }).click();
98+
await expect(page.getByLabel("Snapshot name", { exact: true })).toBeFocused();
99+
await page.getByLabel("Snapshot name", { exact: true }).fill("mobile-policy");
100+
await page.getByRole("button", { name: "Save configuration", exact: true }).click();
101+
await expect(page.getByRole("alert").filter({ hasText: "Browser storage is unavailable or full." })).toBeVisible();
102+
expect(JSON.parse((await download(page, "Download agent.json")).text).id).toBe("mobile-policy");
103+
expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true);
104+
});
105+
106+
test("the library links to model setup and clipboard failure leaves selectable config", async ({ page }) => {
107+
await page.goto("/");
108+
await page.getByRole("link", { name: "Set up a model", exact: false }).click();
109+
await page.getByLabel("Snapshot name", { exact: true }).fill("clipboard-policy");
110+
await page.getByText("Configuration preview", { exact: true }).click();
111+
await page.evaluate(() => { Object.defineProperty(navigator, "clipboard", { value: undefined, configurable: true }); });
112+
await page.getByRole("button", { name: "Copy configuration", exact: true }).click();
113+
await expect(page.getByRole("status").filter({ hasText: "Copy unavailable." })).toBeVisible();
114+
await expect(page.getByTestId("model-config")).toContainText("clipboard-policy");
115+
});

‎web/MODELS.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Model configurations in the browser
2+
3+
Open **Models** from the navigation or **Set up a model** from the library.
4+
Name each frozen model snapshot, then choose a Python factory or an
5+
OpenAI-compatible HTTP model server. Saved configurations stay in this browser;
6+
download `agent.json` to use a configuration in the Python runner.
7+
8+
Python factories receive `seed`, `seat`, and your JSON options and return a
9+
policy that selects a legal integer action index. HTTP configurations name
10+
an API root, model identifier and optional environment variable for a credential.
11+
The runner reads that variable at execution time. The form validates settings
12+
without importing Python code or contacting a model server.
13+
14+
The evaluation builder uses the live game catalog and supports the runner's
15+
sequential two-player games, balanced seats, reproducible seeds, random/minimax/
16+
MCTS opponents, and optional JSONL or Parquet trajectory export. Download the
17+
configuration into your working folder, install the indicated SDK extras from
18+
your checkout, and run the generated command. The optional episode script
19+
checks one model against random moves before a longer evaluation.
20+
21+
Runner evaluations use native environments and save local reports and datasets.
22+
The browser's Live and History pages display matches persisted by the server.
23+
See [the Python SDK](../python/README.md) for training interfaces and
24+
[the tournament guide](../examples/tournament-cli.md) for reporting details.
25+
26+
Storage is bounded to 12 configurations and 128 KiB. An unreadable or blocked
27+
browser store leaves configuration download available. Exported agent documents
28+
use the existing runner schema and do not include ratings or calibration claims.

‎web/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Browser application
22

3+
See [model setup](MODELS.md) for saved Python/HTTP configurations and evaluation exports.
4+
35
The library reads the installed game catalog, play guides and opponent ladders
46
from the running server. Each guide can be copied as a model prompt. The app
57
includes search, keyboard navigation, error recovery and a layout down to 360px.

‎web/apps/site/src/App.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { MatchSetup } from "./setup";
44
import { MatchPage } from "./match";
55
import { HistoryPage } from "./history";
66
import { Spectator } from "./spectate";
7+
import { ModelsPage } from "./models";
78
import { ReplayPage } from "./replay";
89

910
export function App() {
@@ -66,6 +67,8 @@ export function App() {
6667
}
6768
} else if (path === "/history" || path === "/live") {
6869
page = <HistoryPage key={path} live={path === "/live"} />;
70+
} else if (path === "/models") {
71+
page = <ModelsPage />;
6972
} else if (path !== "/") {
7073
page = (
7174
<section className="feedback">
@@ -102,6 +105,12 @@ export function App() {
102105
<a href="#/live" aria-current={path === "/live" ? "page" : undefined}>
103106
Live
104107
</a>
108+
<a
109+
href="#/models"
110+
aria-current={path === "/models" ? "page" : undefined}
111+
>
112+
Models
113+
</a>
105114
<a
106115
href="#/history"
107116
aria-current={path === "/history" ? "page" : undefined}

‎web/apps/site/src/export-controls.tsx‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { useState } from "react";
2+
3+
export function downloadText(
4+
name: string,
5+
content: string,
6+
type = "application/json",
7+
) {
8+
const url = URL.createObjectURL(new Blob([content], { type }));
9+
const link = document.createElement("a");
10+
link.href = url;
11+
link.download = name;
12+
link.click();
13+
window.setTimeout(() => URL.revokeObjectURL(url), 1000);
14+
}
15+
16+
export function CopyButton({ text, label }: { text: string; label: string }) {
17+
const [message, setMessage] = useState("");
18+
return (
19+
<span className="copy-control">
20+
<button
21+
type="button"
22+
className="secondary"
23+
onClick={() => {
24+
void (async () => {
25+
try {
26+
await navigator.clipboard.writeText(text);
27+
setMessage("Copied.");
28+
} catch {
29+
setMessage("Copy unavailable. Select and copy the text instead.");
30+
}
31+
})();
32+
}}
33+
>
34+
{label}
35+
</button>
36+
<span role="status">{message}</span>
37+
</span>
38+
);
39+
}

‎web/apps/site/src/library.tsx‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,8 @@ export function Library() {
139139
evaluate with the same game environments.
140140
</p>
141141
</div>
142-
<a
143-
className="button secondary"
144-
href="https://github.com/CurtisTechSolutions/GamesForAI/blob/main/python/README.md"
145-
target="_blank"
146-
rel="noreferrer"
147-
>
148-
Python SDK <span aria-hidden="true">↗</span>
142+
<a className="button secondary" href="#/models">
143+
Set up a model <span aria-hidden="true">→</span>
149144
</a>
150145
</aside>
151146
</>

0 commit comments

Comments
 (0)