From 1d0eb27aaab8cf9e10fa38a0482e35f66f443a30 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:49:39 +0200 Subject: [PATCH 1/4] Add lean NeMo Gym taskset --- environments/README.md | 1 - tests/v1/test_nemo_gym.py | 107 ++++++++++++++ verifiers/v1/README.md | 16 +++ verifiers/v1/tasksets/__init__.py | 8 +- verifiers/v1/tasksets/nemo_gym/__init__.py | 11 ++ verifiers/v1/tasksets/nemo_gym/taskset.py | 157 +++++++++++++++++++++ 6 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 tests/v1/test_nemo_gym.py create mode 100644 verifiers/v1/tasksets/nemo_gym/__init__.py create mode 100644 verifiers/v1/tasksets/nemo_gym/taskset.py diff --git a/environments/README.md b/environments/README.md index da47ab0b0d..48faad1463 100644 --- a/environments/README.md +++ b/environments/README.md @@ -59,7 +59,6 @@ This folder contains installable example environments that showcase common usage - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards. - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint. - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup. - - **nemo_gym_env**: Minimal v1 example that wraps a packaged NeMo Gym task with `NeMoGymTaskset` and `NeMoGymHarness`. - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls. - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary. diff --git a/tests/v1/test_nemo_gym.py b/tests/v1/test_nemo_gym.py new file mode 100644 index 0000000000..272edd42de --- /dev/null +++ b/tests/v1/test_nemo_gym.py @@ -0,0 +1,107 @@ +import json +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from verifiers.v1.env import EnvConfig, Environment +from verifiers.v1.loaders import default_harness_id, taskset_config_type +from verifiers.v1.tasksets.nemo_gym import ( + NeMoGymConfig, + NeMoGymTask, + NeMoGymTaskset, +) +from verifiers.v1.types import SystemMessage, UserMessage + + +def test_nemo_gym_loads_packaged_example(tmp_path, monkeypatch) -> None: + data = ( + tmp_path + / "resources_servers" + / "example_single_tool_call" + / "data" + / "example.jsonl" + ) + data.parent.mkdir(parents=True) + row = { + "responses_create_params": { + "input": [ + {"role": "developer", "content": "Be helpful."}, + {"role": "user", "content": "What's it like in SF?"}, + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + } + } + data.write_text(json.dumps(row) + "\n") + monkeypatch.setitem(sys.modules, "nemo_gym", SimpleNamespace(PARENT_DIR=tmp_path)) + + [task] = NeMoGymTaskset(NeMoGymConfig()).load_tasks() + + assert task.name == "example_single_tool_call:0" + assert task.prompt == [ + SystemMessage(content="Be helpful."), + UserMessage(content="What's it like in SF?"), + ] + assert "nemo_gym_call" in task.system_prompt + assert "get_weather" in task.system_prompt + assert task.nemo_gym_row == row + + +def test_nemo_gym_uses_the_standard_verifiers_harness() -> None: + env = Environment( + EnvConfig.model_validate( + { + "taskset": {"id": "nemo_gym"}, + "harness": {}, + } + ) + ) + task = NeMoGymTask( + idx=0, + prompt="hello", + nemo_gym_row={"responses_create_params": {"tools": [{"name": "tool"}]}}, + ) + + [toolset] = env.taskset.tools(task) + + assert env.harness.config.id == "default" + assert toolset.server_name == "nemo_gym" + assert toolset.config.nemo_env == "example_single_tool_call" + + +@pytest.mark.asyncio +async def test_nemo_gym_forwards_tools_to_the_resource_server() -> None: + taskset = NeMoGymTaskset(NeMoGymConfig()) + task = NeMoGymTask( + idx=0, + prompt="hello", + nemo_gym_row={"responses_create_params": {"tools": [{"name": "get_weather"}]}}, + ) + [toolset] = taskset.tools(task) + toolset.tool_names = {"get_weather"} + response = Mock() + response.json.return_value = {"weather_description": "cold"} + toolset.client = AsyncMock() + toolset.client.post.return_value = response + + result = await toolset.call("get_weather", {"city": "sf"}) + + toolset.client.post.assert_awaited_once_with("/get_weather", json={"city": "sf"}) + response.raise_for_status.assert_called_once() + assert result == {"weather_description": "cold"} + + +def test_nemo_gym_is_only_a_taskset() -> None: + assert default_harness_id("nemo_gym") == "default" + assert taskset_config_type("nemo_gym") is NeMoGymConfig diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index f2d24586c2..17030e00ad 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -112,6 +112,22 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex # the codex CLI agent ``` +### NeMo Gym + +The built-in `nemo_gym` taskset defaults to NeMo Gym's own +`example_single_tool_call` quickstart. It loads the example JSONL as ordinary typed tasks; +the selected Verifiers harness owns the agent loop, while NeMo Gym's packaged resource +server executes the declared tools. Model and tool messages use the normal Verifiers trace. +There is no NeMo-specific harness or program. + +```bash +uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 +``` + +The taskset does not interpret the example's weather behavior. It forwards declared calls +to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool. +The standard Verifiers harness drives the model/tool loop and captures the trace. + ### Swappable runtime Where code runs, behind one `Runtime` contract — the same contract backs the harness diff --git a/verifiers/v1/tasksets/__init__.py b/verifiers/v1/tasksets/__init__.py index 956f606325..3df6565c46 100644 --- a/verifiers/v1/tasksets/__init__.py +++ b/verifiers/v1/tasksets/__init__.py @@ -5,5 +5,11 @@ `verifiers.v1.tasksets.textarena`.""" from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset +from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset -__all__ = ["HarborConfig", "HarborTaskset"] +__all__ = [ + "HarborConfig", + "HarborTaskset", + "NeMoGymConfig", + "NeMoGymTaskset", +] diff --git a/verifiers/v1/tasksets/nemo_gym/__init__.py b/verifiers/v1/tasksets/nemo_gym/__init__.py new file mode 100644 index 0000000000..6485d75de0 --- /dev/null +++ b/verifiers/v1/tasksets/nemo_gym/__init__.py @@ -0,0 +1,11 @@ +from verifiers.v1.tasksets.nemo_gym.taskset import ( + NeMoGymConfig, + NeMoGymTask, + NeMoGymTaskset, +) + +__all__ = [ + "NeMoGymConfig", + "NeMoGymTask", + "NeMoGymTaskset", +] diff --git a/verifiers/v1/tasksets/nemo_gym/taskset.py b/verifiers/v1/tasksets/nemo_gym/taskset.py new file mode 100644 index 0000000000..e238113a66 --- /dev/null +++ b/verifiers/v1/tasksets/nemo_gym/taskset.py @@ -0,0 +1,157 @@ +"""Run a packaged NeMo Gym dataset with any MCP-capable Verifiers harness.""" + +import importlib +import json +from pathlib import Path +from typing import Any + +import httpx + +from verifiers.v1.decorators import tool +from verifiers.v1.dialects.responses import ResponsesDialect +from verifiers.v1.mcp import Toolset, ToolsetConfig +from verifiers.v1.task import Task +from verifiers.v1.taskset import Taskset, TasksetConfig + +DEFAULT_ENV = "example_single_tool_call" + + +class NeMoGymTask(Task): + nemo_gym_row: dict[str, Any] + + +class NeMoGymConfig(TasksetConfig): + nemo_env: str = DEFAULT_ENV + data_name: str = "example.jsonl" + + +class _NeMoGymToolsConfig(ToolsetConfig): + nemo_env: str + + +class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): + """Expose a NeMo Gym resource server through one generic MCP call.""" + + TOOL_PREFIX = "nemo_gym" + + async def setup(self) -> None: + from nemo_gym import PARENT_DIR + from nemo_gym.base_resources_server import SimpleResourcesServer + from nemo_gym.config_types import BaseServerConfig + from nemo_gym.global_config import ( + GlobalConfigDictParser, + GlobalConfigDictParserConfig, + ) + from nemo_gym.server_utils import ServerClient + from omegaconf import OmegaConf + + root = Path(PARENT_DIR) + path = ( + root + / "resources_servers" + / self.config.nemo_env + / "configs" + / f"{self.config.nemo_env}.yaml" + ) + parser = GlobalConfigDictParser() + config = parser.parse_no_environment( + OmegaConf.merge( + OmegaConf.load(path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + resource = next( + server + for server in parser.filter_for_server_instance_configs(config) + if server.SERVER_TYPE == "resources_servers" + ) + entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem + module = importlib.import_module( + f"resources_servers.{self.config.nemo_env}.{entrypoint}" + ) + server_cls: Any = next( + value + for value in vars(module).values() + if isinstance(value, type) + and issubclass(value, SimpleResourcesServer) + and value.__module__ == module.__name__ + ) + server_config = server_cls.model_fields["config"].annotation.model_validate( + { + "name": resource.name, + **resource.get_inner_run_server_config().model_dump(), + } + ) + server = server_cls( + config=server_config, + server_client=ServerClient( + head_server_config=BaseServerConfig.model_validate( + config["head_server"] + ), + global_config_dict=config, + ), + ) + self.client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=server.setup_webserver()), + base_url="http://nemo-gym", + ) + + async def setup_task(self, task: NeMoGymTask) -> None: + self.tool_names = { + spec["name"] + for spec in task.nemo_gym_row["responses_create_params"].get("tools", []) + } + response = await self.client.post("/seed_session", json=task.nemo_gym_row) + response.raise_for_status() + + @tool + async def call(self, name: str, arguments: dict[str, Any]) -> Any: + """Call a tool declared by the current NeMo Gym task.""" + if name not in self.tool_names: + raise ValueError(f"unknown NeMo Gym tool: {name}") + response = await self.client.post(f"/{name}", json=arguments) + response.raise_for_status() + return response.json() + + +class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]): + def load_tasks(self) -> list[NeMoGymTask]: + try: + from nemo_gym import PARENT_DIR + except ImportError as exc: + raise ImportError( + "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`." + ) from exc + + path = ( + Path(PARENT_DIR) + / "resources_servers" + / self.config.nemo_env + / "data" + / self.config.data_name + ) + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + dialect = ResponsesDialect() + return [ + NeMoGymTask( + idx=idx, + name=f"{self.config.nemo_env}:{idx}", + prompt=dialect.parse_request(row["responses_create_params"])[0], + system_prompt=( + "Call `nemo_gym_call` with the matching tool name and arguments. " + f"Available NeMo Gym tools: " + f"{json.dumps(row['responses_create_params'].get('tools', []))}" + ), + nemo_gym_row=row, + ) + for idx, row in enumerate(rows) + ] + + def tools(self, task: NeMoGymTask) -> list[Toolset]: + if not task.nemo_gym_row["responses_create_params"].get("tools"): + return [] + return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))] + + +if __name__ == "__main__": + _NeMoGymTools.run() From 59b494fa9e810a08df2e24a1935ff2a1775d9e60 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:51:15 +0200 Subject: [PATCH 2/4] Remove NeMo Gym test --- tests/v1/test_nemo_gym.py | 107 -------------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 tests/v1/test_nemo_gym.py diff --git a/tests/v1/test_nemo_gym.py b/tests/v1/test_nemo_gym.py deleted file mode 100644 index 272edd42de..0000000000 --- a/tests/v1/test_nemo_gym.py +++ /dev/null @@ -1,107 +0,0 @@ -import json -import sys -from types import SimpleNamespace -from unittest.mock import AsyncMock, Mock - -import pytest - -from verifiers.v1.env import EnvConfig, Environment -from verifiers.v1.loaders import default_harness_id, taskset_config_type -from verifiers.v1.tasksets.nemo_gym import ( - NeMoGymConfig, - NeMoGymTask, - NeMoGymTaskset, -) -from verifiers.v1.types import SystemMessage, UserMessage - - -def test_nemo_gym_loads_packaged_example(tmp_path, monkeypatch) -> None: - data = ( - tmp_path - / "resources_servers" - / "example_single_tool_call" - / "data" - / "example.jsonl" - ) - data.parent.mkdir(parents=True) - row = { - "responses_create_params": { - "input": [ - {"role": "developer", "content": "Be helpful."}, - {"role": "user", "content": "What's it like in SF?"}, - ], - "tools": [ - { - "type": "function", - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - } - ], - } - } - data.write_text(json.dumps(row) + "\n") - monkeypatch.setitem(sys.modules, "nemo_gym", SimpleNamespace(PARENT_DIR=tmp_path)) - - [task] = NeMoGymTaskset(NeMoGymConfig()).load_tasks() - - assert task.name == "example_single_tool_call:0" - assert task.prompt == [ - SystemMessage(content="Be helpful."), - UserMessage(content="What's it like in SF?"), - ] - assert "nemo_gym_call" in task.system_prompt - assert "get_weather" in task.system_prompt - assert task.nemo_gym_row == row - - -def test_nemo_gym_uses_the_standard_verifiers_harness() -> None: - env = Environment( - EnvConfig.model_validate( - { - "taskset": {"id": "nemo_gym"}, - "harness": {}, - } - ) - ) - task = NeMoGymTask( - idx=0, - prompt="hello", - nemo_gym_row={"responses_create_params": {"tools": [{"name": "tool"}]}}, - ) - - [toolset] = env.taskset.tools(task) - - assert env.harness.config.id == "default" - assert toolset.server_name == "nemo_gym" - assert toolset.config.nemo_env == "example_single_tool_call" - - -@pytest.mark.asyncio -async def test_nemo_gym_forwards_tools_to_the_resource_server() -> None: - taskset = NeMoGymTaskset(NeMoGymConfig()) - task = NeMoGymTask( - idx=0, - prompt="hello", - nemo_gym_row={"responses_create_params": {"tools": [{"name": "get_weather"}]}}, - ) - [toolset] = taskset.tools(task) - toolset.tool_names = {"get_weather"} - response = Mock() - response.json.return_value = {"weather_description": "cold"} - toolset.client = AsyncMock() - toolset.client.post.return_value = response - - result = await toolset.call("get_weather", {"city": "sf"}) - - toolset.client.post.assert_awaited_once_with("/get_weather", json={"city": "sf"}) - response.raise_for_status.assert_called_once() - assert result == {"weather_description": "cold"} - - -def test_nemo_gym_is_only_a_taskset() -> None: - assert default_harness_id("nemo_gym") == "default" - assert taskset_config_type("nemo_gym") is NeMoGymConfig From 1850666644c874ab00ce719967cfa3bbfd18f4c6 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:04:56 +0200 Subject: [PATCH 3/4] Support NeMo Gym resource configs --- verifiers/v1/README.md | 9 ++-- verifiers/v1/tasksets/nemo_gym/taskset.py | 57 ++++++++++++++--------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index 17030e00ad..cb0529514f 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -122,11 +122,14 @@ There is no NeMo-specific harness or program. ```bash uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 +uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 \ + --taskset.resource-server workplace_assistant ``` -The taskset does not interpret the example's weather behavior. It forwards declared calls -to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool. -The standard Verifiers harness drives the model/tool loop and captures the trace. +The taskset does not interpret the example behavior. It loads rows from the configured +`resource_server`, uses `config_name` only when the server's YAML has a different name, +and forwards declared calls through one generic `nemo_gym_call` tool. The standard +Verifiers harness drives the model/tool loop and captures the trace. ### Swappable runtime diff --git a/verifiers/v1/tasksets/nemo_gym/taskset.py b/verifiers/v1/tasksets/nemo_gym/taskset.py index e238113a66..e4762d64fa 100644 --- a/verifiers/v1/tasksets/nemo_gym/taskset.py +++ b/verifiers/v1/tasksets/nemo_gym/taskset.py @@ -13,7 +13,7 @@ from verifiers.v1.task import Task from verifiers.v1.taskset import Taskset, TasksetConfig -DEFAULT_ENV = "example_single_tool_call" +DEFAULT_RESOURCE_SERVER = "example_single_tool_call" class NeMoGymTask(Task): @@ -21,12 +21,14 @@ class NeMoGymTask(Task): class NeMoGymConfig(TasksetConfig): - nemo_env: str = DEFAULT_ENV + resource_server: str = DEFAULT_RESOURCE_SERVER + config_name: str | None = None data_name: str = "example.jsonl" class _NeMoGymToolsConfig(ToolsetConfig): - nemo_env: str + resource_server: str + config_name: str | None = None class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): @@ -35,23 +37,27 @@ class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): TOOL_PREFIX = "nemo_gym" async def setup(self) -> None: - from nemo_gym import PARENT_DIR - from nemo_gym.base_resources_server import SimpleResourcesServer - from nemo_gym.config_types import BaseServerConfig - from nemo_gym.global_config import ( - GlobalConfigDictParser, - GlobalConfigDictParserConfig, - ) - from nemo_gym.server_utils import ServerClient - from omegaconf import OmegaConf - - root = Path(PARENT_DIR) + nemo_gym = importlib.import_module("nemo_gym") + SimpleResourcesServer = importlib.import_module( + "nemo_gym.base_resources_server" + ).SimpleResourcesServer + BaseServerConfig = importlib.import_module( + "nemo_gym.config_types" + ).BaseServerConfig + global_config = importlib.import_module("nemo_gym.global_config") + GlobalConfigDictParser = global_config.GlobalConfigDictParser + GlobalConfigDictParserConfig = global_config.GlobalConfigDictParserConfig + ServerClient = importlib.import_module("nemo_gym.server_utils").ServerClient + OmegaConf = importlib.import_module("omegaconf").OmegaConf + + root = Path(nemo_gym.PARENT_DIR) + config_name = self.config.config_name or self.config.resource_server path = ( root / "resources_servers" - / self.config.nemo_env + / self.config.resource_server / "configs" - / f"{self.config.nemo_env}.yaml" + / f"{config_name}.yaml" ) parser = GlobalConfigDictParser() config = parser.parse_no_environment( @@ -67,7 +73,7 @@ async def setup(self) -> None: ) entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem module = importlib.import_module( - f"resources_servers.{self.config.nemo_env}.{entrypoint}" + f"resources_servers.{self.config.resource_server}.{entrypoint}" ) server_cls: Any = next( value @@ -117,16 +123,16 @@ async def call(self, name: str, arguments: dict[str, Any]) -> Any: class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]): def load_tasks(self) -> list[NeMoGymTask]: try: - from nemo_gym import PARENT_DIR + nemo_gym = importlib.import_module("nemo_gym") except ImportError as exc: raise ImportError( "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`." ) from exc path = ( - Path(PARENT_DIR) + Path(nemo_gym.PARENT_DIR) / "resources_servers" - / self.config.nemo_env + / self.config.resource_server / "data" / self.config.data_name ) @@ -135,7 +141,7 @@ def load_tasks(self) -> list[NeMoGymTask]: return [ NeMoGymTask( idx=idx, - name=f"{self.config.nemo_env}:{idx}", + name=f"{self.config.resource_server}:{idx}", prompt=dialect.parse_request(row["responses_create_params"])[0], system_prompt=( "Call `nemo_gym_call` with the matching tool name and arguments. " @@ -150,7 +156,14 @@ def load_tasks(self) -> list[NeMoGymTask]: def tools(self, task: NeMoGymTask) -> list[Toolset]: if not task.nemo_gym_row["responses_create_params"].get("tools"): return [] - return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))] + return [ + _NeMoGymTools( + _NeMoGymToolsConfig( + resource_server=self.config.resource_server, + config_name=self.config.config_name, + ) + ) + ] if __name__ == "__main__": From 0fb0045a1a70b327fb5b89f8f6500a0782f0874a Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:12:03 +0200 Subject: [PATCH 4/4] Add NeMo Gym workplace example env --- environments/README.md | 1 + environments/nemo_gym_workplace_v1/README.md | 21 +++++++ .../nemo_gym_workplace_v1/__init__.py | 3 + .../nemo_gym_workplace_v1/taskset.py | 20 +++++++ .../nemo_gym_workplace_v1/pyproject.toml | 13 +++++ verifiers/v1/README.md | 14 +++-- verifiers/v1/tasksets/nemo_gym/taskset.py | 57 +++++++------------ 7 files changed, 88 insertions(+), 41 deletions(-) create mode 100644 environments/nemo_gym_workplace_v1/README.md create mode 100644 environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py create mode 100644 environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py create mode 100644 environments/nemo_gym_workplace_v1/pyproject.toml diff --git a/environments/README.md b/environments/README.md index 48faad1463..759fa59ebd 100644 --- a/environments/README.md +++ b/environments/README.md @@ -59,6 +59,7 @@ This folder contains installable example environments that showcase common usage - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards. - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint. - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup. + - **nemo_gym_workplace_v1**: Thin v1 config over the reusable `NeMoGymTaskset`, pinned to NeMo Gym's `workplace_assistant` resource server. - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls. - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary. diff --git a/environments/nemo_gym_workplace_v1/README.md b/environments/nemo_gym_workplace_v1/README.md new file mode 100644 index 0000000000..dd74fc742b --- /dev/null +++ b/environments/nemo_gym_workplace_v1/README.md @@ -0,0 +1,21 @@ +# nemo-gym-workplace-v1 + +NeMo Gym's `workplace_assistant` example through the built-in `NeMoGymTaskset`. +This environment pins the taskset config only; the standard Verifiers harness owns the +rollout loop, and NeMo Gym's packaged resource server owns tool execution. + +## Develop + +Install + run: + +```bash +uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \ + --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1 +``` + +## Layout + +- `nemo_gym_workplace_v1/taskset.py` — a thin config wrapper over `NeMoGymTaskset`. + +Tune the packaged dataset from the CLI with `--taskset.data-name`, and use ordinary +Verifiers harness flags for rollout behavior. diff --git a/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py new file mode 100644 index 0000000000..d988f140f3 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py @@ -0,0 +1,3 @@ +from nemo_gym_workplace_v1.taskset import NemoGymWorkplaceTaskset + +__all__ = ["NemoGymWorkplaceTaskset"] diff --git a/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py new file mode 100644 index 0000000000..93106cd8b2 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py @@ -0,0 +1,20 @@ +from typing import Literal + +import verifiers.v1 as vf +from verifiers.v1.tasksets.nemo_gym import ( + NeMoGymConfig, + NeMoGymTask, + NeMoGymTaskset, +) + + +class NemoGymWorkplaceConfig(NeMoGymConfig): + nemo_env: Literal["workplace_assistant"] = "workplace_assistant" + + +class NemoGymWorkplaceTaskset(vf.Taskset[NeMoGymTask, NemoGymWorkplaceConfig]): + def load_tasks(self) -> list[NeMoGymTask]: + return NeMoGymTaskset(self.config).load_tasks() + + def tools(self, task: NeMoGymTask) -> list[vf.Toolset]: + return NeMoGymTaskset(self.config).tools(task) diff --git a/environments/nemo_gym_workplace_v1/pyproject.toml b/environments/nemo_gym_workplace_v1/pyproject.toml new file mode 100644 index 0000000000..a154848d67 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "nemo-gym-workplace-v1" +version = "0.1.0" +description = "nemo-gym-workplace-v1 - NeMo Gym workplace_assistant through the built-in taskset." +requires-python = ">=3.11" +dependencies = ["verifiers"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["nemo_gym_workplace_v1"] diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index cb0529514f..434e0001fc 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -82,6 +82,7 @@ Taskset examples (the `*_v1` packages under `environments/`): | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge | | `deepwiki-v1` | an **existing remote** tool server, by URL | | `wordle-v1` | configuring the vendored `textarena` integration | +| `nemo-gym-workplace-v1` | configuring the built-in `nemo_gym` taskset for a harder packaged NeMo Gym example | Harness examples (under `environments/`): @@ -122,14 +123,15 @@ There is no NeMo-specific harness or program. ```bash uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 -uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 \ - --taskset.resource-server workplace_assistant +uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \ + --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1 ``` -The taskset does not interpret the example behavior. It loads rows from the configured -`resource_server`, uses `config_name` only when the server's YAML has a different name, -and forwards declared calls through one generic `nemo_gym_call` tool. The standard -Verifiers harness drives the model/tool loop and captures the trace. +The taskset does not interpret the example's weather behavior. It forwards declared calls +to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool. +The standard Verifiers harness drives the model/tool loop and captures the trace. The +`nemo-gym-workplace-v1` example pins only the taskset config to NeMo Gym's harder +`workplace_assistant` resource server. ### Swappable runtime diff --git a/verifiers/v1/tasksets/nemo_gym/taskset.py b/verifiers/v1/tasksets/nemo_gym/taskset.py index e4762d64fa..e238113a66 100644 --- a/verifiers/v1/tasksets/nemo_gym/taskset.py +++ b/verifiers/v1/tasksets/nemo_gym/taskset.py @@ -13,7 +13,7 @@ from verifiers.v1.task import Task from verifiers.v1.taskset import Taskset, TasksetConfig -DEFAULT_RESOURCE_SERVER = "example_single_tool_call" +DEFAULT_ENV = "example_single_tool_call" class NeMoGymTask(Task): @@ -21,14 +21,12 @@ class NeMoGymTask(Task): class NeMoGymConfig(TasksetConfig): - resource_server: str = DEFAULT_RESOURCE_SERVER - config_name: str | None = None + nemo_env: str = DEFAULT_ENV data_name: str = "example.jsonl" class _NeMoGymToolsConfig(ToolsetConfig): - resource_server: str - config_name: str | None = None + nemo_env: str class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): @@ -37,27 +35,23 @@ class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): TOOL_PREFIX = "nemo_gym" async def setup(self) -> None: - nemo_gym = importlib.import_module("nemo_gym") - SimpleResourcesServer = importlib.import_module( - "nemo_gym.base_resources_server" - ).SimpleResourcesServer - BaseServerConfig = importlib.import_module( - "nemo_gym.config_types" - ).BaseServerConfig - global_config = importlib.import_module("nemo_gym.global_config") - GlobalConfigDictParser = global_config.GlobalConfigDictParser - GlobalConfigDictParserConfig = global_config.GlobalConfigDictParserConfig - ServerClient = importlib.import_module("nemo_gym.server_utils").ServerClient - OmegaConf = importlib.import_module("omegaconf").OmegaConf - - root = Path(nemo_gym.PARENT_DIR) - config_name = self.config.config_name or self.config.resource_server + from nemo_gym import PARENT_DIR + from nemo_gym.base_resources_server import SimpleResourcesServer + from nemo_gym.config_types import BaseServerConfig + from nemo_gym.global_config import ( + GlobalConfigDictParser, + GlobalConfigDictParserConfig, + ) + from nemo_gym.server_utils import ServerClient + from omegaconf import OmegaConf + + root = Path(PARENT_DIR) path = ( root / "resources_servers" - / self.config.resource_server + / self.config.nemo_env / "configs" - / f"{config_name}.yaml" + / f"{self.config.nemo_env}.yaml" ) parser = GlobalConfigDictParser() config = parser.parse_no_environment( @@ -73,7 +67,7 @@ async def setup(self) -> None: ) entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem module = importlib.import_module( - f"resources_servers.{self.config.resource_server}.{entrypoint}" + f"resources_servers.{self.config.nemo_env}.{entrypoint}" ) server_cls: Any = next( value @@ -123,16 +117,16 @@ async def call(self, name: str, arguments: dict[str, Any]) -> Any: class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]): def load_tasks(self) -> list[NeMoGymTask]: try: - nemo_gym = importlib.import_module("nemo_gym") + from nemo_gym import PARENT_DIR except ImportError as exc: raise ImportError( "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`." ) from exc path = ( - Path(nemo_gym.PARENT_DIR) + Path(PARENT_DIR) / "resources_servers" - / self.config.resource_server + / self.config.nemo_env / "data" / self.config.data_name ) @@ -141,7 +135,7 @@ def load_tasks(self) -> list[NeMoGymTask]: return [ NeMoGymTask( idx=idx, - name=f"{self.config.resource_server}:{idx}", + name=f"{self.config.nemo_env}:{idx}", prompt=dialect.parse_request(row["responses_create_params"])[0], system_prompt=( "Call `nemo_gym_call` with the matching tool name and arguments. " @@ -156,14 +150,7 @@ def load_tasks(self) -> list[NeMoGymTask]: def tools(self, task: NeMoGymTask) -> list[Toolset]: if not task.nemo_gym_row["responses_create_params"].get("tools"): return [] - return [ - _NeMoGymTools( - _NeMoGymToolsConfig( - resource_server=self.config.resource_server, - config_name=self.config.config_name, - ) - ) - ] + return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))] if __name__ == "__main__":