diff --git a/environments/README.md b/environments/README.md index da47ab0b0d..759fa59ebd 100644 --- a/environments/README.md +++ b/environments/README.md @@ -59,7 +59,7 @@ This folder contains installable example environments that showcase common usage - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards. - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint. - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup. - - **nemo_gym_env**: Minimal v1 example that wraps a packaged NeMo Gym task with `NeMoGymTaskset` and `NeMoGymHarness`. + - **nemo_gym_workplace_v1**: Thin v1 config over the reusable `NeMoGymTaskset`, pinned to NeMo Gym's `workplace_assistant` resource server. - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls. - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary. diff --git a/environments/nemo_gym_workplace_v1/README.md b/environments/nemo_gym_workplace_v1/README.md new file mode 100644 index 0000000000..dd74fc742b --- /dev/null +++ b/environments/nemo_gym_workplace_v1/README.md @@ -0,0 +1,21 @@ +# nemo-gym-workplace-v1 + +NeMo Gym's `workplace_assistant` example through the built-in `NeMoGymTaskset`. +This environment pins the taskset config only; the standard Verifiers harness owns the +rollout loop, and NeMo Gym's packaged resource server owns tool execution. + +## Develop + +Install + run: + +```bash +uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \ + --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1 +``` + +## Layout + +- `nemo_gym_workplace_v1/taskset.py` — a thin config wrapper over `NeMoGymTaskset`. + +Tune the packaged dataset from the CLI with `--taskset.data-name`, and use ordinary +Verifiers harness flags for rollout behavior. diff --git a/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py new file mode 100644 index 0000000000..d988f140f3 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py @@ -0,0 +1,3 @@ +from nemo_gym_workplace_v1.taskset import NemoGymWorkplaceTaskset + +__all__ = ["NemoGymWorkplaceTaskset"] diff --git a/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py new file mode 100644 index 0000000000..93106cd8b2 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py @@ -0,0 +1,20 @@ +from typing import Literal + +import verifiers.v1 as vf +from verifiers.v1.tasksets.nemo_gym import ( + NeMoGymConfig, + NeMoGymTask, + NeMoGymTaskset, +) + + +class NemoGymWorkplaceConfig(NeMoGymConfig): + nemo_env: Literal["workplace_assistant"] = "workplace_assistant" + + +class NemoGymWorkplaceTaskset(vf.Taskset[NeMoGymTask, NemoGymWorkplaceConfig]): + def load_tasks(self) -> list[NeMoGymTask]: + return NeMoGymTaskset(self.config).load_tasks() + + def tools(self, task: NeMoGymTask) -> list[vf.Toolset]: + return NeMoGymTaskset(self.config).tools(task) diff --git a/environments/nemo_gym_workplace_v1/pyproject.toml b/environments/nemo_gym_workplace_v1/pyproject.toml new file mode 100644 index 0000000000..a154848d67 --- /dev/null +++ b/environments/nemo_gym_workplace_v1/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "nemo-gym-workplace-v1" +version = "0.1.0" +description = "nemo-gym-workplace-v1 - NeMo Gym workplace_assistant through the built-in taskset." +requires-python = ">=3.11" +dependencies = ["verifiers"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["nemo_gym_workplace_v1"] diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index f2d24586c2..434e0001fc 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -82,6 +82,7 @@ Taskset examples (the `*_v1` packages under `environments/`): | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge | | `deepwiki-v1` | an **existing remote** tool server, by URL | | `wordle-v1` | configuring the vendored `textarena` integration | +| `nemo-gym-workplace-v1` | configuring the built-in `nemo_gym` taskset for a harder packaged NeMo Gym example | Harness examples (under `environments/`): @@ -112,6 +113,26 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex # the codex CLI agent ``` +### NeMo Gym + +The built-in `nemo_gym` taskset defaults to NeMo Gym's own +`example_single_tool_call` quickstart. It loads the example JSONL as ordinary typed tasks; +the selected Verifiers harness owns the agent loop, while NeMo Gym's packaged resource +server executes the declared tools. Model and tool messages use the normal Verifiers trace. +There is no NeMo-specific harness or program. + +```bash +uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 +uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \ + --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1 +``` + +The taskset does not interpret the example's weather behavior. It forwards declared calls +to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool. +The standard Verifiers harness drives the model/tool loop and captures the trace. The +`nemo-gym-workplace-v1` example pins only the taskset config to NeMo Gym's harder +`workplace_assistant` resource server. + ### Swappable runtime Where code runs, behind one `Runtime` contract — the same contract backs the harness diff --git a/verifiers/v1/tasksets/__init__.py b/verifiers/v1/tasksets/__init__.py index 956f606325..3df6565c46 100644 --- a/verifiers/v1/tasksets/__init__.py +++ b/verifiers/v1/tasksets/__init__.py @@ -5,5 +5,11 @@ `verifiers.v1.tasksets.textarena`.""" from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset +from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset -__all__ = ["HarborConfig", "HarborTaskset"] +__all__ = [ + "HarborConfig", + "HarborTaskset", + "NeMoGymConfig", + "NeMoGymTaskset", +] diff --git a/verifiers/v1/tasksets/nemo_gym/__init__.py b/verifiers/v1/tasksets/nemo_gym/__init__.py new file mode 100644 index 0000000000..6485d75de0 --- /dev/null +++ b/verifiers/v1/tasksets/nemo_gym/__init__.py @@ -0,0 +1,11 @@ +from verifiers.v1.tasksets.nemo_gym.taskset import ( + NeMoGymConfig, + NeMoGymTask, + NeMoGymTaskset, +) + +__all__ = [ + "NeMoGymConfig", + "NeMoGymTask", + "NeMoGymTaskset", +] diff --git a/verifiers/v1/tasksets/nemo_gym/taskset.py b/verifiers/v1/tasksets/nemo_gym/taskset.py new file mode 100644 index 0000000000..e238113a66 --- /dev/null +++ b/verifiers/v1/tasksets/nemo_gym/taskset.py @@ -0,0 +1,157 @@ +"""Run a packaged NeMo Gym dataset with any MCP-capable Verifiers harness.""" + +import importlib +import json +from pathlib import Path +from typing import Any + +import httpx + +from verifiers.v1.decorators import tool +from verifiers.v1.dialects.responses import ResponsesDialect +from verifiers.v1.mcp import Toolset, ToolsetConfig +from verifiers.v1.task import Task +from verifiers.v1.taskset import Taskset, TasksetConfig + +DEFAULT_ENV = "example_single_tool_call" + + +class NeMoGymTask(Task): + nemo_gym_row: dict[str, Any] + + +class NeMoGymConfig(TasksetConfig): + nemo_env: str = DEFAULT_ENV + data_name: str = "example.jsonl" + + +class _NeMoGymToolsConfig(ToolsetConfig): + nemo_env: str + + +class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]): + """Expose a NeMo Gym resource server through one generic MCP call.""" + + TOOL_PREFIX = "nemo_gym" + + async def setup(self) -> None: + from nemo_gym import PARENT_DIR + from nemo_gym.base_resources_server import SimpleResourcesServer + from nemo_gym.config_types import BaseServerConfig + from nemo_gym.global_config import ( + GlobalConfigDictParser, + GlobalConfigDictParserConfig, + ) + from nemo_gym.server_utils import ServerClient + from omegaconf import OmegaConf + + root = Path(PARENT_DIR) + path = ( + root + / "resources_servers" + / self.config.nemo_env + / "configs" + / f"{self.config.nemo_env}.yaml" + ) + parser = GlobalConfigDictParser() + config = parser.parse_no_environment( + OmegaConf.merge( + OmegaConf.load(path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + resource = next( + server + for server in parser.filter_for_server_instance_configs(config) + if server.SERVER_TYPE == "resources_servers" + ) + entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem + module = importlib.import_module( + f"resources_servers.{self.config.nemo_env}.{entrypoint}" + ) + server_cls: Any = next( + value + for value in vars(module).values() + if isinstance(value, type) + and issubclass(value, SimpleResourcesServer) + and value.__module__ == module.__name__ + ) + server_config = server_cls.model_fields["config"].annotation.model_validate( + { + "name": resource.name, + **resource.get_inner_run_server_config().model_dump(), + } + ) + server = server_cls( + config=server_config, + server_client=ServerClient( + head_server_config=BaseServerConfig.model_validate( + config["head_server"] + ), + global_config_dict=config, + ), + ) + self.client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=server.setup_webserver()), + base_url="http://nemo-gym", + ) + + async def setup_task(self, task: NeMoGymTask) -> None: + self.tool_names = { + spec["name"] + for spec in task.nemo_gym_row["responses_create_params"].get("tools", []) + } + response = await self.client.post("/seed_session", json=task.nemo_gym_row) + response.raise_for_status() + + @tool + async def call(self, name: str, arguments: dict[str, Any]) -> Any: + """Call a tool declared by the current NeMo Gym task.""" + if name not in self.tool_names: + raise ValueError(f"unknown NeMo Gym tool: {name}") + response = await self.client.post(f"/{name}", json=arguments) + response.raise_for_status() + return response.json() + + +class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]): + def load_tasks(self) -> list[NeMoGymTask]: + try: + from nemo_gym import PARENT_DIR + except ImportError as exc: + raise ImportError( + "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`." + ) from exc + + path = ( + Path(PARENT_DIR) + / "resources_servers" + / self.config.nemo_env + / "data" + / self.config.data_name + ) + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + dialect = ResponsesDialect() + return [ + NeMoGymTask( + idx=idx, + name=f"{self.config.nemo_env}:{idx}", + prompt=dialect.parse_request(row["responses_create_params"])[0], + system_prompt=( + "Call `nemo_gym_call` with the matching tool name and arguments. " + f"Available NeMo Gym tools: " + f"{json.dumps(row['responses_create_params'].get('tools', []))}" + ), + nemo_gym_row=row, + ) + for idx, row in enumerate(rows) + ] + + def tools(self, task: NeMoGymTask) -> list[Toolset]: + if not task.nemo_gym_row["responses_create_params"].get("tools"): + return [] + return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))] + + +if __name__ == "__main__": + _NeMoGymTools.run()