Add NeMo Gym

environments/README.md

@@ -59,7 +59,6 @@ This folder contains installable example environments that showcase common usage
   - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards.
   - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint.
   - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup.
-  - **nemo_gym_env**: Minimal v1 example that wraps a packaged NeMo Gym task with `NeMoGymTaskset` and `NeMoGymHarness`.
   - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls.
   - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary.
 

verifiers/v1/README.md

@@ -112,6 +112,25 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent
 ```
 
+### NeMo Gym
+
+The built-in `nemo_gym` taskset defaults to NeMo Gym's own
+`example_single_tool_call` quickstart. It loads the example JSONL as ordinary typed tasks;
+the selected Verifiers harness owns the agent loop, while NeMo Gym's packaged resource
+server executes the declared tools. Model and tool messages use the normal Verifiers trace.
+There is no NeMo-specific harness or program.
+
+```bash
+uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1
+uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1 \
+  --taskset.resource-server workplace_assistant
+```
+
+The taskset does not interpret the example behavior. It loads rows from the configured
+`resource_server`, uses `config_name` only when the server's YAML has a different name,
+and forwards declared calls through one generic `nemo_gym_call` tool. The standard
+Verifiers harness drives the model/tool loop and captures the trace.
+
 ### Swappable runtime
 
 Where code runs, behind one `Runtime` contract — the same contract backs the harness

verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,11 @@
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "NeMoGymConfig",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/__init__.py

@@ -0,0 +1,11 @@
+from verifiers.v1.tasksets.nemo_gym.taskset import (
+    NeMoGymConfig,
+    NeMoGymTask,
+    NeMoGymTaskset,
+)
+
+__all__ = [
+    "NeMoGymConfig",
+    "NeMoGymTask",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/taskset.py

@@ -0,0 +1,170 @@
+"""Run a packaged NeMo Gym dataset with any MCP-capable Verifiers harness."""
+
+import importlib
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from verifiers.v1.decorators import tool
+from verifiers.v1.dialects.responses import ResponsesDialect
+from verifiers.v1.mcp import Toolset, ToolsetConfig
+from verifiers.v1.task import Task
+from verifiers.v1.taskset import Taskset, TasksetConfig
+
+DEFAULT_RESOURCE_SERVER = "example_single_tool_call"
+
+
+class NeMoGymTask(Task):
+    nemo_gym_row: dict[str, Any]
+
+
+class NeMoGymConfig(TasksetConfig):
+    resource_server: str = DEFAULT_RESOURCE_SERVER
+    config_name: str | None = None
+    data_name: str = "example.jsonl"
+
+
+class _NeMoGymToolsConfig(ToolsetConfig):
+    resource_server: str
+    config_name: str | None = None
+
+
+class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]):
+    """Expose a NeMo Gym resource server through one generic MCP call."""
+
+    TOOL_PREFIX = "nemo_gym"
+
+    async def setup(self) -> None:
+        nemo_gym = importlib.import_module("nemo_gym")
+        SimpleResourcesServer = importlib.import_module(
+            "nemo_gym.base_resources_server"
+        ).SimpleResourcesServer
+        BaseServerConfig = importlib.import_module(
+            "nemo_gym.config_types"
+        ).BaseServerConfig
+        global_config = importlib.import_module("nemo_gym.global_config")
+        GlobalConfigDictParser = global_config.GlobalConfigDictParser
+        GlobalConfigDictParserConfig = global_config.GlobalConfigDictParserConfig
+        ServerClient = importlib.import_module("nemo_gym.server_utils").ServerClient
+        OmegaConf = importlib.import_module("omegaconf").OmegaConf
+
+        root = Path(nemo_gym.PARENT_DIR)
+        config_name = self.config.config_name or self.config.resource_server
+        path = (
+            root
+            / "resources_servers"
+            / self.config.resource_server
+            / "configs"
+            / f"{config_name}.yaml"
+        )
+        parser = GlobalConfigDictParser()
+        config = parser.parse_no_environment(
+            OmegaConf.merge(
+                OmegaConf.load(path),
+                GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+            )
+        )
+        resource = next(
+            server
+            for server in parser.filter_for_server_instance_configs(config)
+            if server.SERVER_TYPE == "resources_servers"
+        )
+        entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem
+        module = importlib.import_module(
+            f"resources_servers.{self.config.resource_server}.{entrypoint}"
+        )
+        server_cls: Any = next(
+            value
+            for value in vars(module).values()
+            if isinstance(value, type)
+            and issubclass(value, SimpleResourcesServer)
+            and value.__module__ == module.__name__
+        )
+        server_config = server_cls.model_fields["config"].annotation.model_validate(
+            {
+                "name": resource.name,
+                **resource.get_inner_run_server_config().model_dump(),
+            }
+        )
+        server = server_cls(
+            config=server_config,
+            server_client=ServerClient(
+                head_server_config=BaseServerConfig.model_validate(
+                    config["head_server"]
+                ),
+                global_config_dict=config,
+            ),
+        )
+        self.client = httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=server.setup_webserver()),
+            base_url="http://nemo-gym",
+        )
+
+    async def setup_task(self, task: NeMoGymTask) -> None:
+        self.tool_names = {
+            spec["name"]
+            for spec in task.nemo_gym_row["responses_create_params"].get("tools", [])
+        }
+        response = await self.client.post("/seed_session", json=task.nemo_gym_row)
+        response.raise_for_status()
+
+    @tool
+    async def call(self, name: str, arguments: dict[str, Any]) -> Any:
+        """Call a tool declared by the current NeMo Gym task."""
+        if name not in self.tool_names:
+            raise ValueError(f"unknown NeMo Gym tool: {name}")
+        response = await self.client.post(f"/{name}", json=arguments)
+        response.raise_for_status()
+        return response.json()
+
+
+class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]):
+    def load_tasks(self) -> list[NeMoGymTask]:
+        try:
+            nemo_gym = importlib.import_module("nemo_gym")
+        except ImportError as exc:
+            raise ImportError(
+                "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`."
+            ) from exc
+
+        path = (
+            Path(nemo_gym.PARENT_DIR)
+            / "resources_servers"
+            / self.config.resource_server
+            / "data"
+            / self.config.data_name
+        )
+        rows = [json.loads(line) for line in path.read_text().splitlines() if line]
+        dialect = ResponsesDialect()
+        return [
+            NeMoGymTask(
+                idx=idx,
+                name=f"{self.config.resource_server}:{idx}",
+                prompt=dialect.parse_request(row["responses_create_params"])[0],
+                system_prompt=(
+                    "Call `nemo_gym_call` with the matching tool name and arguments. "
+                    f"Available NeMo Gym tools: "
+                    f"{json.dumps(row['responses_create_params'].get('tools', []))}"
+                ),
+                nemo_gym_row=row,
+            )
+            for idx, row in enumerate(rows)
+        ]
+
+    def tools(self, task: NeMoGymTask) -> list[Toolset]:
+        if not task.nemo_gym_row["responses_create_params"].get("tools"):
+            return []
+        return [
+            _NeMoGymTools(
+                _NeMoGymToolsConfig(
+                    resource_server=self.config.resource_server,
+                    config_name=self.config.config_name,
+                )
+            )
+        ]
+
+
+if __name__ == "__main__":
+    _NeMoGymTools.run()