Add NeMo Gym

environments/README.md

@@ -59,7 +59,7 @@ This folder contains installable example environments that showcase common usage
   - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards.
   - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint.
   - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup.
-  - **nemo_gym_env**: Minimal v1 example that wraps a packaged NeMo Gym task with `NeMoGymTaskset` and `NeMoGymHarness`.
+  - **nemo_gym_workplace_v1**: Thin v1 config over the reusable `NeMoGymTaskset`, pinned to NeMo Gym's `workplace_assistant` resource server.
   - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls.
   - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary.
 

environments/nemo_gym_workplace_v1/README.md

@@ -0,0 +1,21 @@
+# nemo-gym-workplace-v1
+
+NeMo Gym's `workplace_assistant` example through the built-in `NeMoGymTaskset`.
+This environment pins the taskset config only; the standard Verifiers harness owns the
+rollout loop, and NeMo Gym's packaged resource server owns tool execution.
+
+## Develop
+
+Install + run:
+
+```bash
+uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \
+  --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1
+```
+
+## Layout
+
+- `nemo_gym_workplace_v1/taskset.py` — a thin config wrapper over `NeMoGymTaskset`.
+
+Tune the packaged dataset from the CLI with `--taskset.data-name`, and use ordinary
+Verifiers harness flags for rollout behavior.

environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/__init__.py

@@ -0,0 +1,3 @@
+from nemo_gym_workplace_v1.taskset import NemoGymWorkplaceTaskset
+
+__all__ = ["NemoGymWorkplaceTaskset"]

environments/nemo_gym_workplace_v1/nemo_gym_workplace_v1/taskset.py

@@ -0,0 +1,20 @@
+from typing import Literal
+
+import verifiers.v1 as vf
+from verifiers.v1.tasksets.nemo_gym import (
+    NeMoGymConfig,
+    NeMoGymTask,
+    NeMoGymTaskset,
+)
+
+
+class NemoGymWorkplaceConfig(NeMoGymConfig):
+    nemo_env: Literal["workplace_assistant"] = "workplace_assistant"
+
+
+class NemoGymWorkplaceTaskset(vf.Taskset[NeMoGymTask, NemoGymWorkplaceConfig]):
+    def load_tasks(self) -> list[NeMoGymTask]:
+        return NeMoGymTaskset(self.config).load_tasks()
+
+    def tools(self, task: NeMoGymTask) -> list[vf.Toolset]:
+        return NeMoGymTaskset(self.config).tools(task)

environments/nemo_gym_workplace_v1/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "nemo-gym-workplace-v1"
+version = "0.1.0"
+description = "nemo-gym-workplace-v1 - NeMo Gym workplace_assistant through the built-in taskset."
+requires-python = ">=3.11"
+dependencies = ["verifiers"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["nemo_gym_workplace_v1"]

verifiers/v1/README.md

@@ -82,6 +82,7 @@ Taskset examples (the `*_v1` packages under `environments/`):
 | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge |
 | `deepwiki-v1` | an **existing remote** tool server, by URL |
 | `wordle-v1` | configuring the vendored `textarena` integration |
+| `nemo-gym-workplace-v1` | configuring the built-in `nemo_gym` taskset for a harder packaged NeMo Gym example |
 
 Harness examples (under `environments/`):
 
@@ -112,6 +113,26 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent
 ```
 
+### NeMo Gym
+
+The built-in `nemo_gym` taskset defaults to NeMo Gym's own
+`example_single_tool_call` quickstart. It loads the example JSONL as ordinary typed tasks;
+the selected Verifiers harness owns the agent loop, while NeMo Gym's packaged resource
+server executes the declared tools. Model and tool messages use the normal Verifiers trace.
+There is no NeMo-specific harness or program.
+
+```bash
+uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1
+uv run --with-editable . --with-editable environments/nemo_gym_workplace_v1 \
+  --with nemo-gym==0.3.0 eval nemo-gym-workplace-v1 -n 1 -r 1 -c 1
+```
+
+The taskset does not interpret the example's weather behavior. It forwards declared calls
+to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool.
+The standard Verifiers harness drives the model/tool loop and captures the trace. The
+`nemo-gym-workplace-v1` example pins only the taskset config to NeMo Gym's harder
+`workplace_assistant` resource server.
+
 ### Swappable runtime
 
 Where code runs, behind one `Runtime` contract — the same contract backs the harness

verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,11 @@
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "NeMoGymConfig",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/__init__.py

@@ -0,0 +1,11 @@
+from verifiers.v1.tasksets.nemo_gym.taskset import (
+    NeMoGymConfig,
+    NeMoGymTask,
+    NeMoGymTaskset,
+)
+
+__all__ = [
+    "NeMoGymConfig",
+    "NeMoGymTask",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/taskset.py

@@ -0,0 +1,157 @@
+"""Run a packaged NeMo Gym dataset with any MCP-capable Verifiers harness."""
+
+import importlib
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from verifiers.v1.decorators import tool
+from verifiers.v1.dialects.responses import ResponsesDialect
+from verifiers.v1.mcp import Toolset, ToolsetConfig
+from verifiers.v1.task import Task
+from verifiers.v1.taskset import Taskset, TasksetConfig
+
+DEFAULT_ENV = "example_single_tool_call"
+
+
+class NeMoGymTask(Task):
+    nemo_gym_row: dict[str, Any]
+
+
+class NeMoGymConfig(TasksetConfig):
+    nemo_env: str = DEFAULT_ENV
+    data_name: str = "example.jsonl"
+
+
+class _NeMoGymToolsConfig(ToolsetConfig):
+    nemo_env: str
+
+
+class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]):
+    """Expose a NeMo Gym resource server through one generic MCP call."""
+
+    TOOL_PREFIX = "nemo_gym"
+
+    async def setup(self) -> None:
+        from nemo_gym import PARENT_DIR
+        from nemo_gym.base_resources_server import SimpleResourcesServer
+        from nemo_gym.config_types import BaseServerConfig
+        from nemo_gym.global_config import (
+            GlobalConfigDictParser,
+            GlobalConfigDictParserConfig,
+        )
+        from nemo_gym.server_utils import ServerClient
+        from omegaconf import OmegaConf
+
+        root = Path(PARENT_DIR)
+        path = (
+            root
+            / "resources_servers"
+            / self.config.nemo_env
+            / "configs"
+            / f"{self.config.nemo_env}.yaml"
+        )
+        parser = GlobalConfigDictParser()
+        config = parser.parse_no_environment(
+            OmegaConf.merge(
+                OmegaConf.load(path),
+                GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+            )
+        )
+        resource = next(
+            server
+            for server in parser.filter_for_server_instance_configs(config)
+            if server.SERVER_TYPE == "resources_servers"
+        )
+        entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem
+        module = importlib.import_module(
+            f"resources_servers.{self.config.nemo_env}.{entrypoint}"
+        )
+        server_cls: Any = next(
+            value
+            for value in vars(module).values()
+            if isinstance(value, type)
+            and issubclass(value, SimpleResourcesServer)
+            and value.__module__ == module.__name__
+        )
+        server_config = server_cls.model_fields["config"].annotation.model_validate(
+            {
+                "name": resource.name,
+                **resource.get_inner_run_server_config().model_dump(),
+            }
+        )
+        server = server_cls(
+            config=server_config,
+            server_client=ServerClient(
+                head_server_config=BaseServerConfig.model_validate(
+                    config["head_server"]
+                ),
+                global_config_dict=config,
+            ),
+        )
+        self.client = httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=server.setup_webserver()),
+            base_url="http://nemo-gym",
+        )
+
+    async def setup_task(self, task: NeMoGymTask) -> None:
+        self.tool_names = {
+            spec["name"]
+            for spec in task.nemo_gym_row["responses_create_params"].get("tools", [])
+        }
+        response = await self.client.post("/seed_session", json=task.nemo_gym_row)
+        response.raise_for_status()
+
+    @tool
+    async def call(self, name: str, arguments: dict[str, Any]) -> Any:
+        """Call a tool declared by the current NeMo Gym task."""
+        if name not in self.tool_names:
+            raise ValueError(f"unknown NeMo Gym tool: {name}")
+        response = await self.client.post(f"/{name}", json=arguments)
+        response.raise_for_status()
+        return response.json()
+
+
+class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]):
+    def load_tasks(self) -> list[NeMoGymTask]:
+        try:
+            from nemo_gym import PARENT_DIR
+        except ImportError as exc:
+            raise ImportError(
+                "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`."
+            ) from exc
+
+        path = (
+            Path(PARENT_DIR)
+            / "resources_servers"
+            / self.config.nemo_env
+            / "data"
+            / self.config.data_name
+        )
+        rows = [json.loads(line) for line in path.read_text().splitlines() if line]
+        dialect = ResponsesDialect()
+        return [
+            NeMoGymTask(
+                idx=idx,
+                name=f"{self.config.nemo_env}:{idx}",
+                prompt=dialect.parse_request(row["responses_create_params"])[0],
+                system_prompt=(
+                    "Call `nemo_gym_call` with the matching tool name and arguments. "
+                    f"Available NeMo Gym tools: "
+                    f"{json.dumps(row['responses_create_params'].get('tools', []))}"
+                ),
+                nemo_gym_row=row,
+            )
+            for idx, row in enumerate(rows)
+        ]
+
+    def tools(self, task: NeMoGymTask) -> list[Toolset]:
+        if not task.nemo_gym_row["responses_create_params"].get("tools"):
+            return []
+        return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))]
+
+
+if __name__ == "__main__":
+    _NeMoGymTools.run()