From 7dc15bb7da7cec1820a7d3b29653b35b00fb9bff Mon Sep 17 00:00:00 2001
From: eexwhyzee <13672394+eexwhyzee@users.noreply.github.com>
Date: Wed, 24 Jun 2026 15:07:36 -0700
Subject: [PATCH 1/4] Add CLI support for v1 RFT env configs

---
 packages/prime/src/prime_cli/commands/rl.py | 211 +++++++++++++++-----
 packages/prime/tests/test_rl_api.py         |  31 +++
 packages/prime/tests/test_rl_config.py      |  94 +++++++++
 3 files changed, 289 insertions(+), 47 deletions(-)

diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py
index ca5105cf..e661a62d 100644
--- a/packages/prime/src/prime_cli/commands/rl.py
+++ b/packages/prime/src/prime_cli/commands/rl.py
@@ -42,6 +42,22 @@
 
 console = get_console()
 
+V1_ENV_CONFIG_FIELDS = (
+    "taskset",
+    "harness",
+    "pool",
+    "timeout",
+    "retries",
+    "sampling",
+    "ratio",
+    "group_size",
+    "max_turns",
+    "max_input_tokens",
+    "max_output_tokens",
+    "max_total_tokens",
+    "multiplex",
+)
+
 RL_RUN_JSON_HELP = json_output_help(
     ".run = {id, name?, status, base_model, environments[], "
     "rollouts_per_example, max_steps, batch_size, created_at, updated_at, ...}",
@@ -258,6 +274,12 @@ def generate_rl_config_template(environment: str | None = None) -> str:
 # [[env]] # add multiple [[env]] sections for multi-env training
 # id = "primeintellect/another-env"
 # args = {{ split = "train", max_examples = 1000 }}
+#
+# v1 environment shape: use taskset/harness instead of legacy id.
+# [[env]]
+# name = "alphabet-sort"
+# taskset = {{ id = "alphabet-sort-v1", min_turns = 3, max_turns = 5, power_per_turn = false }}
+# harness = {{ id = "default", runtime = {{ type = "subprocess" }} }}
 
 # Optional: online evaluation
 # [eval]
@@ -298,29 +320,125 @@ def generate_rl_config_template(environment: str | None = None) -> str:
 '''
 
 
+def _validate_env_id_value(v: str, *, field_name: str) -> str:
+    """Validate a legacy env id or v1 plugin id.
+
+    Bare ids are importable runtime ids. Slash-shaped ids are Hub refs and
+    must include both owner and name.
+    """
+    v = v.strip()
+    if not v:
+        raise ValueError(f"{field_name} cannot be empty")
+    if "/" in v:
+        owner, name = v.split("/", 1)
+        if not owner.strip() or not name.strip():
+            raise ValueError(
+                f"{field_name} must have both owner and name (e.g., 'primeintellect/vf-math')"
+            )
+    return v
+
+
+def _dict_id(config: Dict[str, Any] | None) -> Any:
+    return config.get("id") if isinstance(config, dict) else None
+
+
+def _is_hub_env_id(value: Any) -> bool:
+    return isinstance(value, str) and "/" in value
+
+
+def _split_hub_env_id(env_id: str) -> str:
+    return env_id.rsplit("@", 1)[0]
+
+
+def _env_display_name(env: Dict[str, Any], index: int | None = None) -> str:
+    taskset_id = _dict_id(env.get("taskset") if isinstance(env.get("taskset"), dict) else None)
+    value = env.get("slug") or env.get("name") or env.get("id") or taskset_id
+    if value:
+        return str(value)
+    return f"env-{index}" if index is not None else "?"
+
+
 class EnvConfig(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
-    id: str
+    id: str | None = None
     name: str | None = None
+    taskset: Dict[str, Any] | None = None
+    harness: Dict[str, Any] | None = None
+    pool: Dict[str, Any] | None = None
+    timeout: Dict[str, Any] | None = None
+    retries: Dict[str, Any] | None = None
+    sampling: Dict[str, Any] | None = None
+    ratio: float | None = Field(default=None, gt=0)
+    group_size: int | None = Field(default=None, ge=1)
+    max_turns: int | None = Field(default=None, ge=1)
+    max_input_tokens: int | None = Field(default=None, ge=1)
+    max_output_tokens: int | None = Field(default=None, ge=1)
+    max_total_tokens: int | None = Field(default=None, ge=1)
+    multiplex: int | None = Field(default=None, ge=1)
     args: Dict[str, Any] = Field(default_factory=dict)
     max_retries: int | None = Field(default=None, ge=0)
     version: str | None = None
 
+    @field_validator("id")
+    @classmethod
+    def validate_environment_id(cls, v: str | None) -> str | None:
+        if v is None:
+            return None
+        return _validate_env_id_value(v, field_name="Environment ID")
+
     @model_validator(mode="after")
     def parse_version_from_id(self) -> "EnvConfig":
         """Extract version from id if specified as 'owner/name@version'."""
-        if "@" in self.id:
+        if self.id and "@" in self.id:
             id_part, version_part = self.id.rsplit("@", 1)
-            self.id = id_part
+            self.id = _validate_env_id_value(id_part, field_name="Environment ID")
             if self.version is None and version_part:
                 self.version = version_part
         return self
 
+    @model_validator(mode="after")
+    def validate_env_selector(self) -> "EnvConfig":
+        taskset_id = _dict_id(self.taskset)
+        if self.id is None and not taskset_id:
+            raise ValueError("Environment config requires either legacy id or v1 taskset.id")
+        if taskset_id is not None:
+            assert self.taskset is not None
+            self.taskset["id"] = _validate_env_id_value(str(taskset_id), field_name="taskset.id")
+        harness_id = _dict_id(self.harness)
+        if harness_id is not None:
+            assert self.harness is not None
+            self.harness["id"] = _validate_env_id_value(str(harness_id), field_name="harness.id")
+        return self
+
+    @property
+    def is_v1(self) -> bool:
+        return bool(_dict_id(self.taskset))
+
+    @property
+    def display_name(self) -> str:
+        return _env_display_name(self.to_api_dict())
+
+    def hub_env_ids(self) -> list[str]:
+        refs: list[str] = []
+        if _is_hub_env_id(self.id):
+            refs.append(str(self.id))
+        for config in (self.taskset, self.harness):
+            plugin_id = _dict_id(config)
+            if _is_hub_env_id(plugin_id):
+                refs.append(str(plugin_id))
+        return refs
+
     def to_api_dict(self) -> Dict[str, Any]:
-        result: Dict[str, Any] = {"id": self.id}
+        result: Dict[str, Any] = {}
+        if self.id is not None:
+            result["id"] = self.id
         if self.name is not None:
             result["name"] = self.name
+        for field_name in V1_ENV_CONFIG_FIELDS:
+            value = getattr(self, field_name)
+            if value is not None:
+                result[field_name] = value
         if self.args:
             result["args"] = self.args
         if self.max_retries is not None:
@@ -330,41 +448,34 @@ def to_api_dict(self) -> Dict[str, Any]:
         return result
 
 
-class EvalEnvConfig(BaseModel):
-    model_config = ConfigDict(extra="forbid")
-
-    id: str
-    name: str | None = None
-    args: Dict[str, Any] = Field(default_factory=dict)
+class EvalEnvConfig(EnvConfig):
     num_examples: int | None = None
-    rollouts_per_example: int | None = None
-    max_retries: int | None = Field(default=None, ge=0)
-    version: str | None = None
+    rollouts_per_example: int | None = Field(default=None, ge=1)
+
+    @field_validator("num_examples")
+    @classmethod
+    def validate_num_examples(cls, v: int | None) -> int | None:
+        if v is not None and v != -1 and v < 1:
+            raise ValueError("num_examples must be -1 (all) or a positive integer")
+        return v
 
     @model_validator(mode="after")
-    def parse_version_from_id(self) -> "EvalEnvConfig":
-        """Extract version from id if specified as 'owner/name@version'."""
-        if "@" in self.id:
-            id_part, version_part = self.id.rsplit("@", 1)
-            self.id = id_part
-            if self.version is None and version_part:
-                self.version = version_part
+    def validate_eval_group_size_aliases(self) -> "EvalEnvConfig":
+        if (
+            self.group_size is not None
+            and self.rollouts_per_example is not None
+            and self.group_size != self.rollouts_per_example
+        ):
+            raise ValueError("group_size and rollouts_per_example cannot differ")
         return self
 
     def to_api_dict(self) -> Dict[str, Any]:
-        result: Dict[str, Any] = {"id": self.id}
-        if self.name is not None:
-            result["name"] = self.name
-        if self.args:
-            result["args"] = self.args
+        result = super().to_api_dict()
         if self.num_examples is not None:
             result["num_examples"] = self.num_examples
         if self.rollouts_per_example is not None:
-            result["rollouts_per_example"] = self.rollouts_per_example
-        if self.max_retries is not None:
-            result["max_retries"] = self.max_retries
-        if self.version is not None:
-            result["version"] = self.version
+            key = "group_size" if self.is_v1 else "rollouts_per_example"
+            result.setdefault(key, self.rollouts_per_example)
         return result
 
 
@@ -1034,7 +1145,8 @@ def _render_failure_analysis(analysis: Dict[str, Any]) -> None:
 def _format_run_for_display(run: RLRun) -> Dict[str, Any]:
     created_at = run.created_at.strftime("%Y-%m-%d %H:%M") if run.created_at else ""
     env_names = [
-        env.get("slug") or env.get("name") or env.get("id") or "?" for env in run.environments
+        _env_display_name(env, index=index) if isinstance(env, dict) else str(env)
+        for index, env in enumerate(run.environments)
     ]
     envs_display = ", ".join(env_names[:3])
     if len(env_names) > 3:
@@ -1240,7 +1352,7 @@ def _fetch_pricing() -> None:
         console.print(f"  Loss:         {cfg.loss}")
         if cfg.teacher is not None:
             console.print(f"  Teacher:      {cfg.teacher.model}")
-        console.print(f"  Environments: {', '.join(e.id for e in cfg.env)}")
+        console.print(f"  Environments: {', '.join(e.display_name for e in cfg.env)}")
         if app_config.team_id:
             console.print(f"  Team:         {app_config.team_id}")
 
@@ -1296,7 +1408,7 @@ def _fetch_pricing() -> None:
         # Eval
         if cfg.eval.env:
             console.print("\n[cyan]Evaluation[/cyan]")
-            console.print(f"  Environments: {', '.join(e.id for e in cfg.eval.env)}")
+            console.print(f"  Environments: {', '.join(e.display_name for e in cfg.eval.env)}")
             if cfg.eval.interval:
                 console.print(f"  Interval:     {cfg.eval.interval}")
 
@@ -1355,14 +1467,21 @@ def _format(list_p: Any, eff_p: Any) -> str:
 
         console.print()
 
-        # Check action status for hub environments
-        hub_envs = [e for e in cfg.env if "/" in e.id]
-        if hub_envs and not skip_action_check:
+        # Check action status for Hub-backed environment refs. Bare v1 ids
+        # like taskset.id="alphabet-sort-v1" are runtime-importable names and
+        # intentionally do not resolve through the Hub.
+        hub_env_ids: list[str] = []
+        for env_config in cfg.env:
+            for env_id in env_config.hub_env_ids():
+                if env_id not in hub_env_ids:
+                    hub_env_ids.append(env_id)
+
+        if hub_env_ids and not skip_action_check:
             console.print("[dim]Checking Environment Actions...[/dim]")
             failed_envs = []
 
-            for env_config in hub_envs:
-                env_id_base = env_config.id.split("@")[0]
+            for env_id in hub_env_ids:
+                env_id_base = _split_hub_env_id(env_id)
                 owner, name = env_id_base.split("/", 1)
                 try:
                     status_resp = rl_client.get_environment_status(owner, name)
@@ -1370,23 +1489,21 @@ def _format(list_p: Any, eff_p: Any) -> str:
                     action_status = action.get("status")
 
                     if action_status == "FAILED":
-                        console.print(f"  [red]✗[/red] {env_config.id} [dim](failed)[/dim]")
-                        failed_envs.append(env_config.id)
+                        console.print(f"  [red]✗[/red] {env_id} [dim](failed)[/dim]")
+                        failed_envs.append(env_id)
                     elif action_status == "SUCCESS":
-                        console.print(f"  [green]✓[/green] {env_config.id} [dim](success)[/dim]")
+                        console.print(f"  [green]✓[/green] {env_id} [dim](success)[/dim]")
                     elif action_status in ("RUNNING", "PENDING"):
-                        console.print(
-                            f"  [yellow]○[/yellow] {env_config.id} [dim](in progress)[/dim]"
-                        )
+                        console.print(f"  [yellow]○[/yellow] {env_id} [dim](in progress)[/dim]")
                     else:
-                        console.print(f"  [dim]-[/dim] {env_config.id} [dim](no action)[/dim]")
+                        console.print(f"  [dim]-[/dim] {env_id} [dim](no action)[/dim]")
                 except APIError:
-                    console.print(f"  [dim]-[/dim] {env_config.id} [dim](could not check)[/dim]")
+                    console.print(f"  [dim]-[/dim] {env_id} [dim](could not check)[/dim]")
 
             if failed_envs:
                 console.print("\n[red]Error: Action failed for environments:[/red]\n")
                 for env_id in failed_envs:
-                    env_id_base = env_id.split("@")[0]
+                    env_id_base = _split_hub_env_id(env_id)
                     owner, name = env_id_base.split("/", 1)
                     url = f"{app_config.frontend_url}/dashboard/environments/{owner}/{name}/actions"
                     console.print(f"  [red]✗[/red] {env_id}")
diff --git a/packages/prime/tests/test_rl_api.py b/packages/prime/tests/test_rl_api.py
index 3642c824..85773c3b 100644
--- a/packages/prime/tests/test_rl_api.py
+++ b/packages/prime/tests/test_rl_api.py
@@ -104,6 +104,37 @@ def test_create_run_sends_max_inflight_rollouts() -> None:
     assert run.max_inflight_rollouts == 96
 
 
+def test_create_run_sends_v1_env_payload_unchanged() -> None:
+    api_client = FakeAPIClient()
+    client = RLClient(api_client)  # type: ignore[arg-type]
+    v1_env = {
+        "name": "alphabet-sort",
+        "taskset": {"id": "alphabet-sort-v1", "min_turns": 3},
+        "harness": {"id": "default", "runtime": {"type": "subprocess"}},
+        "group_size": 8,
+    }
+    eval_config = {
+        "environments": [
+            {
+                "taskset": {"id": "alphabet-sort-v1", "split": "test"},
+                "harness": {"id": "default"},
+                "group_size": 4,
+            }
+        ]
+    }
+
+    client.create_run(
+        model_name="Qwen/Qwen3.5-0.8B",
+        environments=[v1_env],
+        eval_config=eval_config,
+    )
+
+    assert api_client.posts[0][0] == "/rft/runs"
+    payload = api_client.posts[0][1]
+    assert payload["environments"] == [v1_env]
+    assert payload["eval"] == eval_config
+
+
 def test_create_run_sends_sft_loss_and_teacher_config() -> None:
     api_client = FakeAPIClient()
     client = RLClient(api_client)  # type: ignore[arg-type]
diff --git a/packages/prime/tests/test_rl_config.py b/packages/prime/tests/test_rl_config.py
index 96f448f5..cc620a01 100644
--- a/packages/prime/tests/test_rl_config.py
+++ b/packages/prime/tests/test_rl_config.py
@@ -4,6 +4,7 @@
 import pytest
 import typer
 from prime_cli.commands.rl import (
+    EnvConfig,
     RLConfig,
     _flatten_config_schema,
     _is_full_finetune,
@@ -191,6 +192,99 @@ def test_load_config_accepts_eval_sampling_reasoning_effort(tmp_path: Path) -> N
     }
 
 
+def test_load_config_accepts_v1_train_env_without_legacy_id(tmp_path: Path) -> None:
+    config_path = tmp_path / "rl.toml"
+    config_path.write_text(
+        'model = "Qwen/Qwen3.5-0.8B"\n'
+        "[[env]]\n"
+        'name = "alphabet-sort"\n'
+        'taskset = { id = "alphabet-sort-v1", min_turns = 3, '
+        "max_turns = 5, power_per_turn = false }\n"
+        'harness = { id = "default", runtime = { type = "subprocess" } }\n'
+        "group_size = 8\n"
+        "ratio = 0.5\n"
+    )
+
+    cfg = load_config(str(config_path))
+
+    env = cfg.env[0]
+    assert env.id is None
+    assert env.display_name == "alphabet-sort"
+    assert env.to_api_dict() == {
+        "name": "alphabet-sort",
+        "taskset": {
+            "id": "alphabet-sort-v1",
+            "min_turns": 3,
+            "max_turns": 5,
+            "power_per_turn": False,
+        },
+        "harness": {"id": "default", "runtime": {"type": "subprocess"}},
+        "ratio": 0.5,
+        "group_size": 8,
+    }
+
+
+def test_load_config_rejects_env_without_legacy_id_or_taskset_id(tmp_path: Path) -> None:
+    config_path = tmp_path / "rl.toml"
+    config_path.write_text('model = "Qwen/Qwen3.5-0.8B"\n[[env]]\nname = "missing-selector"\n')
+
+    with pytest.raises(typer.Exit):
+        load_config(str(config_path))
+
+
+def test_eval_env_accepts_v1_group_size_alias(tmp_path: Path) -> None:
+    config_path = tmp_path / "rl.toml"
+    config_path.write_text(
+        'model = "Qwen/Qwen3.5-0.8B"\n'
+        "[[eval.env]]\n"
+        'taskset = { id = "alphabet-sort-v1", split = "test" }\n'
+        'harness = { id = "default" }\n'
+        "num_examples = 32\n"
+        "group_size = 4\n"
+    )
+
+    cfg = load_config(str(config_path))
+
+    assert cfg.eval.to_api_dict() == {
+        "environments": [
+            {
+                "taskset": {"id": "alphabet-sort-v1", "split": "test"},
+                "harness": {"id": "default"},
+                "group_size": 4,
+                "num_examples": 32,
+            }
+        ]
+    }
+
+
+def test_eval_env_emits_group_size_for_v1_rollouts_alias(tmp_path: Path) -> None:
+    config_path = tmp_path / "rl.toml"
+    config_path.write_text(
+        'model = "Qwen/Qwen3.5-0.8B"\n'
+        "[[eval.env]]\n"
+        'taskset = { id = "alphabet-sort-v1" }\n'
+        "rollouts_per_example = 4\n"
+    )
+
+    cfg = load_config(str(config_path))
+
+    env_payload = cfg.eval.to_api_dict()["environments"][0]
+    assert env_payload["group_size"] == 4
+    assert "rollouts_per_example" not in env_payload
+
+
+def test_env_config_hub_refs_include_only_slash_shaped_ids() -> None:
+    env = EnvConfig.model_validate(
+        {
+            "taskset": {"id": "dev/alphabet-sort-taskset@0.1.0"},
+            "harness": {"id": "default"},
+        }
+    )
+
+    assert env.display_name == "dev/alphabet-sort-taskset@0.1.0"
+    assert env.hub_env_ids() == ["dev/alphabet-sort-taskset@0.1.0"]
+
+
 def test_load_config_rejects_eval_sampling_with_both_reasoning_controls(
     tmp_path: Path,
 ) -> None:

From 58e659d5e102fbb16af98881903fa97da1024898 Mon Sep 17 00:00:00 2001
From: Oscar Moxon <oscarjmoxon@gmail.com>
Date: Mon, 29 Jun 2026 21:49:41 -0700
Subject: [PATCH 2/4] Add v1 env hosted eval CLI support

---
 .../prime/src/prime_cli/commands/evals.py     | 193 +++++++++++++++++-
 .../prime/src/prime_cli/utils/hosted_eval.py  |   3 +-
 packages/prime/tests/test_hosted_eval.py      | 189 +++++++++++++++++
 3 files changed, 373 insertions(+), 12 deletions(-)

diff --git a/packages/prime/src/prime_cli/commands/evals.py b/packages/prime/src/prime_cli/commands/evals.py
index 3c1977a3..d036348b 100644
--- a/packages/prime/src/prime_cli/commands/evals.py
+++ b/packages/prime/src/prime_cli/commands/evals.py
@@ -150,6 +150,7 @@
     "env_dir_path",
     "eval_name",
     "extra_env_kwargs",
+    "harness",
     "header",
     "headers",
     "independent_scoring",
@@ -162,6 +163,7 @@
     "rollouts_per_example",
     "sampling_args",
     "state_columns",
+    "taskset",
     "temperature",
     "timeout_minutes",
 }
@@ -358,6 +360,51 @@ def _validate_hosted_config_field(
         raise typer.Exit(1)
 
 
+def _format_pydantic_validation_error(exc: Exception) -> str:
+    errors = getattr(exc, "errors", lambda: [])()
+    messages = [
+        str(error.get("msg")).removeprefix("Value error, ")
+        for error in errors
+        if isinstance(error, dict) and error.get("msg")
+    ]
+    return "; ".join(messages) or str(exc)
+
+
+def _build_hosted_v1_environment_config(merged: dict[str, Any]) -> dict[str, Any]:
+    from pydantic import ValidationError as PydanticValidationError
+
+    from .rl import EvalEnvConfig
+
+    _validate_json_object_field(merged, "taskset")
+    _validate_json_object_field(merged, "harness")
+
+    raw_environment = {
+        field_name: merged[field_name]
+        for field_name in ("taskset", "harness")
+        if merged.get(field_name) is not None
+    }
+
+    try:
+        return EvalEnvConfig.model_validate(raw_environment).to_api_dict()
+    except (PydanticValidationError, ValueError) as exc:
+        console.print(
+            "[red]Error:[/red] hosted eval config "
+            + _format_pydantic_validation_error(exc)
+        )
+        raise typer.Exit(1) from exc
+
+
+def _hosted_v1_environment_display_name(
+    environment: dict[str, Any], index: Optional[int] = None
+) -> str:
+    taskset = environment.get("taskset")
+    taskset_id = taskset.get("id") if isinstance(taskset, dict) else None
+    value = environment.get("name") or environment.get("id") or taskset_id
+    if value:
+        return str(value)
+    return f"v1-env-{index}" if index is not None else "v1 environment"
+
+
 def _resolve_hosted_config_model(raw_config: dict[str, Any], config_path: Path) -> str:
     raw_endpoint_id = raw_config.get("endpoint_id")
     raw_model = raw_config.get("model")
@@ -441,9 +488,27 @@ def _validate_single_hosted_eval_config(
         raise typer.Exit(1)
 
     env_id = merged.get("env_id")
-    if type(env_id) is not str or not env_id:
-        console.print("[red]Error:[/red] hosted eval config requires a non-empty `env_id`")
+    has_env_id = env_id is not None
+    has_v1_selector = merged.get("taskset") is not None or merged.get("harness") is not None
+
+    if has_env_id and has_v1_selector:
+        console.print(
+            "[red]Error:[/red] hosted eval config cannot combine `env_id` "
+            "with `taskset` or `harness`"
+        )
         raise typer.Exit(1)
+    if has_env_id:
+        if type(env_id) is not str or not env_id:
+            console.print("[red]Error:[/red] hosted eval config requires a non-empty `env_id`")
+            raise typer.Exit(1)
+    else:
+        if not has_v1_selector:
+            console.print(
+                "[red]Error:[/red] hosted eval config requires either `env_id` "
+                "or `taskset.id`"
+            )
+            raise typer.Exit(1)
+        merged["environment"] = _build_hosted_v1_environment_config(merged)
 
     _validate_json_object_field(merged, "env_args")
     _validate_json_object_field(merged, "sampling_args")
@@ -487,14 +552,87 @@ def _validate_single_hosted_eval_config(
 
 
 def _load_hosted_eval_configs(config_path_str: str) -> list[dict[str, Any]]:
-    from verifiers.utils.eval_utils import load_toml_config
+    from verifiers.utils.env_config_utils import normalize_env_config_sections
+    from verifiers.utils.eval_utils import load_toml_config, normalize_env_id_alias
+    from verifiers.utils.import_utils import load_toml
+
+    def normalize_hosted_eval_sections(raw: dict[str, Any]) -> dict[str, Any]:
+        if raw.get("taskset") is not None or raw.get("harness") is not None:
+            return dict(raw)
+        return normalize_env_config_sections(raw)
 
     config_path = Path(config_path_str)
     try:
-        loaded_configs = load_toml_config(
-            config_path,
-            extra_valid_fields=HOSTED_EVAL_CONFIG_EXTRA_FIELDS,
+        if not config_path.exists():
+            raise FileNotFoundError(f"Config file not found: {config_path}")
+
+        with open(config_path, "rb") as file:
+            raw_config = load_toml(file)
+
+        eval_list = raw_config.get("eval", [])
+        if not isinstance(eval_list, list):
+            raise ValueError(
+                f"Config file uses [eval] but should use [[eval]] (double brackets) "
+                f"for array of tables: {config_path}"
+            )
+        uses_v1_selector = any(
+            isinstance(eval_config, dict)
+            and ("taskset" in eval_config or "harness" in eval_config)
+            for eval_config in eval_list
         )
+        if not uses_v1_selector:
+            if any(
+                isinstance(eval_config, dict)
+                and "env_id" not in eval_config
+                and "id" not in eval_config
+                for eval_config in eval_list
+            ):
+                raise ValueError(
+                    "hosted eval config requires either `env_id` or `taskset.id`"
+                )
+            loaded_configs = load_toml_config(
+                config_path,
+                extra_valid_fields=HOSTED_EVAL_CONFIG_EXTRA_FIELDS,
+            )
+            return [
+                _validate_single_hosted_eval_config(dict(config), config_path)
+                for config in loaded_configs
+            ]
+
+        ablation_list = raw_config.get("ablation", [])
+        if not isinstance(ablation_list, list):
+            raise ValueError(
+                f"Config file uses [ablation] but should use [[ablation]] "
+                f"(double brackets) for array of tables: {config_path}"
+            )
+        if ablation_list:
+            raise ValueError("hosted eval v1 environment configs do not support ablations")
+        if not eval_list:
+            raise ValueError(
+                f"Config file must contain at least one [[eval]] section: {config_path}"
+            )
+
+        global_defaults = {
+            key: value for key, value in raw_config.items() if key not in ("eval", "ablation")
+        }
+        loaded_configs = []
+        for eval_config in eval_list:
+            normalized_eval = normalize_env_id_alias(eval_config, "[[eval]]")
+            merged = {**global_defaults, **normalized_eval}
+            if "endpoint_id" in normalized_eval and "model" not in normalized_eval:
+                merged.pop("model", None)
+            if "model" in normalized_eval and "endpoint_id" not in normalized_eval:
+                merged.pop("endpoint_id", None)
+            loaded_configs.append(normalize_hosted_eval_sections(merged))
+
+        for merged in loaded_configs:
+            endpoints_path = merged.get("endpoints_path")
+            if isinstance(endpoints_path, str):
+                endpoints_path_obj = Path(endpoints_path)
+                if not endpoints_path_obj.is_absolute():
+                    merged["endpoints_path"] = str(
+                        (config_path.parent / endpoints_path_obj).resolve()
+                    )
     except Exception as exc:
         console.print(f"[red]Error:[/red] {exc}")
         raise typer.Exit(1) from exc
@@ -551,11 +689,21 @@ def _build_hosted_evaluation_payload(config: HostedEvalConfig) -> dict[str, Any]
     if config.api_key_var:
         eval_config["api_key_var"] = config.api_key_var
 
+    has_environment_id = config.environment_id is not None
+    has_environments = config.environments is not None
+    if has_environment_id == has_environments:
+        raise ValueError(
+            "Hosted eval config must set exactly one of environment_id or environments"
+        )
+
     payload: dict[str, Any] = {
-        "environment_ids": [config.environment_id],
         "inference_model": config.inference_model,
         "eval_config": eval_config,
     }
+    if config.environment_id is not None:
+        payload["environment_ids"] = [config.environment_id]
+    if config.environments is not None:
+        payload["environments"] = config.environments
     if config.name:
         payload["name"] = config.name
 
@@ -569,6 +717,7 @@ def _create_hosted_evaluations(
     payload = _build_hosted_evaluation_payload(config)
 
     if environment_ids is not None:
+        payload.pop("environments", None)
         payload["environment_ids"] = environment_ids
 
     if client.config.team_id:
@@ -1633,10 +1782,17 @@ def run_eval_cmd(
                 or None
             )
 
-            effective_targets.append(
-                {
+            selector = (
+                {"environment": target_config["environment"]}
+                if target_config.get("environment") is not None
+                else {
                     "env_id": target_config["env_id"],
                     "env_dir_path": target_config.get("env_dir_path") or env_dir_path,
+                }
+            )
+            effective_targets.append(
+                {
+                    **selector,
                     "model": (
                         parsed_verifiers_args.model
                         if "model" in cli_overrides
@@ -1732,7 +1888,9 @@ def run_eval_cmd(
         grouped_targets: dict[tuple[Any, ...], dict[str, Any]] = {}
         target_order: list[tuple[Any, ...]] = []
         for target in effective_targets:
+            selector_mode = "v1_env" if target.get("environment") is not None else "published_env"
             group_key = (
+                selector_mode,
                 target["model"],
                 target["num_examples"],
                 target["rollouts_per_example"],
@@ -1760,6 +1918,7 @@ def run_eval_cmd(
                     "targets": [],
                     "platform_slugs": [],
                     "environment_ids": [],
+                    "environments": [],
                 }
                 target_order.append(group_key)
             grouped_targets[group_key]["targets"].append(target)
@@ -1768,6 +1927,15 @@ def run_eval_cmd(
             for group_key in target_order:
                 group = grouped_targets[group_key]
                 for grouped_target in group["targets"]:
+                    if grouped_target.get("environment") is not None:
+                        group["environments"].append(grouped_target["environment"])
+                        group["platform_slugs"].append(
+                            _hosted_v1_environment_display_name(
+                                grouped_target["environment"],
+                                len(group["environments"]),
+                            )
+                        )
+                        continue
                     platform_slug, environment_id = _resolve_hosted_environment(
                         grouped_target["env_id"],
                         env_dir_path=grouped_target["env_dir_path"],
@@ -1785,11 +1953,14 @@ def run_eval_cmd(
             for group_key in target_order:
                 group = grouped_targets[group_key]
                 target = group["target"]
+                environment_ids = group["environment_ids"] or None
+                environments = group["environments"] or None
                 hosted_config = HostedEvalConfig(
-                    environment_id=group["environment_ids"][0],
+                    environment_id=environment_ids[0] if environment_ids else None,
                     inference_model=target["model"],
                     num_examples=target["num_examples"],
                     rollouts_per_example=target["rollouts_per_example"],
+                    environments=environments,
                     env_args=target.get("env_args"),
                     name=target.get("eval_name"),
                     timeout_minutes=target.get("timeout_minutes"),
@@ -1811,7 +1982,7 @@ def run_eval_cmd(
                 )
                 result = _create_hosted_evaluations(
                     hosted_config,
-                    environment_ids=group["environment_ids"],
+                    environment_ids=environment_ids,
                 )
                 all_platform_slugs.extend(group["platform_slugs"])
                 all_evaluation_ids.extend(result.get("evaluation_ids") or [result["evaluation_id"]])
diff --git a/packages/prime/src/prime_cli/utils/hosted_eval.py b/packages/prime/src/prime_cli/utils/hosted_eval.py
index f714c4bb..78bbba5a 100644
--- a/packages/prime/src/prime_cli/utils/hosted_eval.py
+++ b/packages/prime/src/prime_cli/utils/hosted_eval.py
@@ -35,10 +35,11 @@ def color(self) -> str:
 
 @dataclass
 class HostedEvalConfig:
-    environment_id: str
+    environment_id: Optional[str]
     inference_model: str
     num_examples: int
     rollouts_per_example: int
+    environments: Optional[list[dict[str, Any]]] = None
     env_args: Optional[dict[str, Any]] = None
     name: Optional[str] = None
     timeout_minutes: Optional[int] = None
diff --git a/packages/prime/tests/test_hosted_eval.py b/packages/prime/tests/test_hosted_eval.py
index 148e2428..cf83187b 100644
--- a/packages/prime/tests/test_hosted_eval.py
+++ b/packages/prime/tests/test_hosted_eval.py
@@ -364,6 +364,45 @@ def post(self, endpoint, json=None):
     assert captured["json"]["eval_config"]["allow_tunnel_access"] is True
 
 
+def test_create_hosted_evaluation_sends_v1_environments_payload(monkeypatch):
+    captured = {}
+
+    class DummyConfig:
+        team_id = None
+
+    class DummyAPIClient:
+        def __init__(self):
+            self.config = DummyConfig()
+
+        def post(self, endpoint, json=None):
+            captured["endpoint"] = endpoint
+            captured["json"] = json
+            return {"evaluation_id": "eval-123"}
+
+    monkeypatch.setattr("prime_cli.commands.evals.APIClient", DummyAPIClient)
+
+    environment = {
+        "taskset": {"id": "gsm8k-v1", "split": "test"},
+        "harness": {"id": "default"},
+    }
+
+    _create_hosted_evaluations(
+        HostedEvalConfig(
+            environment_id=None,
+            environments=[environment],
+            inference_model="openai/gpt-4.1-mini",
+            num_examples=1,
+            rollouts_per_example=1,
+        )
+    )
+
+    assert captured["endpoint"] == "/hosted-evaluations"
+    assert "environment_ids" not in captured["json"]
+    assert captured["json"]["environments"] == [environment]
+    assert captured["json"]["eval_config"]["num_examples"] == 1
+    assert captured["json"]["eval_config"]["rollouts_per_example"] == 1
+
+
 def test_create_hosted_evaluation_includes_sampling_args_in_payload(monkeypatch):
     captured = {}
 
@@ -731,6 +770,91 @@ def fake_run_hosted_evaluation(config, environment_ids=None):
     }
 
 
+def test_hosted_eval_config_accepts_v1_taskset_and_harness(tmp_path):
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+num_examples = 7
+rollouts_per_example = 2
+
+[[eval]]
+taskset = { id = "gsm8k-v1", split = "test" }
+harness = { id = "default" }
+""".strip()
+    )
+
+    loaded = _load_hosted_eval_configs(str(config_path))[0]
+
+    assert "env_id" not in loaded
+    assert loaded["environment"] == {
+        "taskset": {"id": "gsm8k-v1", "split": "test"},
+        "harness": {"id": "default"},
+    }
+    assert loaded["num_examples"] == 7
+    assert loaded["rollouts_per_example"] == 2
+
+
+def test_eval_run_hosted_supports_v1_eval_toml(monkeypatch, tmp_path):
+    captured = {}
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+num_examples = 1
+rollouts_per_example = 1
+
+[[eval]]
+taskset = { id = "gsm8k-v1" }
+harness = { id = "default" }
+""".strip()
+    )
+
+    monkeypatch.setattr(
+        "prime_cli.commands.evals._resolve_hosted_environment",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(
+            AssertionError("v1 hosted evals should not resolve published environments")
+        ),
+    )
+
+    def fake_run_hosted_evaluation(config, environment_ids=None):
+        captured["environment_id"] = config.environment_id
+        captured["environment_ids"] = environment_ids
+        captured["environments"] = config.environments
+        captured["inference_model"] = config.inference_model
+        captured["num_examples"] = config.num_examples
+        captured["rollouts_per_example"] = config.rollouts_per_example
+        return {"evaluation_id": "eval-123"}
+
+    monkeypatch.setattr(
+        "prime_cli.commands.evals._create_hosted_evaluations",
+        fake_run_hosted_evaluation,
+    )
+
+    result = runner.invoke(
+        app,
+        ["eval", "run", str(config_path), "--hosted"],
+        env={"PRIME_DISABLE_VERSION_CHECK": "1"},
+    )
+
+    assert result.exit_code == 0, result.output
+    assert captured == {
+        "environment_id": None,
+        "environment_ids": None,
+        "environments": [
+            {
+                "taskset": {"id": "gsm8k-v1"},
+                "harness": {"id": "default"},
+            }
+        ],
+        "inference_model": "openai/gpt-4.1-mini",
+        "num_examples": 1,
+        "rollouts_per_example": 1,
+    }
+    assert "Environment:" in result.output
+    assert "gsm8k-v1" in result.output
+
+
 def test_eval_run_hosted_cli_overrides_supported_toml_fields(monkeypatch, tmp_path):
     captured = {}
     config_path = tmp_path / "eval.toml"
@@ -1132,6 +1256,71 @@ def test_eval_run_hosted_rejects_explicit_unsupported_toml_fields(
     assert f"`{expected_field}`" in result.output
 
 
+def test_eval_run_hosted_rejects_toml_without_selector(tmp_path):
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+
+[[eval]]
+env_args = { split = "test" }
+""".strip()
+    )
+
+    result = runner.invoke(
+        app,
+        ["eval", "run", str(config_path), "--hosted"],
+        env={"PRIME_DISABLE_VERSION_CHECK": "1"},
+    )
+
+    assert result.exit_code == 1
+    assert "requires either `env_id` or `taskset.id`" in result.output
+
+
+def test_eval_run_hosted_rejects_mixed_env_id_and_taskset(tmp_path):
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+
+[[eval]]
+env_id = "primeintellect/gsm8k"
+taskset = { id = "gsm8k-v1" }
+""".strip()
+    )
+
+    result = runner.invoke(
+        app,
+        ["eval", "run", str(config_path), "--hosted"],
+        env={"PRIME_DISABLE_VERSION_CHECK": "1"},
+    )
+
+    assert result.exit_code == 1
+    assert "cannot combine `env_id` with `taskset` or `harness`" in result.output
+
+
+def test_eval_run_hosted_rejects_v1_selector_without_taskset_id(tmp_path):
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+
+[[eval]]
+harness = { id = "default" }
+""".strip()
+    )
+
+    result = runner.invoke(
+        app,
+        ["eval", "run", str(config_path), "--hosted"],
+        env={"PRIME_DISABLE_VERSION_CHECK": "1"},
+    )
+
+    assert result.exit_code == 1
+    assert "Environment config requires either legacy id" in result.output
+    assert "taskset.id" in result.output
+
+
 def test_eval_run_hosted_toml_preserves_cli_env_dir_path(monkeypatch, tmp_path):
     captured = {}
     config_path = tmp_path / "eval.toml"

From 6353b60062b9e1feefb593b6a278b53ddecc9114 Mon Sep 17 00:00:00 2001
From: Oscar Moxon <oscarjmoxon@gmail.com>
Date: Mon, 29 Jun 2026 22:33:54 -0700
Subject: [PATCH 3/4] ENG-4321 Detect top-level v1 hosted eval selectors

---
 .../prime/src/prime_cli/commands/evals.py     |  3 ++-
 packages/prime/tests/test_hosted_eval.py      | 25 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/packages/prime/src/prime_cli/commands/evals.py b/packages/prime/src/prime_cli/commands/evals.py
index d036348b..af028e2d 100644
--- a/packages/prime/src/prime_cli/commands/evals.py
+++ b/packages/prime/src/prime_cli/commands/evals.py
@@ -575,7 +575,8 @@ def normalize_hosted_eval_sections(raw: dict[str, Any]) -> dict[str, Any]:
                 f"Config file uses [eval] but should use [[eval]] (double brackets) "
                 f"for array of tables: {config_path}"
             )
-        uses_v1_selector = any(
+        has_global_v1_selector = "taskset" in raw_config or "harness" in raw_config
+        uses_v1_selector = has_global_v1_selector or any(
             isinstance(eval_config, dict)
             and ("taskset" in eval_config or "harness" in eval_config)
             for eval_config in eval_list
diff --git a/packages/prime/tests/test_hosted_eval.py b/packages/prime/tests/test_hosted_eval.py
index cf83187b..166cb6d5 100644
--- a/packages/prime/tests/test_hosted_eval.py
+++ b/packages/prime/tests/test_hosted_eval.py
@@ -795,6 +795,31 @@ def test_hosted_eval_config_accepts_v1_taskset_and_harness(tmp_path):
     assert loaded["rollouts_per_example"] == 2
 
 
+def test_hosted_eval_config_accepts_top_level_v1_taskset_and_harness(tmp_path):
+    config_path = tmp_path / "eval.toml"
+    config_path.write_text(
+        """
+model = "openai/gpt-4.1-mini"
+taskset = { id = "gsm8k-v1", split = "test" }
+harness = { id = "default" }
+
+[[eval]]
+num_examples = 7
+rollouts_per_example = 2
+""".strip()
+    )
+
+    loaded = _load_hosted_eval_configs(str(config_path))[0]
+
+    assert "env_id" not in loaded
+    assert loaded["environment"] == {
+        "taskset": {"id": "gsm8k-v1", "split": "test"},
+        "harness": {"id": "default"},
+    }
+    assert loaded["num_examples"] == 7
+    assert loaded["rollouts_per_example"] == 2
+
+
 def test_eval_run_hosted_supports_v1_eval_toml(monkeypatch, tmp_path):
     captured = {}
     config_path = tmp_path / "eval.toml"

From 3e9ba7f202b90f914f08dce58f43e63f160ffa36 Mon Sep 17 00:00:00 2001
From: Oscar Moxon <oscarjmoxon@gmail.com>
Date: Mon, 29 Jun 2026 22:48:16 -0700
Subject: [PATCH 4/4] ENG-4321 Address hosted eval parser review feedback

---
 .../prime/src/prime_cli/commands/evals.py     | 42 +++++++++++----
 packages/prime/tests/test_hosted_eval.py      | 52 ++++---------------
 2 files changed, 41 insertions(+), 53 deletions(-)

diff --git a/packages/prime/src/prime_cli/commands/evals.py b/packages/prime/src/prime_cli/commands/evals.py
index af028e2d..3724ad45 100644
--- a/packages/prime/src/prime_cli/commands/evals.py
+++ b/packages/prime/src/prime_cli/commands/evals.py
@@ -581,13 +581,19 @@ def normalize_hosted_eval_sections(raw: dict[str, Any]) -> dict[str, Any]:
             and ("taskset" in eval_config or "harness" in eval_config)
             for eval_config in eval_list
         )
-        if not uses_v1_selector:
-            if any(
-                isinstance(eval_config, dict)
-                and "env_id" not in eval_config
-                and "id" not in eval_config
-                for eval_config in eval_list
-            ):
+        has_global_legacy_selector = "env_id" in raw_config or "id" in raw_config
+        evals_missing_legacy_selector = any(
+            isinstance(eval_config, dict)
+            and "env_id" not in eval_config
+            and "id" not in eval_config
+            for eval_config in eval_list
+        )
+        uses_hosted_merge = uses_v1_selector or (
+            has_global_legacy_selector
+            and (evals_missing_legacy_selector or "id" in raw_config)
+        )
+        if not uses_hosted_merge:
+            if evals_missing_legacy_selector:
                 raise ValueError(
                     "hosted eval config requires either `env_id` or `taskset.id`"
                 )
@@ -607,15 +613,27 @@ def normalize_hosted_eval_sections(raw: dict[str, Any]) -> dict[str, Any]:
                 f"(double brackets) for array of tables: {config_path}"
             )
         if ablation_list:
-            raise ValueError("hosted eval v1 environment configs do not support ablations")
+            if uses_v1_selector:
+                raise ValueError(
+                    "hosted eval v1 environment configs do not support ablations"
+                )
+            raise ValueError(
+                "hosted eval configs with top-level `env_id` defaults on "
+                "selector-less [[eval]] sections do not support ablations"
+            )
         if not eval_list:
             raise ValueError(
                 f"Config file must contain at least one [[eval]] section: {config_path}"
             )
 
-        global_defaults = {
-            key: value for key, value in raw_config.items() if key not in ("eval", "ablation")
-        }
+        global_defaults = normalize_env_id_alias(
+            {
+                key: value
+                for key, value in raw_config.items()
+                if key not in ("eval", "ablation")
+            },
+            "global defaults",
+        )
         loaded_configs = []
         for eval_config in eval_list:
             normalized_eval = normalize_env_id_alias(eval_config, "[[eval]]")
@@ -634,6 +652,8 @@ def normalize_hosted_eval_sections(raw: dict[str, Any]) -> dict[str, Any]:
                     merged["endpoints_path"] = str(
                         (config_path.parent / endpoints_path_obj).resolve()
                     )
+    except typer.Exit:
+        raise
     except Exception as exc:
         console.print(f"[red]Error:[/red] {exc}")
         raise typer.Exit(1) from exc
diff --git a/packages/prime/tests/test_hosted_eval.py b/packages/prime/tests/test_hosted_eval.py
index 166cb6d5..9f9154ff 100644
--- a/packages/prime/tests/test_hosted_eval.py
+++ b/packages/prime/tests/test_hosted_eval.py
@@ -347,9 +347,11 @@ def post(self, endpoint, json=None):
 
     monkeypatch.setattr("prime_cli.commands.evals.APIClient", DummyAPIClient)
 
+    environment = {"taskset": {"id": "gsm8k-v1"}, "harness": {"id": "default"}}
     result = _create_hosted_evaluations(
         HostedEvalConfig(
-            environment_id="env-123",
+            environment_id=None,
+            environments=[environment],
             inference_model="openai/gpt-4.1-mini",
             num_examples=5,
             rollouts_per_example=3,
@@ -359,50 +361,13 @@ def post(self, endpoint, json=None):
     assert result["evaluation_id"] == "eval-123"
     assert captured["endpoint"] == "/hosted-evaluations"
     assert captured["json"]["team_id"] == "team-123"
+    assert "environment_ids" not in captured["json"]
+    assert captured["json"]["environments"] == [environment]
     assert captured["json"]["eval_config"]["allow_sandbox_access"] is True
     assert captured["json"]["eval_config"]["allow_instances_access"] is False
     assert captured["json"]["eval_config"]["allow_tunnel_access"] is True
 
 
-def test_create_hosted_evaluation_sends_v1_environments_payload(monkeypatch):
-    captured = {}
-
-    class DummyConfig:
-        team_id = None
-
-    class DummyAPIClient:
-        def __init__(self):
-            self.config = DummyConfig()
-
-        def post(self, endpoint, json=None):
-            captured["endpoint"] = endpoint
-            captured["json"] = json
-            return {"evaluation_id": "eval-123"}
-
-    monkeypatch.setattr("prime_cli.commands.evals.APIClient", DummyAPIClient)
-
-    environment = {
-        "taskset": {"id": "gsm8k-v1", "split": "test"},
-        "harness": {"id": "default"},
-    }
-
-    _create_hosted_evaluations(
-        HostedEvalConfig(
-            environment_id=None,
-            environments=[environment],
-            inference_model="openai/gpt-4.1-mini",
-            num_examples=1,
-            rollouts_per_example=1,
-        )
-    )
-
-    assert captured["endpoint"] == "/hosted-evaluations"
-    assert "environment_ids" not in captured["json"]
-    assert captured["json"]["environments"] == [environment]
-    assert captured["json"]["eval_config"]["num_examples"] == 1
-    assert captured["json"]["eval_config"]["rollouts_per_example"] == 1
-
-
 def test_create_hosted_evaluation_includes_sampling_args_in_payload(monkeypatch):
     captured = {}
 
@@ -1322,6 +1287,7 @@ def test_eval_run_hosted_rejects_mixed_env_id_and_taskset(tmp_path):
 
     assert result.exit_code == 1
     assert "cannot combine `env_id` with `taskset` or `harness`" in result.output
+    assert "Error: 1" not in result.output
 
 
 def test_eval_run_hosted_rejects_v1_selector_without_taskset_id(tmp_path):
@@ -1392,20 +1358,22 @@ def fake_run_hosted_evaluation(config, environment_ids=None):
     }
 
 
-def test_hosted_eval_config_accepts_id_alias(tmp_path):
+def test_hosted_eval_config_accepts_top_level_id_alias(tmp_path):
     config_path = tmp_path / "eval.toml"
     config_path.write_text(
         """
 model = "openai/gpt-4.1-mini"
+id = "gsm8k"
 
 [[eval]]
-id = "gsm8k"
+num_examples = 3
 """.strip()
     )
 
     loaded = _load_hosted_eval_configs(str(config_path))[0]
 
     assert loaded["env_id"] == "gsm8k"
+    assert loaded["num_examples"] == 3
 
 
 def test_eval_run_hosted_endpoint_id_uses_default_endpoints_path_from_cwd(monkeypatch, tmp_path):