From 5b6eef3831792e14ec14332c61bef88b542d75bc Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 11:09:13 -0700 Subject: [PATCH 1/8] ENG-4782: show FFT models in prime train models - adds AvailableFFTModel / list_available_fft_models to HostedTrainingClient - prime train models fetches /training/available-fft-models by default and renders a second Rich table when the endpoint returns any results - JSON output gains available_fft_models when non-empty (backwards compatible) - --fft-only flag suppresses the LoRA section - graceful fallback on 401/403/404 so older backends still render LoRA - picks up an incidental uv.lock re-sort from pre-commit uv-run --- packages/prime/src/prime_cli/api/training.py | 64 ++++- packages/prime/src/prime_cli/commands/rl.py | 169 +++++++++---- packages/prime/tests/test_rl_models.py | 235 +++++++++++++++++-- uv.lock | 6 +- 4 files changed, 406 insertions(+), 68 deletions(-) diff --git a/packages/prime/src/prime_cli/api/training.py b/packages/prime/src/prime_cli/api/training.py index 7b40d65db..f773c92be 100644 --- a/packages/prime/src/prime_cli/api/training.py +++ b/packages/prime/src/prime_cli/api/training.py @@ -6,11 +6,12 @@ token; admin role is gated server-side. """ +from datetime import datetime from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field -from prime_cli.core import APIClient +from prime_cli.core import APIClient, APIError, NotFoundError, UnauthorizedError class HostedTrainingRunResponse(BaseModel): @@ -30,6 +31,38 @@ class AvailableGpuTypesResponse(BaseModel): model_config = ConfigDict(populate_by_name=True) +class FFTModelClusterInfo(BaseModel): + """One cluster the caller can dispatch an FFT run to that already has + the parent model cached — mirrors the backend + `FFTModelClusterInfo` in `platform/backend/app/packages/training/schemas.py`. + """ + + cluster_id: str = Field(..., alias="clusterId") + cluster_name: str = Field(..., alias="clusterName") + gpu_type: Optional[str] = Field(None, alias="gpuType") + cache_synced_at: Optional[datetime] = Field(None, alias="cacheSyncedAt") + + model_config = ConfigDict(populate_by_name=True) + + +class AvailableFFTModel(BaseModel): + """A model that is cached and ready for FFT dispatch on at least one + eligible PrimeCluster.""" + + name: str = Field(..., description="Model name") + clusters: List[FFTModelClusterInfo] = Field(default_factory=list) + + model_config = ConfigDict(populate_by_name=True) + + +class AvailableFFTModelsResponse(BaseModel): + """Response from GET /v1/training/available-fft-models.""" + + models: List[AvailableFFTModel] = Field(default_factory=list) + + model_config = ConfigDict(populate_by_name=True) + + class HostedTrainingClient: """Client for the hosted full-FT training endpoint.""" @@ -71,6 +104,35 @@ def list_available_gpu_types(self, team_id: Optional[str] = None) -> AvailableGp response = self.client.get("/training/available-gpu-types", params=params) return AvailableGpuTypesResponse.model_validate(response) + def list_available_fft_models(self, team_id: Optional[str] = None) -> List[AvailableFFTModel]: + """GET /v1/training/available-fft-models. Models that are already + cached on at least one PrimeCluster the caller can dispatch a + full-FT run to. + + Returns an empty list when the caller has no eligible clusters + (403), when the endpoint is not deployed yet on the target + backend (404) or when auth is missing (401) — the CLI is called + in a "silent when empty" context alongside the LoRA listing and + should not crash the primary output on a still-rolling-out + endpoint. + """ + params: Dict[str, Any] = {} + if team_id: + params["team_id"] = team_id + try: + response = self.client.get("/training/available-fft-models", params=params) + except (NotFoundError, UnauthorizedError): + return [] + except APIError as exc: + # 403 (not a team member) surfaces as APIError with an + # HTTP 403 prefix. Treat like NotFound so a personal caller + # querying with an unrelated team_id doesn't fail the whole + # command. + if "HTTP 403" in str(exc): + return [] + raise + return AvailableFFTModelsResponse.model_validate(response).models + def build_payload_from_toml( cfg: Dict[str, Any], diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index b5f0be35f..b28c644bc 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -70,6 +70,8 @@ "effective_training_price_per_mtok?, " "effective_inference_input_price_per_mtok?, " "effective_inference_output_price_per_mtok?, promo_label?}", + ".available_fft_models[]? = {name, clusters[{cluster_id, cluster_name, " + "gpu_type?, cache_synced_at?}]}", ) RL_LIST_JSON_HELP = json_output_help( @@ -1673,69 +1675,148 @@ def _format(list_p: Any, eff_p: Any) -> str: @app.command("models", rich_help_panel="Commands", epilog=RL_MODELS_JSON_HELP) def list_models( output: str = typer.Option("table", "--output", "-o", help="Output format: table or json"), + fft_only: bool = typer.Option( + False, + "--fft-only", + help="Suppress the LoRA/Hosted Training section and only show FFT models.", + ), ) -> None: - """List available models for Hosted Training.""" + """List available models for Hosted Training. + + Renders two sections when both are populated: the shared-cluster LoRA + models (pricing per 1M tokens) and the models pre-cached on FFT + dispatch clusters. The FFT section is silent when the API returns + nothing so users on backends without the endpoint see the pre-4782 + output. + """ + from ..api.training import AvailableFFTModel, HostedTrainingClient + validate_output_format(output, console) try: api_client = APIClient() rl_client = RLClient(api_client) + training_client = HostedTrainingClient(api_client) config = Config() - models = rl_client.list_models(team_id=config.team_id) + models: List = [] + if not fft_only: + models = rl_client.list_models(team_id=config.team_id) + + fft_models: List[AvailableFFTModel] = [] + try: + fft_models = training_client.list_available_fft_models(team_id=config.team_id) + except APIError: + # Never let an FFT fetch failure break the LoRA output — the + # endpoint is younger and may still be rolling out. When the + # caller explicitly asked for FFT-only we bubble up the + # error instead of silently hiding it. + if fft_only: + raise + fft_models = [] if output == "json": - output_data_as_json({"models": [m.model_dump() for m in models]}, console) + payload: Dict[str, Any] = {"models": [m.model_dump() for m in models]} + if fft_models: + payload["available_fft_models"] = [m.model_dump() for m in fft_models] + output_data_as_json(payload, console) return - if not models: - console.print("[yellow]No models available for Hosted Training.[/yellow]") + if not fft_only: + _render_lora_models_table(models) + + if fft_models: + _render_fft_models_table(fft_models) + elif fft_only: + console.print("[yellow]No FFT models available.[/yellow]") console.print( - "[dim]This could mean no healthy Hosted Training clusters are running.[/dim]" + "[dim]No dispatchable clusters have a warm model cache yet, or the " + "endpoint isn't deployed on this backend.[/dim]" ) - return - table = Table( - title="Hosted Training - Models", + except APIError as e: + console.print(f"[red]Error:[/red] {e}") + raise typer.Exit(1) + + +def _render_lora_models_table(models: List) -> None: + """Render the classic LoRA Hosted Training model listing.""" + if not models: + console.print("[yellow]No models available for Hosted Training.[/yellow]") + console.print("[dim]This could mean no healthy Hosted Training clusters are running.[/dim]") + return + + table = Table( + title="Hosted Training - Models", + ) + table.add_column("Model", style="cyan") + table.add_column("Status") + table.add_column("Input", style="green", justify="right") + table.add_column("Output", style="green", justify="right") + table.add_column("Train", style="green", justify="right") + + promo_labels: List[str] = [] + for model in sorted(models, key=lambda model: _model_name_sort_key(model.name)): + if model.at_capacity: + status = "[red]At Capacity[/red]" + else: + status = "[green]Available[/green]" + list_train, eff_train = model.resolve_prices("training") + list_input, eff_input = model.resolve_prices("inference_input") + list_output, eff_output = model.resolve_prices("inference_output") + table.add_row( + model.name, + status, + format_promo_price(list_input, eff_input) or "-", + format_promo_price(list_output, eff_output) or "-", + format_promo_price(list_train, eff_train) or "-", ) - table.add_column("Model", style="cyan") - table.add_column("Status") - table.add_column("Input", style="green", justify="right") - table.add_column("Output", style="green", justify="right") - table.add_column("Train", style="green", justify="right") - - promo_labels: List[str] = [] - for model in sorted(models, key=lambda model: _model_name_sort_key(model.name)): - if model.at_capacity: - status = "[red]At Capacity[/red]" - else: - status = "[green]Available[/green]" - list_train, eff_train = model.resolve_prices("training") - list_input, eff_input = model.resolve_prices("inference_input") - list_output, eff_output = model.resolve_prices("inference_output") - table.add_row( - model.name, - status, - format_promo_price(list_input, eff_input) or "-", - format_promo_price(list_output, eff_output) or "-", - format_promo_price(list_train, eff_train) or "-", - ) - if model.promo_label and model.promo_label not in promo_labels: - promo_labels.append(model.promo_label) + if model.promo_label and model.promo_label not in promo_labels: + promo_labels.append(model.promo_label) + + caption = ( + "[dim]Prices are per 1M tokens. All models support context windows of 64K tokens.[/dim]" + ) + caption_lines = [caption] + if promo_labels: + joined = ", ".join(rich_escape(label) for label in promo_labels) + caption_lines.append(f"[bold yellow]{joined}[/bold yellow]") + table.caption = "\n".join(caption_lines) + console.print(table) + - caption = ( - "[dim]Prices are per 1M tokens. All models support context windows of 64K tokens.[/dim]" +def _render_fft_models_table(fft_models: List) -> None: + """Render the FFT dispatch model listing. + + Each model may be cached on multiple clusters/GPU types; we collapse + those into a single row with cluster+GPU joined so the table stays + scannable when a large model is warm everywhere. + """ + table = Table(title="Hosted Training - Full Finetuning") + table.add_column("Model", style="cyan") + table.add_column("GPU Type(s)", style="magenta") + table.add_column("Available On", style="green") + table.add_column("Cache Synced", style="dim", justify="right") + + for model in sorted(fft_models, key=lambda m: _model_name_sort_key(m.name)): + gpu_types = sorted({c.gpu_type for c in model.clusters if c.gpu_type}) + cluster_names = sorted({c.cluster_name for c in model.clusters}) + latest_sync = max( + (c.cache_synced_at for c in model.clusters if c.cache_synced_at), + default=None, + ) + table.add_row( + model.name, + ", ".join(gpu_types) or "-", + ", ".join(cluster_names) or "-", + latest_sync.strftime("%Y-%m-%d %H:%M") if latest_sync else "-", ) - caption_lines = [caption] - if promo_labels: - joined = ", ".join(rich_escape(label) for label in promo_labels) - caption_lines.append(f"[bold yellow]{joined}[/bold yellow]") - table.caption = "\n".join(caption_lines) - console.print(table) - except APIError as e: - console.print(f"[red]Error:[/red] {e}") - raise typer.Exit(1) + table.caption = ( + "[dim]Models pre-cached on clusters you can dispatch FFT runs to. " + "Cache Synced shows the most recent cluster manifest.[/dim]" + ) + console.print(table) @app.command("gpus", rich_help_panel="Commands") diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 1101e7692..b30546de5 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -5,6 +5,7 @@ import pytest from prime_cli.commands.rl import _model_name_sort_key +from prime_cli.core.client import NotFoundError from prime_cli.main import app from prime_cli.utils.formatters import strip_ansi from typer.testing import CliRunner @@ -37,11 +38,59 @@ def _models_payload() -> Dict[str, Any]: } -def _mock_get_factory(calls: List[str]): +def _fft_models_payload() -> Dict[str, Any]: + return { + "models": [ + { + "name": "meta-llama/Llama-3.1-8B-Instruct", + "clusters": [ + { + "clusterId": "cluster-a", + "clusterName": "athens", + "gpuType": "H200_141GB", + "cacheSyncedAt": "2026-06-01T10:15:00Z", + }, + { + "clusterId": "cluster-b", + "clusterName": "berlin", + "gpuType": "H100_80GB", + "cacheSyncedAt": "2026-06-02T08:00:00Z", + }, + ], + }, + { + "name": "qwen/qwen3-8b", + "clusters": [ + { + "clusterId": "cluster-a", + "clusterName": "athens", + "gpuType": "H200_141GB", + "cacheSyncedAt": "2026-06-01T10:15:00Z", + } + ], + }, + ] + } + + +def _mock_get_factory( + calls: List[str], + *, + fft_payload: Dict[str, Any] | None = None, +): + """Mock APIClient.get for the models command. + + By default returns an empty FFT list so tests that pre-date the FFT + endpoint continue to exercise the LoRA-only rendering path. Pass + ``fft_payload`` to opt into a populated FFT response. + """ + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: calls.append(endpoint) if endpoint == "/rft/models": return _models_payload() + if endpoint == "/training/available-fft-models": + return fft_payload if fft_payload is not None else {"models": []} raise AssertionError(f"Unexpected endpoint: {endpoint}") return mock_get @@ -60,7 +109,9 @@ def test_models_table_renders_pricing(monkeypatch: pytest.MonkeyPatch) -> None: assert "$3" in result.output # Null pricing renders as a dash. assert "-" in result.output - assert calls == ["/rft/models"] + # LoRA endpoint is always hit; FFT endpoint is polled every time so + # the table shows up transparently when it starts returning data. + assert calls == ["/rft/models", "/training/available-fft-models"] def test_models_json_includes_pricing(monkeypatch: pytest.MonkeyPatch) -> None: @@ -82,7 +133,11 @@ def test_models_handles_backend_without_pricing_fields( """Older backends may not return the pricing fields at all.""" def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: - return {"models": [{"name": "qwen/qwen3-8b", "atCapacity": False}]} + if endpoint == "/rft/models": + return {"models": [{"name": "qwen/qwen3-8b", "atCapacity": False}]} + if endpoint == "/training/available-fft-models": + return {"models": []} + raise AssertionError(f"Unexpected endpoint: {endpoint}") monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) @@ -114,6 +169,8 @@ def test_models_table_renders_promo_arrow_and_caption( monkeypatch: pytest.MonkeyPatch, ) -> None: def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/training/available-fft-models": + return {"models": []} return _promo_payload() monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) @@ -135,6 +192,21 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> assert plain.count("Free RFT week") == 1 +def _lora_only_mock(payload: Dict[str, Any]): + """Return a mock_get that serves ``payload`` for /rft/models and an + empty FFT list for /training/available-fft-models — the shape most + LoRA-focused tests want.""" + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/rft/models": + return payload + if endpoint == "/training/available-fft-models": + return {"models": []} + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + return mock_get + + def test_models_table_no_promo_when_effective_equals_original( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -154,10 +226,7 @@ def test_models_table_no_promo_when_effective_equals_original( ] } - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: - return payload - - monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + monkeypatch.setattr("prime_cli.core.APIClient.get", _lora_only_mock(payload)) result = CliRunner().invoke(app, ["rl", "models"], env={"COLUMNS": "200"}) @@ -186,10 +255,7 @@ def test_models_zero_original_with_promo_does_not_render_free( ] } - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: - return payload - - monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + monkeypatch.setattr("prime_cli.core.APIClient.get", _lora_only_mock(payload)) result = CliRunner().invoke(app, ["rl", "models"], env={"COLUMNS": "200"}) @@ -228,10 +294,7 @@ def test_models_promo_label_deduplicated_across_models( ] } - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: - return payload - - monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + monkeypatch.setattr("prime_cli.core.APIClient.get", _lora_only_mock(payload)) result = CliRunner().invoke(app, ["rl", "models"], env={"COLUMNS": "200"}) @@ -263,10 +326,7 @@ def test_models_table_renders_promo_with_list_fields( ] } - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: - return payload - - monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + monkeypatch.setattr("prime_cli.core.APIClient.get", _lora_only_mock(payload)) result = CliRunner().invoke(app, ["rl", "models"], env={"COLUMNS": "200"}) @@ -282,6 +342,8 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> def test_models_json_includes_effective_fields(monkeypatch: pytest.MonkeyPatch) -> None: def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/training/available-fft-models": + return {"models": []} return _promo_payload() monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) @@ -348,3 +410,138 @@ def test_model_name_sort_key_handles_active_params_case_insensitively() -> None: "org/model-30B-A10b", "org/model-30B", ] + + +def test_models_command_renders_fft_section_when_available( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both LoRA and FFT tables render side by side when the FFT endpoint + returns any results.""" + calls: List[str] = [] + monkeypatch.setattr( + "prime_cli.core.APIClient.get", + _mock_get_factory(calls, fft_payload=_fft_models_payload()), + ) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + # LoRA table survives. + assert "Hosted Training - Models" in plain + assert "qwen/qwen3-8b" in plain + # FFT table shows up too. + assert "Full Finetuning" in plain + assert "meta-llama/Llama-3.1-8B-Instruct" in plain + # Model cached on two clusters + two GPU types. + assert "H100_80GB" in plain + assert "H200_141GB" in plain + assert "athens" in plain + assert "berlin" in plain + # Both endpoints were hit. + assert "/rft/models" in calls + assert "/training/available-fft-models" in calls + + +def test_models_json_output_includes_available_fft_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "prime_cli.core.APIClient.get", + _mock_get_factory([], fft_payload=_fft_models_payload()), + ) + + result = CliRunner().invoke(app, ["train", "models", "--output", "json"]) + + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert [m["name"] for m in data["models"]] == [ + "qwen/qwen3-8b", + "openai/gpt-oss-20b", + ] + fft = data["available_fft_models"] + assert [m["name"] for m in fft] == [ + "meta-llama/Llama-3.1-8B-Instruct", + "qwen/qwen3-8b", + ] + first = fft[0] + assert [c["cluster_name"] for c in first["clusters"]] == ["athens", "berlin"] + assert first["clusters"][0]["gpu_type"] == "H200_141GB" + + +def test_models_json_omits_fft_key_when_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """JSON output stays backwards compatible: no available_fft_models + key when the FFT endpoint returns an empty list.""" + monkeypatch.setattr("prime_cli.core.APIClient.get", _mock_get_factory([])) + + result = CliRunner().invoke(app, ["train", "models", "--output", "json"]) + + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert "models" in data + assert "available_fft_models" not in data + + +def test_models_command_survives_fft_endpoint_404( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Older backends that haven't shipped the FFT endpoint yet should + still get the LoRA listing rendered — the CLI must not crash.""" + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/rft/models": + return _models_payload() + if endpoint == "/training/available-fft-models": + raise NotFoundError("HTTP 404: available-fft-models not deployed on this backend") + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + assert "qwen/qwen3-8b" in plain + assert "Full Finetuning" not in plain + + +def test_models_fft_only_suppresses_lora_section( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: List[str] = [] + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + calls.append(endpoint) + if endpoint == "/training/available-fft-models": + return _fft_models_payload() + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models", "--fft-only"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + assert "Full Finetuning" in plain + assert "meta-llama/Llama-3.1-8B-Instruct" in plain + # LoRA table title should not appear. + assert "Hosted Training - Models" not in plain + # Only the FFT endpoint was fetched. + assert calls == ["/training/available-fft-models"] + + +def test_list_available_fft_models_returns_empty_on_404( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """API client method swallows 404 so `prime train models` can silently + fall back to LoRA-only rendering on backends that haven't shipped the + endpoint yet.""" + from prime_cli.api.training import HostedTrainingClient + from prime_cli.core import APIClient + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + raise NotFoundError("HTTP 404: not found") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + client = HostedTrainingClient(APIClient()) + assert client.list_available_fft_models(team_id=None) == [] diff --git a/uv.lock b/uv.lock index f1c79e93d..c99fcc3e4 100644 --- a/uv.lock +++ b/uv.lock @@ -12,12 +12,10 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P7D" [options.exclude-newer-package] -verifiers = false -tasksets = false -prime-pydantic-config = "2099-12-31T23:00:00Z" prime-tunnel = false prime-sandboxes = false -renderers = "2099-12-31T23:00:00Z" +verifiers = false +tasksets = false harnesses = false prime-evals = false prime = false From e43f1e1c11f494c33b4f5001bdf69fe1e641c6c3 Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 11:52:07 -0700 Subject: [PATCH 2/8] ENG-4782: address bot findings + drop cache_synced_at from output - stop swallowing UnauthorizedError in list_available_fft_models so prime train models --fft-only surfaces auth failures instead of a misleading 'No FFT models available' (flagged by cursor + codex) - only NotFoundError is caught now; the command layer still handles the default-mode 'don't crash the LoRA table' fallback - drop cache_synced_at from the FFT pydantic model + Rich table + JSON output (per user feedback, freshness is an implementation detail the picker already gates on) --- packages/prime/src/prime_cli/api/training.py | 34 ++++------ packages/prime/src/prime_cli/commands/rl.py | 14 +--- packages/prime/tests/test_rl_models.py | 67 ++++++++++++++++++++ 3 files changed, 83 insertions(+), 32 deletions(-) diff --git a/packages/prime/src/prime_cli/api/training.py b/packages/prime/src/prime_cli/api/training.py index f773c92be..1d6a4a7a0 100644 --- a/packages/prime/src/prime_cli/api/training.py +++ b/packages/prime/src/prime_cli/api/training.py @@ -6,12 +6,11 @@ token; admin role is gated server-side. """ -from datetime import datetime from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field -from prime_cli.core import APIClient, APIError, NotFoundError, UnauthorizedError +from prime_cli.core import APIClient, NotFoundError class HostedTrainingRunResponse(BaseModel): @@ -33,14 +32,18 @@ class AvailableGpuTypesResponse(BaseModel): class FFTModelClusterInfo(BaseModel): """One cluster the caller can dispatch an FFT run to that already has - the parent model cached — mirrors the backend - `FFTModelClusterInfo` in `platform/backend/app/packages/training/schemas.py`. + the parent model cached. + + Intentionally narrower than the backend `FFTModelClusterInfo` + schema: `cache_synced_at` is dropped because cache freshness is an + implementation detail the user shouldn't reason about — the + dispatch picker already gates on PRESENT-cache clusters, so any + entry surfaced here is warm. """ cluster_id: str = Field(..., alias="clusterId") cluster_name: str = Field(..., alias="clusterName") gpu_type: Optional[str] = Field(None, alias="gpuType") - cache_synced_at: Optional[datetime] = Field(None, alias="cacheSyncedAt") model_config = ConfigDict(populate_by_name=True) @@ -109,28 +112,19 @@ def list_available_fft_models(self, team_id: Optional[str] = None) -> List[Avail cached on at least one PrimeCluster the caller can dispatch a full-FT run to. - Returns an empty list when the caller has no eligible clusters - (403), when the endpoint is not deployed yet on the target - backend (404) or when auth is missing (401) — the CLI is called - in a "silent when empty" context alongside the LoRA listing and - should not crash the primary output on a still-rolling-out - endpoint. + 404 is swallowed to an empty list so the CLI still renders on + older backends that haven't shipped the endpoint yet. Every + other error (auth failure, forbidden, server errors) propagates + — the caller decides whether to surface or hide it based on + whether the LoRA section already ran. """ params: Dict[str, Any] = {} if team_id: params["team_id"] = team_id try: response = self.client.get("/training/available-fft-models", params=params) - except (NotFoundError, UnauthorizedError): + except NotFoundError: return [] - except APIError as exc: - # 403 (not a team member) surfaces as APIError with an - # HTTP 403 prefix. Treat like NotFound so a personal caller - # querying with an unrelated team_id doesn't fail the whole - # command. - if "HTTP 403" in str(exc): - return [] - raise return AvailableFFTModelsResponse.model_validate(response).models diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index b28c644bc..e96024570 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -70,8 +70,7 @@ "effective_training_price_per_mtok?, " "effective_inference_input_price_per_mtok?, " "effective_inference_output_price_per_mtok?, promo_label?}", - ".available_fft_models[]? = {name, clusters[{cluster_id, cluster_name, " - "gpu_type?, cache_synced_at?}]}", + ".available_fft_models[]? = {name, clusters[{cluster_id, cluster_name, gpu_type?}]}", ) RL_LIST_JSON_HELP = json_output_help( @@ -1796,26 +1795,17 @@ def _render_fft_models_table(fft_models: List) -> None: table.add_column("Model", style="cyan") table.add_column("GPU Type(s)", style="magenta") table.add_column("Available On", style="green") - table.add_column("Cache Synced", style="dim", justify="right") for model in sorted(fft_models, key=lambda m: _model_name_sort_key(m.name)): gpu_types = sorted({c.gpu_type for c in model.clusters if c.gpu_type}) cluster_names = sorted({c.cluster_name for c in model.clusters}) - latest_sync = max( - (c.cache_synced_at for c in model.clusters if c.cache_synced_at), - default=None, - ) table.add_row( model.name, ", ".join(gpu_types) or "-", ", ".join(cluster_names) or "-", - latest_sync.strftime("%Y-%m-%d %H:%M") if latest_sync else "-", ) - table.caption = ( - "[dim]Models pre-cached on clusters you can dispatch FFT runs to. " - "Cache Synced shows the most recent cluster manifest.[/dim]" - ) + table.caption = "[dim]Models pre-cached on clusters you can dispatch FFT runs to.[/dim]" console.print(table) diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index b30546de5..8eda3c4c4 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -467,6 +467,8 @@ def test_models_json_output_includes_available_fft_models( first = fft[0] assert [c["cluster_name"] for c in first["clusters"]] == ["athens", "berlin"] assert first["clusters"][0]["gpu_type"] == "H200_141GB" + # cache_synced_at is intentionally omitted from CLI output. + assert "cache_synced_at" not in first["clusters"][0] def test_models_json_omits_fft_key_when_empty(monkeypatch: pytest.MonkeyPatch) -> None: @@ -545,3 +547,68 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) client = HostedTrainingClient(APIClient()) assert client.list_available_fft_models(team_id=None) == [] + + +def test_list_available_fft_models_propagates_auth_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Auth failures must not be silently converted to an empty list — + otherwise `prime train models --fft-only` on an expired key would + exit 0 with a misleading 'No FFT models available' message.""" + from prime_cli.api.training import HostedTrainingClient + from prime_cli.core import APIClient + from prime_cli.core.client import UnauthorizedError + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + raise UnauthorizedError("API key unauthorized.") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + client = HostedTrainingClient(APIClient()) + with pytest.raises(UnauthorizedError): + client.list_available_fft_models(team_id=None) + + +def test_models_fft_only_surfaces_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: + """--fft-only skips the LoRA call, so an auth failure on the FFT + endpoint is the only signal the caller has that their token is + bad. It must exit non-zero with the auth error, not print the + generic 'no models' fallback.""" + from prime_cli.core.client import UnauthorizedError + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/training/available-fft-models": + raise UnauthorizedError("API key unauthorized.") + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models", "--fft-only"], env={"COLUMNS": "200"}) + + assert result.exit_code != 0 + assert "unauthorized" in result.output.lower() + assert "No FFT models available" not in result.output + + +def test_models_default_hides_fft_auth_error_after_lora_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """In default mode, if LoRA succeeds but the FFT call fails with + anything other than 404, the LoRA table must still render — the FFT + section is best-effort and shouldn't cascade the primary output.""" + from prime_cli.core.client import UnauthorizedError + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/rft/models": + return _models_payload() + if endpoint == "/training/available-fft-models": + raise UnauthorizedError("API key unauthorized.") + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + assert "qwen/qwen3-8b" in plain + assert "Full Finetuning" not in plain From df0e6269bea09632df1ca065a70027dc1e6f9f1c Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 12:21:03 -0700 Subject: [PATCH 3/8] ENG-4782: tidy up prime train models output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rename LoRA table title from 'Hosted Training - Models' to 'Hosted Training — LoRA' (em-dash matches the inference command's 'Prime Inference — Models' style, and the label is more accurate now that FFT is a second section) - rename FFT title to 'Hosted Training — Full Finetuning' for consistency - left-justify title + caption on both tables so the two sections stack cleanly instead of floating with centered text - add a blank line between the LoRA and FFT tables so the two sections don't run into each other - rename 'Available On' column to 'Cluster(s)' to match the 'GPU Type(s)' pluralization pattern --- packages/prime/src/prime_cli/commands/rl.py | 16 +++++++++++++--- packages/prime/tests/test_rl_models.py | 6 +++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index e96024570..037aaea41 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -1725,6 +1725,10 @@ def list_models( _render_lora_models_table(models) if fft_models: + if not fft_only: + # Visual break between the two tables so the sections + # don't run into each other. + console.print() _render_fft_models_table(fft_models) elif fft_only: console.print("[yellow]No FFT models available.[/yellow]") @@ -1746,7 +1750,9 @@ def _render_lora_models_table(models: List) -> None: return table = Table( - title="Hosted Training - Models", + title="Hosted Training — LoRA", + title_justify="left", + caption_justify="left", ) table.add_column("Model", style="cyan") table.add_column("Status") @@ -1791,10 +1797,14 @@ def _render_fft_models_table(fft_models: List) -> None: those into a single row with cluster+GPU joined so the table stays scannable when a large model is warm everywhere. """ - table = Table(title="Hosted Training - Full Finetuning") + table = Table( + title="Hosted Training — Full Finetuning", + title_justify="left", + caption_justify="left", + ) table.add_column("Model", style="cyan") table.add_column("GPU Type(s)", style="magenta") - table.add_column("Available On", style="green") + table.add_column("Cluster(s)", style="green") for model in sorted(fft_models, key=lambda m: _model_name_sort_key(m.name)): gpu_types = sorted({c.gpu_type for c in model.clusters if c.gpu_type}) diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 8eda3c4c4..0b37d1c37 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -428,7 +428,7 @@ def test_models_command_renders_fft_section_when_available( assert result.exit_code == 0, result.output plain = strip_ansi(result.output) # LoRA table survives. - assert "Hosted Training - Models" in plain + assert "LoRA" in plain assert "qwen/qwen3-8b" in plain # FFT table shows up too. assert "Full Finetuning" in plain @@ -526,8 +526,8 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> plain = strip_ansi(result.output) assert "Full Finetuning" in plain assert "meta-llama/Llama-3.1-8B-Instruct" in plain - # LoRA table title should not appear. - assert "Hosted Training - Models" not in plain + # LoRA table title should not appear when --fft-only is set. + assert "LoRA" not in plain # Only the FFT endpoint was fetched. assert calls == ["/training/available-fft-models"] From 489e1a9162200dc10fa38c53c8520bd6ec864551 Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 12:25:09 -0700 Subject: [PATCH 4/8] ENG-4782: drop Cluster(s) column from FFT models table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cluster names ('research', etc.) aren't actionable for users dispatching FFT runs — the picker takes gpu_type, not cluster name. Hiding the column tightens the table and removes noise that surprised users in the review pass. JSON output still carries cluster_id/cluster_name so scripts that need cluster-level fan-out can still get it. --- packages/prime/src/prime_cli/commands/rl.py | 3 --- packages/prime/tests/test_rl_models.py | 8 +++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index 037aaea41..622734c98 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -1804,15 +1804,12 @@ def _render_fft_models_table(fft_models: List) -> None: ) table.add_column("Model", style="cyan") table.add_column("GPU Type(s)", style="magenta") - table.add_column("Cluster(s)", style="green") for model in sorted(fft_models, key=lambda m: _model_name_sort_key(m.name)): gpu_types = sorted({c.gpu_type for c in model.clusters if c.gpu_type}) - cluster_names = sorted({c.cluster_name for c in model.clusters}) table.add_row( model.name, ", ".join(gpu_types) or "-", - ", ".join(cluster_names) or "-", ) table.caption = "[dim]Models pre-cached on clusters you can dispatch FFT runs to.[/dim]" diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 0b37d1c37..37e47d1d9 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -433,11 +433,13 @@ def test_models_command_renders_fft_section_when_available( # FFT table shows up too. assert "Full Finetuning" in plain assert "meta-llama/Llama-3.1-8B-Instruct" in plain - # Model cached on two clusters + two GPU types. + # Model cached on two clusters → two GPU types collapse into the row. assert "H100_80GB" in plain assert "H200_141GB" in plain - assert "athens" in plain - assert "berlin" in plain + # Cluster names are intentionally not rendered in the table; users + # dispatch by gpu_type, not by cluster. + assert "athens" not in plain + assert "berlin" not in plain # Both endpoints were hit. assert "/rft/models" in calls assert "/training/available-fft-models" in calls From a87ed284f8b14e26b9bb86cfdac32bee59a86da4 Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 12:30:14 -0700 Subject: [PATCH 5/8] ENG-4782: convert pydantic parse errors to APIError in FFT fetch Bugbot caught: AvailableFFTModelsResponse.model_validate raises pydantic.ValidationError on a schema-drifted backend payload. The command layer catches APIError only, so a bad FFT response would have killed the LoRA output too. Wrap model_validate and re-raise as APIError so the command's existing fallback swallows it in default mode (LoRA still renders) and surfaces it with --fft-only (matching the auth-error behavior). --- packages/prime/src/prime_cli/api/training.py | 13 +++++- packages/prime/tests/test_rl_models.py | 44 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/packages/prime/src/prime_cli/api/training.py b/packages/prime/src/prime_cli/api/training.py index 1d6a4a7a0..09f3fa5f1 100644 --- a/packages/prime/src/prime_cli/api/training.py +++ b/packages/prime/src/prime_cli/api/training.py @@ -9,8 +9,9 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field +from pydantic import ValidationError as PydanticValidationError -from prime_cli.core import APIClient, NotFoundError +from prime_cli.core import APIClient, APIError, NotFoundError class HostedTrainingRunResponse(BaseModel): @@ -117,6 +118,11 @@ def list_available_fft_models(self, team_id: Optional[str] = None) -> List[Avail other error (auth failure, forbidden, server errors) propagates — the caller decides whether to surface or hide it based on whether the LoRA section already ran. + + A schema-drifted response (pydantic ValidationError) is + re-raised as APIError so the command layer's existing + `except APIError` fallback catches it — otherwise a + non-conforming backend payload would kill the LoRA table too. """ params: Dict[str, Any] = {} if team_id: @@ -125,7 +131,10 @@ def list_available_fft_models(self, team_id: Optional[str] = None) -> List[Avail response = self.client.get("/training/available-fft-models", params=params) except NotFoundError: return [] - return AvailableFFTModelsResponse.model_validate(response).models + try: + return AvailableFFTModelsResponse.model_validate(response).models + except PydanticValidationError as exc: + raise APIError(f"Failed to parse available FFT models response: {exc}") from exc def build_payload_from_toml( diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 37e47d1d9..5a181a37c 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -614,3 +614,47 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> plain = strip_ansi(result.output) assert "qwen/qwen3-8b" in plain assert "Full Finetuning" not in plain + + +def test_list_available_fft_models_converts_pydantic_error_to_apierror( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A schema-drifted backend payload must surface as APIError, not + raw pydantic ValidationError — otherwise the command's `except + APIError` fallback would miss it and the LoRA table would fail to + render alongside the FFT section (Bugbot finding on df0e6269).""" + from prime_cli.api.training import HostedTrainingClient + from prime_cli.core import APIClient, APIError + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + # Missing required "clusters" list, wrong shape on "name". + return {"models": [{"name": 12345, "clusters": "not-a-list"}]} + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + client = HostedTrainingClient(APIClient()) + with pytest.raises(APIError): + client.list_available_fft_models(team_id=None) + + +def test_models_command_survives_fft_schema_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end: even if the FFT endpoint returns an unparseable + payload, `prime train models` should still emit the LoRA table + rather than exiting with an unhandled traceback.""" + + def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + if endpoint == "/rft/models": + return _models_payload() + if endpoint == "/training/available-fft-models": + return {"models": [{"name": 12345, "clusters": "not-a-list"}]} + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + assert "qwen/qwen3-8b" in plain + assert "Full Finetuning" not in plain From 7e01e99c17476718fa580b204361aeca1669209c Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 14:34:59 -0700 Subject: [PATCH 6/8] chore: use lowercase typing generics in FFT models addition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoped strictly to the lines this PR added — preexisting typing.Dict / typing.List / typing.Optional usages in the same files (create_run, delete_run, build_payload_from_toml, older tests, etc.) are left untouched to avoid entangling the FFT diff with a repo-wide cleanup. --- packages/prime/src/prime_cli/api/training.py | 10 +++--- packages/prime/src/prime_cli/commands/rl.py | 12 ++++---- packages/prime/tests/test_rl_models.py | 32 ++++++++++---------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/prime/src/prime_cli/api/training.py b/packages/prime/src/prime_cli/api/training.py index 09f3fa5f1..a8852fb67 100644 --- a/packages/prime/src/prime_cli/api/training.py +++ b/packages/prime/src/prime_cli/api/training.py @@ -44,7 +44,7 @@ class FFTModelClusterInfo(BaseModel): cluster_id: str = Field(..., alias="clusterId") cluster_name: str = Field(..., alias="clusterName") - gpu_type: Optional[str] = Field(None, alias="gpuType") + gpu_type: str | None = Field(None, alias="gpuType") model_config = ConfigDict(populate_by_name=True) @@ -54,7 +54,7 @@ class AvailableFFTModel(BaseModel): eligible PrimeCluster.""" name: str = Field(..., description="Model name") - clusters: List[FFTModelClusterInfo] = Field(default_factory=list) + clusters: list[FFTModelClusterInfo] = Field(default_factory=list) model_config = ConfigDict(populate_by_name=True) @@ -62,7 +62,7 @@ class AvailableFFTModel(BaseModel): class AvailableFFTModelsResponse(BaseModel): """Response from GET /v1/training/available-fft-models.""" - models: List[AvailableFFTModel] = Field(default_factory=list) + models: list[AvailableFFTModel] = Field(default_factory=list) model_config = ConfigDict(populate_by_name=True) @@ -108,7 +108,7 @@ def list_available_gpu_types(self, team_id: Optional[str] = None) -> AvailableGp response = self.client.get("/training/available-gpu-types", params=params) return AvailableGpuTypesResponse.model_validate(response) - def list_available_fft_models(self, team_id: Optional[str] = None) -> List[AvailableFFTModel]: + def list_available_fft_models(self, team_id: str | None = None) -> list[AvailableFFTModel]: """GET /v1/training/available-fft-models. Models that are already cached on at least one PrimeCluster the caller can dispatch a full-FT run to. @@ -124,7 +124,7 @@ def list_available_fft_models(self, team_id: Optional[str] = None) -> List[Avail `except APIError` fallback catches it — otherwise a non-conforming backend payload would kill the LoRA table too. """ - params: Dict[str, Any] = {} + params: dict[str, Any] = {} if team_id: params["team_id"] = team_id try: diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index 622734c98..3efc54371 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -1698,11 +1698,11 @@ def list_models( training_client = HostedTrainingClient(api_client) config = Config() - models: List = [] + models: list = [] if not fft_only: models = rl_client.list_models(team_id=config.team_id) - fft_models: List[AvailableFFTModel] = [] + fft_models: list[AvailableFFTModel] = [] try: fft_models = training_client.list_available_fft_models(team_id=config.team_id) except APIError: @@ -1715,7 +1715,7 @@ def list_models( fft_models = [] if output == "json": - payload: Dict[str, Any] = {"models": [m.model_dump() for m in models]} + payload: dict[str, Any] = {"models": [m.model_dump() for m in models]} if fft_models: payload["available_fft_models"] = [m.model_dump() for m in fft_models] output_data_as_json(payload, console) @@ -1742,7 +1742,7 @@ def list_models( raise typer.Exit(1) -def _render_lora_models_table(models: List) -> None: +def _render_lora_models_table(models: list) -> None: """Render the classic LoRA Hosted Training model listing.""" if not models: console.print("[yellow]No models available for Hosted Training.[/yellow]") @@ -1760,7 +1760,7 @@ def _render_lora_models_table(models: List) -> None: table.add_column("Output", style="green", justify="right") table.add_column("Train", style="green", justify="right") - promo_labels: List[str] = [] + promo_labels: list[str] = [] for model in sorted(models, key=lambda model: _model_name_sort_key(model.name)): if model.at_capacity: status = "[red]At Capacity[/red]" @@ -1790,7 +1790,7 @@ def _render_lora_models_table(models: List) -> None: console.print(table) -def _render_fft_models_table(fft_models: List) -> None: +def _render_fft_models_table(fft_models: list) -> None: """Render the FFT dispatch model listing. Each model may be cached on multiple clusters/GPU types; we collapse diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 5a181a37c..84bd003ad 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -38,7 +38,7 @@ def _models_payload() -> Dict[str, Any]: } -def _fft_models_payload() -> Dict[str, Any]: +def _fft_models_payload() -> dict[str, Any]: return { "models": [ { @@ -74,9 +74,9 @@ def _fft_models_payload() -> Dict[str, Any]: def _mock_get_factory( - calls: List[str], + calls: list[str], *, - fft_payload: Dict[str, Any] | None = None, + fft_payload: dict[str, Any] | None = None, ): """Mock APIClient.get for the models command. @@ -85,7 +85,7 @@ def _mock_get_factory( ``fft_payload`` to opt into a populated FFT response. """ - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: calls.append(endpoint) if endpoint == "/rft/models": return _models_payload() @@ -192,12 +192,12 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> assert plain.count("Free RFT week") == 1 -def _lora_only_mock(payload: Dict[str, Any]): +def _lora_only_mock(payload: dict[str, Any]): """Return a mock_get that serves ``payload`` for /rft/models and an empty FFT list for /training/available-fft-models — the shape most LoRA-focused tests want.""" - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: if endpoint == "/rft/models": return payload if endpoint == "/training/available-fft-models": @@ -417,7 +417,7 @@ def test_models_command_renders_fft_section_when_available( ) -> None: """Both LoRA and FFT tables render side by side when the FFT endpoint returns any results.""" - calls: List[str] = [] + calls: list[str] = [] monkeypatch.setattr( "prime_cli.core.APIClient.get", _mock_get_factory(calls, fft_payload=_fft_models_payload()), @@ -492,7 +492,7 @@ def test_models_command_survives_fft_endpoint_404( """Older backends that haven't shipped the FFT endpoint yet should still get the LoRA listing rendered — the CLI must not crash.""" - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: if endpoint == "/rft/models": return _models_payload() if endpoint == "/training/available-fft-models": @@ -512,9 +512,9 @@ def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> def test_models_fft_only_suppresses_lora_section( monkeypatch: pytest.MonkeyPatch, ) -> None: - calls: List[str] = [] + calls: list[str] = [] - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: calls.append(endpoint) if endpoint == "/training/available-fft-models": return _fft_models_payload() @@ -543,7 +543,7 @@ def test_list_available_fft_models_returns_empty_on_404( from prime_cli.api.training import HostedTrainingClient from prime_cli.core import APIClient - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: raise NotFoundError("HTTP 404: not found") monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) @@ -561,7 +561,7 @@ def test_list_available_fft_models_propagates_auth_error( from prime_cli.core import APIClient from prime_cli.core.client import UnauthorizedError - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: raise UnauthorizedError("API key unauthorized.") monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) @@ -577,7 +577,7 @@ def test_models_fft_only_surfaces_auth_error(monkeypatch: pytest.MonkeyPatch) -> generic 'no models' fallback.""" from prime_cli.core.client import UnauthorizedError - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: if endpoint == "/training/available-fft-models": raise UnauthorizedError("API key unauthorized.") raise AssertionError(f"Unexpected endpoint: {endpoint}") @@ -599,7 +599,7 @@ def test_models_default_hides_fft_auth_error_after_lora_succeeds( section is best-effort and shouldn't cascade the primary output.""" from prime_cli.core.client import UnauthorizedError - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: if endpoint == "/rft/models": return _models_payload() if endpoint == "/training/available-fft-models": @@ -626,7 +626,7 @@ def test_list_available_fft_models_converts_pydantic_error_to_apierror( from prime_cli.api.training import HostedTrainingClient from prime_cli.core import APIClient, APIError - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: # Missing required "clusters" list, wrong shape on "name". return {"models": [{"name": 12345, "clusters": "not-a-list"}]} @@ -643,7 +643,7 @@ def test_models_command_survives_fft_schema_drift( payload, `prime train models` should still emit the LoRA table rather than exiting with an unhandled traceback.""" - def mock_get(self: Any, endpoint: str, params: Dict[str, Any] | None = None) -> Dict[str, Any]: + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: if endpoint == "/rft/models": return _models_payload() if endpoint == "/training/available-fft-models": From b5ae42f8e974bb9f5b870431319297692db9c601 Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 14:42:29 -0700 Subject: [PATCH 7/8] ENG-4782: suppress LoRA empty banner when FFT section renders Bugbot caught: LoRA-empty + FFT-populated printed 'No models available for Hosted Training' above the FFT table, which reads as a global failure even though dispatchable FFT models are present. Now only show the banner when both sections are empty. --- packages/prime/src/prime_cli/commands/rl.py | 31 +++++++++---- packages/prime/tests/test_rl_models.py | 50 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index 3efc54371..f1c04a608 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -1722,12 +1722,19 @@ def list_models( return if not fft_only: - _render_lora_models_table(models) + if models: + _render_lora_models_table(models) + elif not fft_models: + # Both sections empty — surface the LoRA fallback so + # the user sees *something*. When FFT has data we + # skip this so the FFT table isn't preceded by a + # misleading "no models available" banner. + _render_empty_lora_message() if fft_models: - if not fft_only: - # Visual break between the two tables so the sections - # don't run into each other. + if not fft_only and models: + # Visual break only when the LoRA table above actually + # rendered — otherwise we'd print a stray blank line. console.print() _render_fft_models_table(fft_models) elif fft_only: @@ -1742,13 +1749,19 @@ def list_models( raise typer.Exit(1) +def _render_empty_lora_message() -> None: + """Print the LoRA-empty fallback used when neither section has data.""" + console.print("[yellow]No models available for Hosted Training.[/yellow]") + console.print("[dim]This could mean no healthy Hosted Training clusters are running.[/dim]") + + def _render_lora_models_table(models: list) -> None: - """Render the classic LoRA Hosted Training model listing.""" - if not models: - console.print("[yellow]No models available for Hosted Training.[/yellow]") - console.print("[dim]This could mean no healthy Hosted Training clusters are running.[/dim]") - return + """Render the classic LoRA Hosted Training model listing. + Caller is responsible for handling the empty-list case (via + `_render_empty_lora_message`) — this helper assumes at least one + row to render. + """ table = Table( title="Hosted Training — LoRA", title_justify="left", diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 84bd003ad..21d84b755 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -658,3 +658,53 @@ def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> plain = strip_ansi(result.output) assert "qwen/qwen3-8b" in plain assert "Full Finetuning" not in plain + + +def test_models_command_suppresses_lora_empty_banner_when_fft_populated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LoRA-empty + FFT-populated: the 'No models available for Hosted + Training' banner would mislead readers into thinking the whole + command failed. Only render the FFT section in that case.""" + + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + if endpoint == "/rft/models": + return {"models": []} + if endpoint == "/training/available-fft-models": + return _fft_models_payload() + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + # The misleading empty-LoRA banner must NOT appear. + assert "No models available for Hosted Training" not in plain + # FFT section still renders. + assert "Full Finetuning" in plain + assert "meta-llama/Llama-3.1-8B-Instruct" in plain + + +def test_models_command_shows_lora_empty_banner_when_both_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both sections empty: the LoRA fallback banner still surfaces so + the user isn't left with a completely blank command output.""" + + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + if endpoint == "/rft/models": + return {"models": []} + if endpoint == "/training/available-fft-models": + return {"models": []} + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models"], env={"COLUMNS": "200"}) + + assert result.exit_code == 0, result.output + plain = strip_ansi(result.output) + assert "No models available for Hosted Training" in plain + assert "Full Finetuning" not in plain From e0ac729a2d7f925e9e8384b8a96af8095ca61030 Mon Sep 17 00:00:00 2001 From: Jannik Straube Date: Fri, 24 Jul 2026 14:54:32 -0700 Subject: [PATCH 8/8] ENG-4782: --fft-only JSON always carries available_fft_models key Codex caught: --fft-only --output json with no FFT models emitted {"models": []}, so scripts consuming .available_fft_models[] would see a missing key on a successful empty result. With --fft-only we now: - drop the misleading models key (LoRA wasn't fetched) - always include available_fft_models (empty allowed) --- packages/prime/src/prime_cli/commands/rl.py | 15 +++++-- packages/prime/tests/test_rl_models.py | 44 +++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/prime/src/prime_cli/commands/rl.py b/packages/prime/src/prime_cli/commands/rl.py index f1c04a608..4315b379b 100644 --- a/packages/prime/src/prime_cli/commands/rl.py +++ b/packages/prime/src/prime_cli/commands/rl.py @@ -1715,9 +1715,18 @@ def list_models( fft_models = [] if output == "json": - payload: dict[str, Any] = {"models": [m.model_dump() for m in models]} - if fft_models: - payload["available_fft_models"] = [m.model_dump() for m in fft_models] + if fft_only: + # --fft-only: LoRA wasn't fetched so emitting `models` + # would be misleading. Always include the FFT key + # (empty allowed) so scripts using the flag can rely + # on `.available_fft_models[]` being present. + payload: dict[str, Any] = { + "available_fft_models": [m.model_dump() for m in fft_models] + } + else: + payload = {"models": [m.model_dump() for m in models]} + if fft_models: + payload["available_fft_models"] = [m.model_dump() for m in fft_models] output_data_as_json(payload, console) return diff --git a/packages/prime/tests/test_rl_models.py b/packages/prime/tests/test_rl_models.py index 21d84b755..a0cb11cfe 100644 --- a/packages/prime/tests/test_rl_models.py +++ b/packages/prime/tests/test_rl_models.py @@ -486,6 +486,50 @@ def test_models_json_omits_fft_key_when_empty(monkeypatch: pytest.MonkeyPatch) - assert "available_fft_models" not in data +def test_models_json_fft_only_always_includes_fft_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With --fft-only, scripts specifically consume + `.available_fft_models[]`. The key must be present even when the + endpoint returned zero models, and the misleading `models: []` key + must be absent since LoRA wasn't fetched.""" + + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + if endpoint == "/training/available-fft-models": + return {"models": []} + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models", "--fft-only", "--output", "json"]) + + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data == {"available_fft_models": []} + + +def test_models_json_fft_only_with_data(monkeypatch: pytest.MonkeyPatch) -> None: + """--fft-only --output json with populated data: still no `models` + key; `available_fft_models` carries the payload.""" + + def mock_get(self: Any, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + if endpoint == "/training/available-fft-models": + return _fft_models_payload() + raise AssertionError(f"Unexpected endpoint: {endpoint}") + + monkeypatch.setattr("prime_cli.core.APIClient.get", mock_get) + + result = CliRunner().invoke(app, ["train", "models", "--fft-only", "--output", "json"]) + + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert "models" not in data + assert [m["name"] for m in data["available_fft_models"]] == [ + "meta-llama/Llama-3.1-8B-Instruct", + "qwen/qwen3-8b", + ] + + def test_models_command_survives_fft_endpoint_404( monkeypatch: pytest.MonkeyPatch, ) -> None: