Recover vanished HF revisions at startup and classify them correctly

src/main.py

@@ -13,8 +13,11 @@
 If vLLM dies during startup for a reason a restart cannot fix (CUDA OOM, a
 MAX_MODEL_LEN the GPU cannot hold, a bad flag, a gated model), the worker stays
 up and answers every job with the cause instead of crash-looping; see
-startup_errors.py. Unrecognised failures still exit non-zero so the platform
-retries them.
+startup_errors.py. One exception is retried in-place first: a Hugging Face
+revision that no longer exists (vLLM pins refs to commit hashes since v0.28, so
+a force-pushed repo invalidates the pin) gets a single relaunch against the
+repo's current state before the error is declared fatal. Unrecognised failures
+still exit non-zero so the platform retries them.
 """
 
 import collections
@@ -149,6 +152,26 @@ def _forward_signal(signum, _frame):
     sys.exit(128 + signum)
 
 
+def drop_pinned_revisions() -> None:
+    """Clear revision pins so the next launch resolves the repo's current state.
+
+    Since v0.28 vLLM resolves every Hugging Face ref to an exact commit hash at
+    launch, so a repo whose history was rewritten 404s on a hash that no longer
+    exists. Relaunching is enough for implicit pins (a fresh launch resolves a
+    fresh hash); explicit *_REVISION values are dropped too, loudly, because
+    serving the current branch beats crash-looping on a commit that is gone.
+    """
+    for name in ("MODEL_REVISION", "TOKENIZER_REVISION", "CODE_REVISION"):
+        value = os.environ.pop(name, None)
+        if value:
+            logging.warning(
+                "Dropping %s=%s — that revision no longer exists on Hugging Face; "
+                "retrying against the repository's current default branch.",
+                name,
+                value,
+            )
+
+
 def main() -> None:
     global vllm_process
 
@@ -170,14 +193,34 @@ def main() -> None:
             "Model pre-flight failed; answering jobs with the cause instead of starting vLLM: %s",
             startup_error,
         )
-    else:
+    # At most two launch attempts: the second only for a vanished Hugging Face
+    # revision, the one startup failure where retrying can genuinely succeed
+    # because vLLM re-resolves the revision on every launch. (The pre-flight
+    # catches a *configured* revision that never existed; this catches one that
+    # existed at pre-flight time and was gone by load time, or a pin vLLM
+    # resolved itself.)
+    attempts_left = 2
+    while startup_error is None:
+        attempts_left -= 1
         vllm_process = start_vllm()
         try:
             wait_for_vllm(vllm_process)
+            break
         except RuntimeError as e:
             logging.error("%s", e)
             stop_vllm(vllm_process)
-            startup_error = startup_errors.classify("".join(recent_output), model=os.getenv("MODEL_NAME"))
+            output = "".join(recent_output)
+            if attempts_left and startup_errors.revision_not_found(output):
+                logging.warning(
+                    "vLLM could not fetch the pinned Hugging Face revision "
+                    "(vLLM pins refs to commit hashes; a force-pushed repo "
+                    "invalidates them). Relaunching once against the current "
+                    "revision."
+                )
+                drop_pinned_revisions()
+                recent_output.clear()
+                continue
+            startup_error = startup_errors.classify(output, model=os.getenv("MODEL_NAME"))
             if startup_error is None:
                 # Nothing recognisable, so let the platform restart us: a failed
                 # download or a bad host is worth another attempt.

src/startup_errors.py

@@ -38,6 +38,12 @@
 _BAD_ARGS = re.compile(r"(?:vllm serve|vllm): error: (.+)")
 
 # --- Model access -----------------------------------------------------------
+# huggingface_hub raises RevisionNotFoundError when a commit/branch/tag does not
+# exist. Since v0.28 vLLM resolves every model ref to an exact commit hash at
+# load time (upstream's fix for "Artifact Pin Decay"), so a repo whose history
+# was force-pushed 404s on a hash that no longer exists. Matched before
+# _NOT_FOUND because the transport error underneath is also a 404.
+_REVISION_NOT_FOUND = re.compile(r"RevisionNotFoundError|Revision Not Found", re.I)
 _GATED = re.compile(r"GatedRepoError|401 Client Error", re.I)
 _NOT_FOUND = re.compile(r"RepositoryNotFoundError|404 Client Error", re.I)
 _UNSUPPORTED_ARCH = re.compile(r"Model architectures \[.*?\] (?:are not supported|failed to be inspected)", re.I)
@@ -73,6 +79,17 @@ def revision_not_found_message(named: str, revision: str) -> str:
     )
 
 
+def revision_not_found(output: str) -> bool:
+    """True when the failure is a Hugging Face revision that no longer exists.
+
+    Split out from classify() because main.py treats this one specially: it is
+    the only startup failure where a fresh attempt against the repo's current
+    state can genuinely succeed (vLLM re-resolves the revision on every launch),
+    so the worker retries once before answering jobs with the error.
+    """
+    return bool(_REVISION_NOT_FOUND.search(output))
+
+
 def classify(output: str, model: Optional[str] = None) -> Optional[str]:
     """One actionable message for a known fatal failure, or None to let it retry."""
     named = model or "The model"
@@ -94,7 +111,12 @@ def classify(output: str, model: Optional[str] = None) -> Optional[str]:
             f"CUDA graph capture, use a quantized checkpoint, or redeploy on a "
             f"larger GPU (or more GPUs with TENSOR_PARALLEL_SIZE). If the model "
             f"loaded but the KV cache did not fit, raising GPU_MEMORY_UTILIZATION "
-            f"a little (default 0.9) can also help."
+            f"a little (default 0.9) can also help. On this vLLM version two more "
+            f"knobs recover memory: MAX_NUM_BATCHED_TOKENS=8192 (the default "
+            f"doubled to 16384 in v0.28, which doubles peak activation memory), "
+            f"and, if KV_CACHE_DTYPE=fp8 is set on an Ampere/Ada GPU, "
+            f"KV_CACHE_DTYPE=auto (fp8 KV forces the FlashInfer backend there, "
+            f"which needs extra workspace memory)."
         )
 
     match = _MAX_LEN_EXCEEDS_MODEL.search(output)
@@ -115,6 +137,19 @@ def classify(output: str, model: Optional[str] = None) -> Optional[str]:
             f"accepted flags and values."
         )
 
+    if _REVISION_NOT_FOUND.search(output):
+        # Distinct from revision_not_found_message: by the time classify sees
+        # this, main.py has already relaunched once against the repo's current
+        # state, so the message must say that and point at the deeper causes.
+        return (
+            f"The Hugging Face revision requested for {named} does not exist "
+            f"(the worker already retried once against the repo's current main "
+            f"branch). If MODEL_REVISION or TOKENIZER_REVISION is set, check it "
+            f"against the revisions listed on the model page, or unset it to use "
+            f"the default branch. This also happens when the repository's history "
+            f"was rewritten (force-pushed) between deploys."
+        )
+
     if _GATED.search(output):
         return gated_message(named)
 

tests/test_main_revision_retry.py

@@ -0,0 +1,87 @@
+"""main() relaunches once for a vanished HF revision, and only for that."""
+
+import sys
+import types
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+import main  # noqa: E402
+
+REVISION_NOT_FOUND = (
+    "huggingface_hub.errors.RevisionNotFoundError: 404 Client Error. Revision Not Found "
+    "for url https://huggingface.co/org/model/resolve/abc123def/config.json."
+)
+TORCH_OOM = "torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 108.00 MiB."
+
+
+@pytest.fixture
+def harness(monkeypatch):
+    """Run main() without vLLM, the RunPod SDK, or a GPU.
+
+    Returns a driver: call it with the outputs each launch attempt should die
+    with (None = the attempt becomes healthy) and it returns the handler module
+    stub, whose startup_error records what jobs would be answered with.
+    """
+    monkeypatch.setenv("MODEL_NAME", "org/model")
+    # The merged pre-flight (model_preflight.py) would otherwise ask the real
+    # HF Hub about "org/model"; these tests are about what happens after it.
+    monkeypatch.setattr(main.model_preflight, "check_model_access", lambda: None)
+    monkeypatch.setattr(main, "stop_vllm", lambda proc: None)
+
+    handler_stub = types.ModuleType("handler")
+    handler_stub.handler = lambda job: None
+    runpod_stub = types.ModuleType("runpod")
+    runpod_stub.serverless = types.SimpleNamespace(start=mock.MagicMock())
+    monkeypatch.setitem(sys.modules, "handler", handler_stub)
+    monkeypatch.setitem(sys.modules, "runpod", runpod_stub)
+
+    start_vllm = mock.MagicMock(return_value=mock.MagicMock())
+    monkeypatch.setattr(main, "start_vllm", start_vllm)
+
+    def drive(*attempt_outputs):
+        outputs = iter(attempt_outputs)
+
+        def fake_wait(proc):
+            output = next(outputs)
+            if output is not None:
+                main.recent_output.clear()
+                main.recent_output.append(output)
+                raise RuntimeError("vLLM serve exited during startup with code 1")
+
+        monkeypatch.setattr(main, "wait_for_vllm", fake_wait)
+        main.recent_output.clear()
+        main.main()
+        handler_stub.launches = start_vllm.call_count
+        return handler_stub
+
+    return drive
+
+
+class TestRevisionRelaunch:
+    def test_vanished_revision_gets_one_relaunch_that_can_succeed(self, harness, monkeypatch):
+        monkeypatch.setenv("MODEL_REVISION", "abc123def")
+
+        handler = harness(REVISION_NOT_FOUND, None)
+
+        assert handler.launches == 2
+        assert handler.startup_error is None
+        # The stale pin must not be replayed into the second launch.
+        import os
+
+        assert "MODEL_REVISION" not in os.environ
+
+    def test_second_revision_failure_answers_jobs_with_the_cause(self, harness):
+        handler = harness(REVISION_NOT_FOUND, REVISION_NOT_FOUND)
+
+        assert handler.launches == 2  # never a third attempt
+        assert "revision" in handler.startup_error.lower()
+
+    def test_other_fatal_failures_do_not_relaunch(self, harness):
+        handler = harness(TORCH_OOM)
+
+        assert handler.launches == 1
+        assert "ran out of GPU memory" in handler.startup_error

tests/test_startup_errors.py

@@ -126,3 +126,43 @@ def test_are_left_for_the_platform_to_retry(self, output):
         # Answering these forever would turn a flaky download into a dead
         # endpoint; a restart is the right response.
         assert classify(output) is None
+
+
+REVISION_NOT_FOUND = (
+    "huggingface_hub.errors.RevisionNotFoundError: 404 Client Error. Revision Not Found "
+    "for url https://huggingface.co/org/model/resolve/abc123def/config.json. "
+    "Invalid rev id: abc123def"
+)
+
+
+class TestRevisionNotFound:
+    def test_names_the_fix_and_the_force_push_cause(self):
+        message = classify(REVISION_NOT_FOUND, model="org/model")
+
+        assert message.startswith("The Hugging Face revision requested for org/model")
+        assert "MODEL_REVISION" in message
+        assert "force-pushed" in message
+
+    def test_wins_over_the_generic_404_match(self):
+        # The transport error underneath a missing revision is also a
+        # "404 Client Error", which the repository-not-found pattern matches.
+        # A vanished revision must not be reported as a missing repository.
+        message = classify(REVISION_NOT_FOUND, model="org/model")
+
+        assert "was not found on Hugging Face" not in message
+
+    def test_predicate_drives_the_relaunch(self):
+        from startup_errors import revision_not_found
+
+        assert revision_not_found(REVISION_NOT_FOUND)
+        assert not revision_not_found(NOT_FOUND)
+        assert not revision_not_found(TORCH_OOM)
+
+
+class TestOomAdviceForV029:
+    def test_mentions_the_batched_tokens_and_fp8_kv_knobs(self):
+        message = classify(TORCH_OOM)
+
+        assert "MAX_NUM_BATCHED_TOKENS=8192" in message
+        assert "KV_CACHE_DTYPE" in message
+        assert "FlashInfer" in message