diff --git a/src/main.py b/src/main.py index 4a40b602..011203be 100644 --- a/src/main.py +++ b/src/main.py @@ -13,8 +13,11 @@ If vLLM dies during startup for a reason a restart cannot fix (CUDA OOM, a MAX_MODEL_LEN the GPU cannot hold, a bad flag, a gated model), the worker stays up and answers every job with the cause instead of crash-looping; see -startup_errors.py. Unrecognised failures still exit non-zero so the platform -retries them. +startup_errors.py. One exception is retried in-place first: a Hugging Face +revision that no longer exists (vLLM pins refs to commit hashes since v0.28, so +a force-pushed repo invalidates the pin) gets a single relaunch against the +repo's current state before the error is declared fatal. Unrecognised failures +still exit non-zero so the platform retries them. """ import collections @@ -149,6 +152,26 @@ def _forward_signal(signum, _frame): sys.exit(128 + signum) +def drop_pinned_revisions() -> None: + """Clear revision pins so the next launch resolves the repo's current state. + + Since v0.28 vLLM resolves every Hugging Face ref to an exact commit hash at + launch, so a repo whose history was rewritten 404s on a hash that no longer + exists. Relaunching is enough for implicit pins (a fresh launch resolves a + fresh hash); explicit *_REVISION values are dropped too, loudly, because + serving the current branch beats crash-looping on a commit that is gone. + """ + for name in ("MODEL_REVISION", "TOKENIZER_REVISION", "CODE_REVISION"): + value = os.environ.pop(name, None) + if value: + logging.warning( + "Dropping %s=%s — that revision no longer exists on Hugging Face; " + "retrying against the repository's current default branch.", + name, + value, + ) + + def main() -> None: global vllm_process @@ -170,14 +193,34 @@ def main() -> None: "Model pre-flight failed; answering jobs with the cause instead of starting vLLM: %s", startup_error, ) - else: + # At most two launch attempts: the second only for a vanished Hugging Face + # revision, the one startup failure where retrying can genuinely succeed + # because vLLM re-resolves the revision on every launch. (The pre-flight + # catches a *configured* revision that never existed; this catches one that + # existed at pre-flight time and was gone by load time, or a pin vLLM + # resolved itself.) + attempts_left = 2 + while startup_error is None: + attempts_left -= 1 vllm_process = start_vllm() try: wait_for_vllm(vllm_process) + break except RuntimeError as e: logging.error("%s", e) stop_vllm(vllm_process) - startup_error = startup_errors.classify("".join(recent_output), model=os.getenv("MODEL_NAME")) + output = "".join(recent_output) + if attempts_left and startup_errors.revision_not_found(output): + logging.warning( + "vLLM could not fetch the pinned Hugging Face revision " + "(vLLM pins refs to commit hashes; a force-pushed repo " + "invalidates them). Relaunching once against the current " + "revision." + ) + drop_pinned_revisions() + recent_output.clear() + continue + startup_error = startup_errors.classify(output, model=os.getenv("MODEL_NAME")) if startup_error is None: # Nothing recognisable, so let the platform restart us: a failed # download or a bad host is worth another attempt. diff --git a/src/startup_errors.py b/src/startup_errors.py index 3be359a1..d57b8915 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -38,6 +38,12 @@ _BAD_ARGS = re.compile(r"(?:vllm serve|vllm): error: (.+)") # --- Model access ----------------------------------------------------------- +# huggingface_hub raises RevisionNotFoundError when a commit/branch/tag does not +# exist. Since v0.28 vLLM resolves every model ref to an exact commit hash at +# load time (upstream's fix for "Artifact Pin Decay"), so a repo whose history +# was force-pushed 404s on a hash that no longer exists. Matched before +# _NOT_FOUND because the transport error underneath is also a 404. +_REVISION_NOT_FOUND = re.compile(r"RevisionNotFoundError|Revision Not Found", re.I) _GATED = re.compile(r"GatedRepoError|401 Client Error", re.I) _NOT_FOUND = re.compile(r"RepositoryNotFoundError|404 Client Error", re.I) _UNSUPPORTED_ARCH = re.compile(r"Model architectures \[.*?\] (?:are not supported|failed to be inspected)", re.I) @@ -73,6 +79,17 @@ def revision_not_found_message(named: str, revision: str) -> str: ) +def revision_not_found(output: str) -> bool: + """True when the failure is a Hugging Face revision that no longer exists. + + Split out from classify() because main.py treats this one specially: it is + the only startup failure where a fresh attempt against the repo's current + state can genuinely succeed (vLLM re-resolves the revision on every launch), + so the worker retries once before answering jobs with the error. + """ + return bool(_REVISION_NOT_FOUND.search(output)) + + def classify(output: str, model: Optional[str] = None) -> Optional[str]: """One actionable message for a known fatal failure, or None to let it retry.""" named = model or "The model" @@ -94,7 +111,12 @@ def classify(output: str, model: Optional[str] = None) -> Optional[str]: f"CUDA graph capture, use a quantized checkpoint, or redeploy on a " f"larger GPU (or more GPUs with TENSOR_PARALLEL_SIZE). If the model " f"loaded but the KV cache did not fit, raising GPU_MEMORY_UTILIZATION " - f"a little (default 0.9) can also help." + f"a little (default 0.9) can also help. On this vLLM version two more " + f"knobs recover memory: MAX_NUM_BATCHED_TOKENS=8192 (the default " + f"doubled to 16384 in v0.28, which doubles peak activation memory), " + f"and, if KV_CACHE_DTYPE=fp8 is set on an Ampere/Ada GPU, " + f"KV_CACHE_DTYPE=auto (fp8 KV forces the FlashInfer backend there, " + f"which needs extra workspace memory)." ) match = _MAX_LEN_EXCEEDS_MODEL.search(output) @@ -115,6 +137,19 @@ def classify(output: str, model: Optional[str] = None) -> Optional[str]: f"accepted flags and values." ) + if _REVISION_NOT_FOUND.search(output): + # Distinct from revision_not_found_message: by the time classify sees + # this, main.py has already relaunched once against the repo's current + # state, so the message must say that and point at the deeper causes. + return ( + f"The Hugging Face revision requested for {named} does not exist " + f"(the worker already retried once against the repo's current main " + f"branch). If MODEL_REVISION or TOKENIZER_REVISION is set, check it " + f"against the revisions listed on the model page, or unset it to use " + f"the default branch. This also happens when the repository's history " + f"was rewritten (force-pushed) between deploys." + ) + if _GATED.search(output): return gated_message(named) diff --git a/tests/test_main_revision_retry.py b/tests/test_main_revision_retry.py new file mode 100644 index 00000000..fef4d1f1 --- /dev/null +++ b/tests/test_main_revision_retry.py @@ -0,0 +1,87 @@ +"""main() relaunches once for a vanished HF revision, and only for that.""" + +import sys +import types +from pathlib import Path +from unittest import mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +import main # noqa: E402 + +REVISION_NOT_FOUND = ( + "huggingface_hub.errors.RevisionNotFoundError: 404 Client Error. Revision Not Found " + "for url https://huggingface.co/org/model/resolve/abc123def/config.json." +) +TORCH_OOM = "torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 108.00 MiB." + + +@pytest.fixture +def harness(monkeypatch): + """Run main() without vLLM, the RunPod SDK, or a GPU. + + Returns a driver: call it with the outputs each launch attempt should die + with (None = the attempt becomes healthy) and it returns the handler module + stub, whose startup_error records what jobs would be answered with. + """ + monkeypatch.setenv("MODEL_NAME", "org/model") + # The merged pre-flight (model_preflight.py) would otherwise ask the real + # HF Hub about "org/model"; these tests are about what happens after it. + monkeypatch.setattr(main.model_preflight, "check_model_access", lambda: None) + monkeypatch.setattr(main, "stop_vllm", lambda proc: None) + + handler_stub = types.ModuleType("handler") + handler_stub.handler = lambda job: None + runpod_stub = types.ModuleType("runpod") + runpod_stub.serverless = types.SimpleNamespace(start=mock.MagicMock()) + monkeypatch.setitem(sys.modules, "handler", handler_stub) + monkeypatch.setitem(sys.modules, "runpod", runpod_stub) + + start_vllm = mock.MagicMock(return_value=mock.MagicMock()) + monkeypatch.setattr(main, "start_vllm", start_vllm) + + def drive(*attempt_outputs): + outputs = iter(attempt_outputs) + + def fake_wait(proc): + output = next(outputs) + if output is not None: + main.recent_output.clear() + main.recent_output.append(output) + raise RuntimeError("vLLM serve exited during startup with code 1") + + monkeypatch.setattr(main, "wait_for_vllm", fake_wait) + main.recent_output.clear() + main.main() + handler_stub.launches = start_vllm.call_count + return handler_stub + + return drive + + +class TestRevisionRelaunch: + def test_vanished_revision_gets_one_relaunch_that_can_succeed(self, harness, monkeypatch): + monkeypatch.setenv("MODEL_REVISION", "abc123def") + + handler = harness(REVISION_NOT_FOUND, None) + + assert handler.launches == 2 + assert handler.startup_error is None + # The stale pin must not be replayed into the second launch. + import os + + assert "MODEL_REVISION" not in os.environ + + def test_second_revision_failure_answers_jobs_with_the_cause(self, harness): + handler = harness(REVISION_NOT_FOUND, REVISION_NOT_FOUND) + + assert handler.launches == 2 # never a third attempt + assert "revision" in handler.startup_error.lower() + + def test_other_fatal_failures_do_not_relaunch(self, harness): + handler = harness(TORCH_OOM) + + assert handler.launches == 1 + assert "ran out of GPU memory" in handler.startup_error diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 3f31deb5..afdc035a 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -126,3 +126,43 @@ def test_are_left_for_the_platform_to_retry(self, output): # Answering these forever would turn a flaky download into a dead # endpoint; a restart is the right response. assert classify(output) is None + + +REVISION_NOT_FOUND = ( + "huggingface_hub.errors.RevisionNotFoundError: 404 Client Error. Revision Not Found " + "for url https://huggingface.co/org/model/resolve/abc123def/config.json. " + "Invalid rev id: abc123def" +) + + +class TestRevisionNotFound: + def test_names_the_fix_and_the_force_push_cause(self): + message = classify(REVISION_NOT_FOUND, model="org/model") + + assert message.startswith("The Hugging Face revision requested for org/model") + assert "MODEL_REVISION" in message + assert "force-pushed" in message + + def test_wins_over_the_generic_404_match(self): + # The transport error underneath a missing revision is also a + # "404 Client Error", which the repository-not-found pattern matches. + # A vanished revision must not be reported as a missing repository. + message = classify(REVISION_NOT_FOUND, model="org/model") + + assert "was not found on Hugging Face" not in message + + def test_predicate_drives_the_relaunch(self): + from startup_errors import revision_not_found + + assert revision_not_found(REVISION_NOT_FOUND) + assert not revision_not_found(NOT_FOUND) + assert not revision_not_found(TORCH_OOM) + + +class TestOomAdviceForV029: + def test_mentions_the_batched_tokens_and_fp8_kv_knobs(self): + message = classify(TORCH_OOM) + + assert "MAX_NUM_BATCHED_TOKENS=8192" in message + assert "KV_CACHE_DTYPE" in message + assert "FlashInfer" in message