From a6adef48083101216b9e983e6ba4d22fe769b714 Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 07:29:15 +0000 Subject: [PATCH 1/3] More accurate handling of LLM errors --- gramps_webapi/api/llm/__init__.py | 20 +++++++-- tests/test_llm_errors.py | 67 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 tests/test_llm_errors.py diff --git a/gramps_webapi/api/llm/__init__.py b/gramps_webapi/api/llm/__init__.py index af7b8d79..b8b48eeb 100644 --- a/gramps_webapi/api/llm/__init__.py +++ b/gramps_webapi/api/llm/__init__.py @@ -5,9 +5,16 @@ from collections.abc import Callable from typing import Any +import httpx from flask import current_app from pydantic_ai import ModelMessagesTypeAdapter -from pydantic_ai.exceptions import ModelRetry, UnexpectedModelBehavior, UsageLimitExceeded +from pydantic_ai.exceptions import ( + ModelAPIError, + ModelHTTPError, + ModelRetry, + UnexpectedModelBehavior, + UsageLimitExceeded, +) from pydantic_ai.messages import ( ModelRequest, ModelResponse, @@ -180,9 +187,16 @@ def answer_with_agent( except UsageLimitExceeded as e: logger.warning("Agent usage limit exceeded: %s", e) abort_with_message(429, "The AI agent exceeded its usage limits for this request.") + except ModelHTTPError as e: + logger.error("Model provider returned an error: %s", e) + abort_with_message(502, "The AI model provider returned an error.") + except (ModelAPIError, httpx.TransportError) as e: + # network failure or timeout talking to the provider + logger.error("Model provider request failed: %r", e) + abort_with_message(504, "The AI model did not respond. Please try again.") except (UnexpectedModelBehavior, ModelRetry) as e: logger.error("Pydantic AI error: %s", e) abort_with_message(500, "Error communicating with the AI model") - except Exception as e: - logger.error("Unexpected error in agent: %s", e) + except Exception: + logger.exception("Unexpected error in agent") abort_with_message(500, "Unexpected error.") diff --git a/tests/test_llm_errors.py b/tests/test_llm_errors.py new file mode 100644 index 00000000..ef104282 --- /dev/null +++ b/tests/test_llm_errors.py @@ -0,0 +1,67 @@ +# +# Gramps Web API - A RESTful API for the Gramps genealogy program +# +# Copyright (C) 2026 David Straub +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +"""Tests for how agent failures are mapped to HTTP status codes.""" + +import httpx +import pytest +from flask import Flask +from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UnexpectedModelBehavior +from werkzeug.exceptions import HTTPException + +from gramps_webapi.api.llm import answer_with_agent + + +@pytest.fixture(name="app") +def fixture_app(): + """A minimal app providing the LLM config.""" + app = Flask(__name__) + app.config.update(LLM_MODEL="test-model", LLM_BASE_URL=None, LLM_SYSTEM_PROMPT=None) + return app + + +@pytest.mark.parametrize( + "error,status", + [ + (httpx.ReadTimeout(""), 504), + (ModelAPIError(model_name="test-model", message="connection error"), 504), + (ModelHTTPError(status_code=503, model_name="test-model"), 502), + (UnexpectedModelBehavior("garbage"), 500), + (RuntimeError("boom"), 500), + ], +) +def test_agent_error_status_codes(app, monkeypatch, error, status): + """Failures from the model provider get a status code of their own.""" + + class _Agent: + def run_sync(self, *args, **kwargs): + raise error + + monkeypatch.setattr("gramps_webapi.api.llm.create_agent", lambda **kwargs: _Agent()) + + with app.app_context(): + with pytest.raises(HTTPException) as excinfo: + answer_with_agent( + prompt="Who was my grandmother?", + tree="tree", + include_private=False, + user_id="user", + ) + + assert excinfo.value.code == status From 56bd2338e5c6ef4895a66167300ebad83ec6f700 Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 07:51:09 +0000 Subject: [PATCH 2/3] Address comment --- gramps_webapi/api/llm/__init__.py | 7 +++++++ tests/test_llm_errors.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/gramps_webapi/api/llm/__init__.py b/gramps_webapi/api/llm/__init__.py index b8b48eeb..1325266b 100644 --- a/gramps_webapi/api/llm/__init__.py +++ b/gramps_webapi/api/llm/__init__.py @@ -189,6 +189,13 @@ def answer_with_agent( abort_with_message(429, "The AI agent exceeded its usage limits for this request.") except ModelHTTPError as e: logger.error("Model provider returned an error: %s", e) + if e.status_code == 429: + abort_with_message( + 429, "The AI model provider is rate limiting requests. Please try again later." + ) + if e.status_code in (502, 503, 504, 529): + # 529 is Anthropic's "overloaded" + abort_with_message(503, "The AI model provider is temporarily unavailable.") abort_with_message(502, "The AI model provider returned an error.") except (ModelAPIError, httpx.TransportError) as e: # network failure or timeout talking to the provider diff --git a/tests/test_llm_errors.py b/tests/test_llm_errors.py index ef104282..30e365ff 100644 --- a/tests/test_llm_errors.py +++ b/tests/test_llm_errors.py @@ -41,7 +41,10 @@ def fixture_app(): [ (httpx.ReadTimeout(""), 504), (ModelAPIError(model_name="test-model", message="connection error"), 504), - (ModelHTTPError(status_code=503, model_name="test-model"), 502), + (ModelHTTPError(status_code=429, model_name="test-model"), 429), + (ModelHTTPError(status_code=503, model_name="test-model"), 503), + (ModelHTTPError(status_code=400, model_name="test-model"), 502), + (ModelHTTPError(status_code=401, model_name="test-model"), 502), (UnexpectedModelBehavior("garbage"), 500), (RuntimeError("boom"), 500), ], From 5b0d1376c2f5429ec09656cc1b91927fa3215d81 Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 15:00:01 +0000 Subject: [PATCH 3/3] Address comment --- gramps_webapi/api/llm/__init__.py | 7 +++++-- tests/test_llm_errors.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/gramps_webapi/api/llm/__init__.py b/gramps_webapi/api/llm/__init__.py index 1325266b..ced69adf 100644 --- a/gramps_webapi/api/llm/__init__.py +++ b/gramps_webapi/api/llm/__init__.py @@ -197,10 +197,13 @@ def answer_with_agent( # 529 is Anthropic's "overloaded" abort_with_message(503, "The AI model provider is temporarily unavailable.") abort_with_message(502, "The AI model provider returned an error.") + except httpx.TimeoutException as e: + logger.error("Model provider request timed out: %r", e) + abort_with_message(504, "The AI model did not respond in time. Please try again.") except (ModelAPIError, httpx.TransportError) as e: - # network failure or timeout talking to the provider + # connection refused, DNS failure, protocol error, or an SDK-wrapped equivalent logger.error("Model provider request failed: %r", e) - abort_with_message(504, "The AI model did not respond. Please try again.") + abort_with_message(503, "The AI model provider is temporarily unavailable.") except (UnexpectedModelBehavior, ModelRetry) as e: logger.error("Pydantic AI error: %s", e) abort_with_message(500, "Error communicating with the AI model") diff --git a/tests/test_llm_errors.py b/tests/test_llm_errors.py index 30e365ff..4a5ba139 100644 --- a/tests/test_llm_errors.py +++ b/tests/test_llm_errors.py @@ -40,7 +40,8 @@ def fixture_app(): "error,status", [ (httpx.ReadTimeout(""), 504), - (ModelAPIError(model_name="test-model", message="connection error"), 504), + (httpx.ConnectError("connection refused"), 503), + (ModelAPIError(model_name="test-model", message="connection error"), 503), (ModelHTTPError(status_code=429, model_name="test-model"), 429), (ModelHTTPError(status_code=503, model_name="test-model"), 503), (ModelHTTPError(status_code=400, model_name="test-model"), 502),