From 3ed4fde93a2018e673c2928d60cbebf1f6c1800d Mon Sep 17 00:00:00 2001 From: Amit Vijapur Date: Tue, 21 Jul 2026 00:38:48 +0800 Subject: [PATCH] fix(spiders): preserve cookies for browser-engine responses in cache (#376) ResponseCacheManager.put() only recognized the flat `dict` shape that static-engine responses use for `Response.cookies`. Browser engines (Playwright) populate cookies as a `tuple` of full cookie dicts, so the isinstance(dict) guard fell through to the `else {}` branch and silently discarded every cookie before it ever reached the cache file. Replaying a cached browser-engine response then rebuilt it with cookies={}. Preserve whichever shape the cookies are in instead of collapsing anything non-dict to {}: serialize a tuple as a JSON array and a dict as a JSON object, then restore the tuple shape on read since JSON arrays deserialize back as `list`. Add regression tests covering both the browser-engine (tuple) and static-engine (dict) cookie round-trip through put()/get(). --- scrapling/spiders/cache.py | 20 +++++++++-- tests/spiders/test_cache.py | 69 +++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.py index 0305aeff6..1aeb154da 100644 --- a/scrapling/spiders/cache.py +++ b/scrapling/spiders/cache.py @@ -6,7 +6,7 @@ from anyio import Path as AsyncPath from scrapling.core.utils import log -from scrapling.core._types import Dict, Optional, Any +from scrapling.core._types import Dict, Optional, Any, Tuple from scrapling.engines.toolbelt.custom import Response @@ -28,13 +28,21 @@ async def get(self, fingerprint: bytes) -> Optional[Response]: async with await anyio.open_file(path, "rb") as f: data: Dict[str, Any] = orjson.loads(await f.read()) + # Browser-engine cookies are cached as a JSON array (see `put`) and come back + # as a `list`; restore the `tuple` shape `Response` expects. Static-engine + # cookies are cached as a JSON object and come back as a `dict` already. + cached_cookies = data["cookies"] + cookies: Tuple[Dict[str, str], ...] | Dict[str, str] = ( + tuple(cached_cookies) if isinstance(cached_cookies, list) else cached_cookies + ) + return Response( url=data["url"], content=b64decode(data["content"]), status=data["status"], reason=data["reason"], encoding=data["encoding"], - cookies=data["cookies"], + cookies=cookies, headers=data["headers"], request_headers=data["request_headers"], method=data["method"], @@ -55,7 +63,13 @@ async def put(self, fingerprint: bytes, response: Response, method: str = "GET") "status": response.status, "reason": response.reason, "encoding": response.encoding, - "cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {}, + # Browser-engine responses store cookies as a `tuple` of full cookie + # dicts; static-engine responses store a flat `dict`. Preserve whichever + # shape it is instead of collapsing non-dict cookies to `{}`, which was + # silently dropping every cookie from browser-engine responses. + "cookies": list(response.cookies) + if isinstance(response.cookies, tuple) + else dict(response.cookies), "headers": dict(response.headers), "request_headers": dict(response.request_headers), "method": method, diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py index eb1b04b66..3b990ed1f 100644 --- a/tests/spiders/test_cache.py +++ b/tests/spiders/test_cache.py @@ -14,14 +14,19 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator -def _make_response(url: str = "https://example.com", body: bytes = b"hello", status: int = 200) -> Response: +def _make_response( + url: str = "https://example.com", + body: bytes = b"hello", + status: int = 200, + cookies: Any = None, +) -> Response: return Response( url=url, content=body, status=status, reason="OK", encoding="utf-8", - cookies={}, + cookies={} if cookies is None else cookies, headers={"content-type": "text/html"}, request_headers={"user-agent": "test"}, method="GET", @@ -49,6 +54,66 @@ async def test_put_get_roundtrip(self): assert dict(restored.headers) == dict(original.headers) assert dict(restored.request_headers) == dict(original.request_headers) + @pytest.mark.anyio + async def test_put_get_roundtrip_preserves_browser_engine_cookies(self): + """Regression test for #376. + + Browser engines (Playwright) populate ``Response.cookies`` as a ``tuple`` of + full cookie dicts (see ``engines/toolbelt/convertor.py``), unlike the flat + ``dict`` the static engine uses. The cache previously only recognized the + ``dict`` shape and silently discarded any other cookies, so a cached + browser-engine response replayed with no cookies at all. + """ + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp = b"\x06" * 20 + browser_cookies = ( + { + "name": "session", + "value": "abc123", + "domain": "example.com", + "path": "/", + "expires": -1, + "httpOnly": True, + "secure": True, + "sameSite": "Lax", + }, + { + "name": "csrftoken", + "value": "xyz789", + "domain": "example.com", + "path": "/", + "expires": -1, + "httpOnly": False, + "secure": True, + "sameSite": "Strict", + }, + ) + original = _make_response(cookies=browser_cookies) + + await cache.put(fp, original, "GET") + restored = await cache.get(fp) + + assert restored is not None + assert restored.cookies == browser_cookies + assert isinstance(restored.cookies, tuple) + + @pytest.mark.anyio + async def test_put_get_roundtrip_preserves_static_engine_cookies(self): + """Flat-dict cookies (static engine) must still round-trip as a ``dict``.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache = ResponseCacheManager(tmpdir) + fp = b"\x07" * 20 + static_cookies = {"session": "abc123", "csrftoken": "xyz789"} + original = _make_response(cookies=static_cookies) + + await cache.put(fp, original, "GET") + restored = await cache.get(fp) + + assert restored is not None + assert restored.cookies == static_cookies + assert isinstance(restored.cookies, dict) + @pytest.mark.anyio async def test_put_overwrites_existing_entry(self): """Re-caching the same fingerprint must replace the stored response.