Skip to content

fix(spiders): preserve cookies for browser-engine responses in cache #379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
D4Vinci merged 2 commits into from
Jul 25, 2026
Merged

fix(spiders): preserve cookies for browser-engine responses in cache #379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ed4fde
fix(spiders): preserve cookies for browser-engine responses in cache …
amitvijapur Jul 20, 2026
a5fed0c
Merge branch 'dev' into fix/cache-drops-browser-cookies
D4Vinci Jul 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions scrapling/spiders/cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from anyio import Path as AsyncPath

from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Any
from scrapling.core._types import Dict, Optional, Any, Tuple
from scrapling.engines.toolbelt.custom import Response


Expand All @@ -28,13 +28,21 @@ async def get(self, fingerprint: bytes) -> Optional[Response]:
async with await anyio.open_file(path, "rb") as f:
data: Dict[str, Any] = orjson.loads(await f.read())

# Browser-engine cookies are cached as a JSON array (see `put`) and come back
# as a `list`; restore the `tuple` shape `Response` expects. Static-engine
# cookies are cached as a JSON object and come back as a `dict` already.
cached_cookies = data["cookies"]
cookies: Tuple[Dict[str, str], ...] | Dict[str, str] = (
tuple(cached_cookies) if isinstance(cached_cookies, list) else cached_cookies
)

return Response(
url=data["url"],
content=b64decode(data["content"]),
status=data["status"],
reason=data["reason"],
encoding=data["encoding"],
cookies=data["cookies"],
cookies=cookies,
headers=data["headers"],
request_headers=data["request_headers"],
method=data["method"],
Expand All @@ -55,7 +63,13 @@ async def put(self, fingerprint: bytes, response: Response, method: str = "GET")
"status": response.status,
"reason": response.reason,
"encoding": response.encoding,
"cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
# Browser-engine responses store cookies as a `tuple` of full cookie
# dicts; static-engine responses store a flat `dict`. Preserve whichever
# shape it is instead of collapsing non-dict cookies to `{}`, which was
# silently dropping every cookie from browser-engine responses.
"cookies": list(response.cookies)
if isinstance(response.cookies, tuple)
else dict(response.cookies),
"headers": dict(response.headers),
"request_headers": dict(response.request_headers),
"method": method,
Expand Down
69 changes: 67 additions & 2 deletions tests/spiders/test_cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,19 @@
from scrapling.core._types import Any, Dict, Set, AsyncGenerator


def _make_response(url: str = "https://example.com", body: bytes = b"<html>hello</html>", status: int = 200) -> Response:
def _make_response(
url: str = "https://example.com",
body: bytes = b"<html>hello</html>",
status: int = 200,
cookies: Any = None,
) -> Response:
return Response(
url=url,
content=body,
status=status,
reason="OK",
encoding="utf-8",
cookies={},
cookies={} if cookies is None else cookies,
headers={"content-type": "text/html"},
request_headers={"user-agent": "test"},
method="GET",
Expand Down Expand Up @@ -49,6 +54,66 @@ async def test_put_get_roundtrip(self):
assert dict(restored.headers) == dict(original.headers)
assert dict(restored.request_headers) == dict(original.request_headers)

@pytest.mark.anyio
async def test_put_get_roundtrip_preserves_browser_engine_cookies(self):
"""Regression test for #376.

Browser engines (Playwright) populate ``Response.cookies`` as a ``tuple`` of
full cookie dicts (see ``engines/toolbelt/convertor.py``), unlike the flat
``dict`` the static engine uses. The cache previously only recognized the
``dict`` shape and silently discarded any other cookies, so a cached
browser-engine response replayed with no cookies at all.
"""
with tempfile.TemporaryDirectory() as tmpdir:
cache = ResponseCacheManager(tmpdir)
fp = b"\x06" * 20
browser_cookies = (
{
"name": "session",
"value": "abc123",
"domain": "example.com",
"path": "/",
"expires": -1,
"httpOnly": True,
"secure": True,
"sameSite": "Lax",
},
{
"name": "csrftoken",
"value": "xyz789",
"domain": "example.com",
"path": "/",
"expires": -1,
"httpOnly": False,
"secure": True,
"sameSite": "Strict",
},
)
original = _make_response(cookies=browser_cookies)

await cache.put(fp, original, "GET")
restored = await cache.get(fp)

assert restored is not None
assert restored.cookies == browser_cookies
assert isinstance(restored.cookies, tuple)

@pytest.mark.anyio
async def test_put_get_roundtrip_preserves_static_engine_cookies(self):
"""Flat-dict cookies (static engine) must still round-trip as a ``dict``."""
with tempfile.TemporaryDirectory() as tmpdir:
cache = ResponseCacheManager(tmpdir)
fp = b"\x07" * 20
static_cookies = {"session": "abc123", "csrftoken": "xyz789"}
original = _make_response(cookies=static_cookies)

await cache.put(fp, original, "GET")
restored = await cache.get(fp)

assert restored is not None
assert restored.cookies == static_cookies
assert isinstance(restored.cookies, dict)

@pytest.mark.anyio
async def test_put_overwrites_existing_entry(self):
"""Re-caching the same fingerprint must replace the stored response.
Expand Down