fix: defer browser headers until session startup

scrapling/engines/_browsers/_base.py

@@ -20,7 +20,7 @@
 from scrapling.parser import Selector
 from scrapling.engines._browsers._page import PageInfo, PagePool
 from scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfig
-from scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__
+from scrapling.engines._browsers._config_tools import get_default_useragent
 from scrapling.engines.toolbelt.navigation import (
     construct_proxy_dict,
     create_intercept_handler,
@@ -446,9 +446,7 @@ def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
         if config.useragent:
             self._context_options["user_agent"] = config.useragent
         elif not config.useragent and config.headless:
-            self._context_options["user_agent"] = (
-                __default_chrome_useragent__ if config.real_chrome else __default_useragent__
-            )
+            self._context_options["user_agent"] = get_default_useragent("chrome" if config.real_chrome else True)
 
         if not config.cdp_url:
             flags = self._browser_options["args"]

scrapling/engines/_browsers/_config_tools.py

@@ -1,4 +1,13 @@
+from functools import lru_cache
+
 from scrapling.engines.toolbelt.fingerprints import generate_headers
 
-__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
-__default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent")
+
+@lru_cache(2, typed=True)
+def get_default_useragent(browser_mode: bool | str = True) -> str | None:
+    """Generate and cache the default user agent for a browser session.
+
+    :param browser_mode: The browser mode passed to the fingerprint generator.
+    :return: The generated user agent, if available.
+    """
+    return generate_headers(browser_mode=browser_mode).get("User-Agent")

tests/fetchers/test_browser_config.py

@@ -0,0 +1,43 @@
+import importlib
+import sys
+from unittest.mock import call, patch
+
+
+_CONFIG_MODULE = "scrapling.engines._browsers._config_tools"
+
+
+def test_config_tools_import_does_not_generate_headers() -> None:
+    """Importing browser configuration must not depend on header generation."""
+    sys.modules.pop(_CONFIG_MODULE, None)
+    try:
+        with patch(
+            "scrapling.engines.toolbelt.fingerprints.generate_headers",
+            side_effect=ValueError("No headers based on this input can be generated."),
+        ) as generate_headers:
+            importlib.import_module(_CONFIG_MODULE)
+
+        generate_headers.assert_not_called()
+    finally:
+        sys.modules.pop(_CONFIG_MODULE, None)
+
+
+def test_default_useragent_is_generated_on_demand() -> None:
+    """Default user agents should retain their modes and be cached after generation."""
+    config_tools = importlib.import_module(_CONFIG_MODULE)
+    config_tools.get_default_useragent.cache_clear()
+    try:
+        with patch.object(
+            config_tools,
+            "generate_headers",
+            side_effect=[{"User-Agent": "chromium"}, {"User-Agent": "chrome"}],
+        ) as generate_headers:
+            assert config_tools.get_default_useragent(True) == "chromium"
+            assert config_tools.get_default_useragent(True) == "chromium"
+            assert config_tools.get_default_useragent("chrome") == "chrome"
+
+        assert generate_headers.call_args_list == [
+            call(browser_mode=True),
+            call(browser_mode="chrome"),
+        ]
+    finally:
+        config_tools.get_default_useragent.cache_clear()