diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py index c9d390cd..abb807a2 100644 --- a/scrapling/engines/_browsers/_base.py +++ b/scrapling/engines/_browsers/_base.py @@ -20,7 +20,7 @@ from scrapling.parser import Selector from scrapling.engines._browsers._page import PageInfo, PagePool from scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfig -from scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__ +from scrapling.engines._browsers._config_tools import get_default_useragent from scrapling.engines.toolbelt.navigation import ( construct_proxy_dict, create_intercept_handler, @@ -446,9 +446,7 @@ def __generate_options__(self, extra_flags: Tuple | None = None) -> None: if config.useragent: self._context_options["user_agent"] = config.useragent elif not config.useragent and config.headless: - self._context_options["user_agent"] = ( - __default_chrome_useragent__ if config.real_chrome else __default_useragent__ - ) + self._context_options["user_agent"] = get_default_useragent("chrome" if config.real_chrome else True) if not config.cdp_url: flags = self._browser_options["args"] diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py index 5b9ee2e5..f1a8a36d 100644 --- a/scrapling/engines/_browsers/_config_tools.py +++ b/scrapling/engines/_browsers/_config_tools.py @@ -1,4 +1,13 @@ +from functools import lru_cache + from scrapling.engines.toolbelt.fingerprints import generate_headers -__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent") -__default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent") + +@lru_cache(2, typed=True) +def get_default_useragent(browser_mode: bool | str = True) -> str | None: + """Generate and cache the default user agent for a browser session. + + :param browser_mode: The browser mode passed to the fingerprint generator. + :return: The generated user agent, if available. + """ + return generate_headers(browser_mode=browser_mode).get("User-Agent") diff --git a/tests/fetchers/test_browser_config.py b/tests/fetchers/test_browser_config.py new file mode 100644 index 00000000..41a2cc4e --- /dev/null +++ b/tests/fetchers/test_browser_config.py @@ -0,0 +1,43 @@ +import importlib +import sys +from unittest.mock import call, patch + + +_CONFIG_MODULE = "scrapling.engines._browsers._config_tools" + + +def test_config_tools_import_does_not_generate_headers() -> None: + """Importing browser configuration must not depend on header generation.""" + sys.modules.pop(_CONFIG_MODULE, None) + try: + with patch( + "scrapling.engines.toolbelt.fingerprints.generate_headers", + side_effect=ValueError("No headers based on this input can be generated."), + ) as generate_headers: + importlib.import_module(_CONFIG_MODULE) + + generate_headers.assert_not_called() + finally: + sys.modules.pop(_CONFIG_MODULE, None) + + +def test_default_useragent_is_generated_on_demand() -> None: + """Default user agents should retain their modes and be cached after generation.""" + config_tools = importlib.import_module(_CONFIG_MODULE) + config_tools.get_default_useragent.cache_clear() + try: + with patch.object( + config_tools, + "generate_headers", + side_effect=[{"User-Agent": "chromium"}, {"User-Agent": "chrome"}], + ) as generate_headers: + assert config_tools.get_default_useragent(True) == "chromium" + assert config_tools.get_default_useragent(True) == "chromium" + assert config_tools.get_default_useragent("chrome") == "chrome" + + assert generate_headers.call_args_list == [ + call(browser_mode=True), + call(browser_mode="chrome"), + ] + finally: + config_tools.get_default_useragent.cache_clear()