From 52aaed9d7be375e30eb5daabce3ee1a3350dca47 Mon Sep 17 00:00:00 2001 From: Vedaant Singh Date: Sun, 12 Apr 2026 13:17:50 +0100 Subject: [PATCH] Fix ruff lint errors and Python 3.9 compatibility --- src/pyscrappy/__init__.py | 4 ++-- src/pyscrappy/core/config.py | 15 ++++++++++----- src/pyscrappy/core/http.py | 4 +++- src/pyscrappy/generic/extractors.py | 1 - src/pyscrappy/generic/pagination.py | 1 - src/pyscrappy/scrapers/alibaba.py | 1 - src/pyscrappy/scrapers/flipkart.py | 1 - src/pyscrappy/scrapers/image_search.py | 7 ++----- src/pyscrappy/scrapers/instagram.py | 6 ++++-- src/pyscrappy/scrapers/news.py | 1 - src/pyscrappy/scrapers/soundcloud.py | 1 - src/pyscrappy/scrapers/spotify.py | 1 - src/pyscrappy/scrapers/swiggy.py | 6 ++++-- src/pyscrappy/scrapers/twitter.py | 7 +++++-- src/pyscrappy/scrapers/wikipedia.py | 2 +- src/pyscrappy/scrapers/youtube.py | 3 +-- src/pyscrappy/scrapers/zomato.py | 11 ++++++++--- 17 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/pyscrappy/__init__.py b/src/pyscrappy/__init__.py index 7381701..af61585 100644 --- a/src/pyscrappy/__init__.py +++ b/src/pyscrappy/__init__.py @@ -99,10 +99,10 @@ def scrape( url: str, - selectors: dict[str, str] | None = None, + selectors: "dict[str, str] | None" = None, max_pages: int = 1, render_js: bool = False, - config: ScraperConfig | None = None, + config: "ScraperConfig | None" = None, ) -> ScrapeResult: """One-liner convenience function to scrape any URL. diff --git a/src/pyscrappy/core/config.py b/src/pyscrappy/core/config.py index 92a09c4..14454ab 100644 --- a/src/pyscrappy/core/config.py +++ b/src/pyscrappy/core/config.py @@ -6,11 +6,16 @@ from typing import Literal _DEFAULT_USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0)" + " Gecko/20100101 Firefox/125.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15" + " (KHTML, like Gecko) Version/17.4 Safari/605.1.15", ] diff --git a/src/pyscrappy/core/http.py b/src/pyscrappy/core/http.py index bec6f15..73b5ad0 100644 --- a/src/pyscrappy/core/http.py +++ b/src/pyscrappy/core/http.py @@ -88,7 +88,9 @@ def get(self, url: str, **kwargs: Any) -> httpx.Response: time.sleep(delay) continue - raise NetworkError(f"Failed to fetch {url} after {self.config.max_retries} attempts") from last_exc + raise NetworkError( + f"Failed to fetch {url} after {self.config.max_retries} attempts" + ) from last_exc def get_raw(self, url: str, **kwargs: Any) -> httpx.Response: """Like :meth:`get` but does NOT raise on non-2xx status codes. diff --git a/src/pyscrappy/generic/extractors.py b/src/pyscrappy/generic/extractors.py index 5ec3f3b..b93ef6d 100644 --- a/src/pyscrappy/generic/extractors.py +++ b/src/pyscrappy/generic/extractors.py @@ -88,7 +88,6 @@ def extract(self, soup: BeautifulSoup) -> dict[str, Any]: } def _remove_noise(self, soup: BeautifulSoup) -> BeautifulSoup: - from copy import copy clean = BeautifulSoup(str(soup), "lxml") for tag in clean.find_all(self._NOISE_TAGS): diff --git a/src/pyscrappy/generic/pagination.py b/src/pyscrappy/generic/pagination.py index 93e9997..e467680 100644 --- a/src/pyscrappy/generic/pagination.py +++ b/src/pyscrappy/generic/pagination.py @@ -7,7 +7,6 @@ from bs4 import BeautifulSoup, Tag - _NEXT_PATTERNS = re.compile( r"(^next$|^>$|^>>$|^›$|^»$|next\s*page|next\s*›|load\s*more)", re.IGNORECASE, diff --git a/src/pyscrappy/scrapers/alibaba.py b/src/pyscrappy/scrapers/alibaba.py index b2a14fd..c4e44bb 100644 --- a/src/pyscrappy/scrapers/alibaba.py +++ b/src/pyscrappy/scrapers/alibaba.py @@ -8,7 +8,6 @@ from bs4 import Tag from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult diff --git a/src/pyscrappy/scrapers/flipkart.py b/src/pyscrappy/scrapers/flipkart.py index d31e527..08a8241 100644 --- a/src/pyscrappy/scrapers/flipkart.py +++ b/src/pyscrappy/scrapers/flipkart.py @@ -8,7 +8,6 @@ from bs4 import Tag from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult diff --git a/src/pyscrappy/scrapers/image_search.py b/src/pyscrappy/scrapers/image_search.py index c1a2db0..b2209b1 100644 --- a/src/pyscrappy/scrapers/image_search.py +++ b/src/pyscrappy/scrapers/image_search.py @@ -4,15 +4,12 @@ import logging import os -import re from typing import Any -from urllib.parse import quote_plus, urljoin - -from bs4 import Tag +from urllib.parse import quote_plus from pyscrappy.core.base import BaseScraper from pyscrappy.core.config import ScraperConfig -from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult +from pyscrappy.core.models import ScrapeMetadata, ScrapeResult logger = logging.getLogger("pyscrappy.image_search") diff --git a/src/pyscrappy/scrapers/instagram.py b/src/pyscrappy/scrapers/instagram.py index e13ef62..bb5d333 100644 --- a/src/pyscrappy/scrapers/instagram.py +++ b/src/pyscrappy/scrapers/instagram.py @@ -15,7 +15,6 @@ from typing import Any from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult @@ -218,7 +217,10 @@ def _parse_hashtag_json( node = edge.get("node", {}) caption_edges = node.get("edge_media_to_caption", {}).get("edges", []) posts.append({ - "caption": caption_edges[0].get("node", {}).get("text", "") if caption_edges else "", + "caption": ( + caption_edges[0].get("node", {}).get("text", "") + if caption_edges else "" + ), "likes": node.get("edge_liked_by", {}).get("count"), "comments": node.get("edge_media_to_comment", {}).get("count"), "url": f"https://www.instagram.com/p/{node.get('shortcode', '')}/", diff --git a/src/pyscrappy/scrapers/news.py b/src/pyscrappy/scrapers/news.py index 1da1b4b..3b25f80 100644 --- a/src/pyscrappy/scrapers/news.py +++ b/src/pyscrappy/scrapers/news.py @@ -156,7 +156,6 @@ def _parse_feed_xml(self, xml_text: str, max_articles: int) -> list[dict[str, An return articles # Namespace handling for Atom - ns = {"atom": "http://www.w3.org/2005/Atom"} # RSS 2.0: for item in root.iter("item"): diff --git a/src/pyscrappy/scrapers/soundcloud.py b/src/pyscrappy/scrapers/soundcloud.py index 1f29523..5659228 100644 --- a/src/pyscrappy/scrapers/soundcloud.py +++ b/src/pyscrappy/scrapers/soundcloud.py @@ -8,7 +8,6 @@ from urllib.parse import quote_plus from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult diff --git a/src/pyscrappy/scrapers/spotify.py b/src/pyscrappy/scrapers/spotify.py index 09ac616..9d4f9f8 100644 --- a/src/pyscrappy/scrapers/spotify.py +++ b/src/pyscrappy/scrapers/spotify.py @@ -15,7 +15,6 @@ from urllib.parse import quote_plus from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult diff --git a/src/pyscrappy/scrapers/swiggy.py b/src/pyscrappy/scrapers/swiggy.py index 75120e4..681b7a0 100644 --- a/src/pyscrappy/scrapers/swiggy.py +++ b/src/pyscrappy/scrapers/swiggy.py @@ -8,7 +8,6 @@ from bs4 import Tag from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult @@ -68,7 +67,10 @@ def scrape( # type: ignore[override] if not restaurants: errors.append(ScrapeError( url=url, - message="No restaurants extracted. Swiggy requires JS rendering — use render_js=True.", + message=( + "No restaurants extracted. Swiggy requires JS rendering" + " — use render_js=True." + ), )) return ScrapeResult( diff --git a/src/pyscrappy/scrapers/twitter.py b/src/pyscrappy/scrapers/twitter.py index fd9570e..54eb8ba 100644 --- a/src/pyscrappy/scrapers/twitter.py +++ b/src/pyscrappy/scrapers/twitter.py @@ -14,7 +14,6 @@ from typing import Any from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult @@ -83,7 +82,11 @@ def _scrape_search( if not tweets: errors.append(ScrapeError( url=url, - message="No tweets extracted. Twitter/X likely requires authentication. Use render_js=True with a logged-in browser session.", + message=( + "No tweets extracted. Twitter/X likely requires" + " authentication. Use render_js=True with a" + " logged-in browser session." + ), )) return ScrapeResult( diff --git a/src/pyscrappy/scrapers/wikipedia.py b/src/pyscrappy/scrapers/wikipedia.py index 7f7faf3..2ce7072 100644 --- a/src/pyscrappy/scrapers/wikipedia.py +++ b/src/pyscrappy/scrapers/wikipedia.py @@ -6,7 +6,7 @@ from typing import Any from urllib.parse import quote -from bs4 import BeautifulSoup, Tag +from bs4 import Tag from pyscrappy.core.base import BaseScraper from pyscrappy.core.config import ScraperConfig diff --git a/src/pyscrappy/scrapers/youtube.py b/src/pyscrappy/scrapers/youtube.py index bff90e2..00e05ef 100644 --- a/src/pyscrappy/scrapers/youtube.py +++ b/src/pyscrappy/scrapers/youtube.py @@ -8,8 +8,7 @@ from urllib.parse import quote_plus from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig -from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult +from pyscrappy.core.models import ScrapeMetadata, ScrapeResult class YouTubeScraper(BaseScraper): diff --git a/src/pyscrappy/scrapers/zomato.py b/src/pyscrappy/scrapers/zomato.py index aa6768f..a924ff4 100644 --- a/src/pyscrappy/scrapers/zomato.py +++ b/src/pyscrappy/scrapers/zomato.py @@ -10,7 +10,6 @@ from bs4 import Tag from pyscrappy.core.base import BaseScraper -from pyscrappy.core.config import ScraperConfig from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult @@ -72,7 +71,10 @@ def scrape( # type: ignore[override] if not restaurants: errors.append(ScrapeError( url=url, - message="No restaurants extracted. Zomato requires JS rendering — use render_js=True.", + message=( + "No restaurants extracted. Zomato requires JS rendering" + " — use render_js=True." + ), )) return ScrapeResult( @@ -198,7 +200,10 @@ def _find_restaurants( "rating": info.get("rating", {}).get("aggregate_rating") if isinstance(info.get("rating"), dict) else info.get("rating"), - "price": info.get("average_cost_for_two") or info.get("cfo", {}).get("text", ""), + "price": ( + info.get("average_cost_for_two") + or info.get("cfo", {}).get("text", "") + ), "address": info.get("location", {}).get("address", "") if isinstance(info.get("location"), dict) else "", "url": f"https://www.zomato.com/restaurant/{info.get('id', '')}",