From a6cc28f981eb1befa19c8ef2a7383d76ab1493f1 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Thu, 8 Oct 2026 16:44:48 +0530 Subject: [PATCH 1/7] Add the remaining pre-commit hooks, adopt sphinx-scrapy (DC-1353) Add blacken-docs, pre-commit-hooks, sphinx-lint, sphinx-scrapy and zizmor, matching the pre-commit configuration of Scrapy. Adopt sphinx-scrapy for the docs and fix the findings of the new hooks. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/publish.yml | 11 ++++++++--- .github/workflows/test.yml | 19 ++++++++++++++----- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ .readthedocs.yml | 12 ++++-------- MANIFEST.in | 2 +- docs/.gitignore | 1 + docs/_ext/__init__.py | 25 ------------------------- docs/conf.py | 22 +--------------------- docs/metadata.rst | 4 ++++ docs/params.rst | 2 ++ docs/requirements.txt | 3 +-- pyproject.toml | 3 +++ tox.ini | 10 ++-------- 13 files changed, 65 insertions(+), 73 deletions(-) create mode 100644 docs/.gitignore delete mode 100644 docs/_ext/__init__.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a0e099..4e80500 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,13 +3,18 @@ on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" +permissions: {} jobs: deploy: runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.13' - name: Install dependencies @@ -17,6 +22,6 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.10.2 + uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2 # zizmor: ignore[use-trusted-publishing] with: password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4dc2b11..65485e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,9 +3,12 @@ on: pull_request: push: branches: [ main ] +permissions: {} jobs: test: runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: @@ -18,9 +21,11 @@ jobs: - python-version: "3.12" - python-version: "3.13" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -31,20 +36,24 @@ jobs: run: | tox -e ${{ matrix.toxenv || 'py' }} - name: coverage - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: token: ${{ secrets.CODECOV_TOKEN }} check: runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: python-version: ['3.13'] # Keep in sync with .readthedocs.yml tox-job: ["pre-commit", "mypy", "types", "docs", "twinecheck"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c92c4d..aae73b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,3 +5,27 @@ repos: - id: ruff args: [ --fix ] - id: ruff-format +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.20.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==26.5.1 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.13.0 + hooks: + - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/.readthedocs.yml b/.readthedocs.yml index 5eff096..afcc1db 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,12 +1,8 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py build: os: ubuntu-24.04 tools: - python: "3.13" # Keep in sync with .github/workflows/tests.yml -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.13" + commands: + - pip install tox + - tox -e docs -- $READTHEDOCS_OUTPUT/html diff --git a/MANIFEST.in b/MANIFEST.in index f424d6d..37f630a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include tests * \ No newline at end of file +recursive-include tests * diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..ba65b13 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/docs/_ext/__init__.py b/docs/_ext/__init__.py deleted file mode 100644 index 5a3839e..0000000 --- a/docs/_ext/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -def setup(app): - # https://stackoverflow.com/a/13663325 - # - # Scrapy’s - # https://github.com/scrapy/scrapy/blob/dba37674e6eaa6c2030c8eb35ebf8127cd488062/docs/_ext/scrapydocs.py#L90C16-L110C6 - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) diff --git a/docs/conf.py b/docs/conf.py index df09da8..290b1d0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,30 +1,10 @@ -import sys -from pathlib import Path - project = "scrapy-spider-metadata" copyright = "2023, Zyte Group Ltd" author = "Zyte Group Ltd" release = "0.0.0" -sys.path.insert(0, str(Path(__file__).parent.absolute())) # _ext extensions = [ - "_ext", - "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", + "sphinx_scrapy", ] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -html_theme = "sphinx_rtd_theme" - -intersphinx_mapping = { - "python": ( - "https://docs.python.org/3", - None, - ), - "scrapy": ( - "https://docs.scrapy.org/en/latest", - None, - ), -} diff --git a/docs/metadata.rst b/docs/metadata.rst index 3152182..cae9d49 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -17,6 +17,7 @@ attribute named ``metadata``: from scrapy import Spider + class MySpider(Spider): name = "my_spider" metadata = { @@ -32,18 +33,21 @@ add or change some values in a subclass: from scrapy import Spider + class BaseSpider(Spider): metadata = { "description": "Base spider.", "category": "Base spiders", } + class BaseNewsSpider(BaseSpider): metadata = { **BaseSpider.metadata, "description": "Base news spider.", } + class CNNSpider(BaseNewsSpider): metadata = { **BaseNewsSpider.metadata, diff --git a/docs/params.rst b/docs/params.rst index c01b271..70f74ea 100644 --- a/docs/params.rst +++ b/docs/params.rst @@ -37,9 +37,11 @@ as its parameter: from scrapy import Request, Spider from scrapy_spider_metadata import Args + class MyParams(BaseModel): pages: int + class MySpider(Args[MyParams], Spider): name = "my_spider" diff --git a/docs/requirements.txt b/docs/requirements.txt index 7633954..fcc27ae 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1 @@ -Sphinx==7.2.4 # https://github.com/sphinx-doc/sphinx/issues/11662#issuecomment-1699989128 -sphinx-rtd-theme==1.3.0 +sphinx-scrapy==0.13.0 diff --git a/pyproject.toml b/pyproject.toml index 582288b..95e4534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,3 +189,6 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel"] [tool.ruff.lint.pydocstyle] convention = "pep257" + +[tool.sphinx-scrapy] +python-version = "3.13" diff --git a/tox.ini b/tox.ini index 1a1b718..9ad62c6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,6 @@ [tox] +requires = + sphinx-scrapy[tox]==0.13.0 envlist = py,pre-commit,mypy,types,docs,twinecheck [pytest] @@ -59,14 +61,6 @@ deps = commands = pytest {posargs:tests_typing} -[testenv:docs] -basepython = python3 -changedir = docs -deps = - -rdocs/requirements.txt -commands = - sphinx-build -W -b html . {envtmpdir}/html - [testenv:twinecheck] basepython = python3 deps = From 17ba3f286cb657d37f1afae5f63c9393497f8c62 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Thu, 8 Oct 2026 17:31:33 +0530 Subject: [PATCH 2/7] Run tox on Python 3.13 in CI, as sphinx-scrapy requires 3.10+ Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65485e6..e50931b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,10 +16,15 @@ jobs: - python-version: "3.9" toxenv: "min" - python-version: "3.9" + toxenv: "py39" - python-version: "3.10" + toxenv: "py310" - python-version: "3.11" + toxenv: "py311" - python-version: "3.12" + toxenv: "py312" - python-version: "3.13" + toxenv: "py313" steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: @@ -27,14 +32,18 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: ${{ matrix.python-version }} + # The first version runs tox itself (sphinx-scrapy requires 3.10+), + # the second one is used by the test environment. + python-version: | + 3.13 + ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install tox - name: tox run: | - tox -e ${{ matrix.toxenv || 'py' }} + tox -e ${{ matrix.toxenv }} - name: coverage uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: From d3ed932acac47102199b08ed8b5fda32865f0d98 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Mon, 12 Oct 2026 18:17:21 +0530 Subject: [PATCH 3/7] Address review: dependabot, docs requirements.in, workflow permissions - Add the Dependabot config used in Scrapy to upgrade GitHub Actions. - Add docs/requirements.in and compile docs/requirements.txt from it with uv. - Set permissions (contents: read) at the workflow level instead of per job. --- .github/dependabot.yml | 12 ++++++ .github/workflows/publish.yml | 5 +-- .github/workflows/test.yml | 7 +--- docs/requirements.in | 1 + docs/requirements.txt | 74 +++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 docs/requirements.in diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..623d48c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4e80500..fbc8a17 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,12 +3,11 @@ on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" -permissions: {} +permissions: + contents: read jobs: deploy: runs-on: ubuntu-latest - permissions: - contents: read steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e50931b..80576ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,12 +3,11 @@ on: pull_request: push: branches: [ main ] -permissions: {} +permissions: + contents: read jobs: test: runs-on: ubuntu-latest - permissions: - contents: read strategy: fail-fast: false matrix: @@ -50,8 +49,6 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} check: runs-on: ubuntu-latest - permissions: - contents: read strategy: fail-fast: false matrix: diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 0000000..fcc27ae --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1 @@ +sphinx-scrapy==0.13.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index fcc27ae..7d91ca6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,75 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile docs/requirements.in -o docs/requirements.txt --python-version 3.13 +alabaster==1.0.0 + # via sphinx +babel==2.18.0 + # via sphinx +certifi==2026.7.22 + # via requests +charset-normalizer==3.5.2 + # via requests +docutils==0.22.4 + # via + # sphinx + # sphinx-llm-friendly + # sphinx-scrapy +idna==3.20 + # via requests +imagesize==2.0.1 + # via sphinx +jinja2==3.1.6 + # via sphinx +markupsafe==3.0.4 + # via jinja2 +packaging==26.3 + # via + # sphinx + # sphinx-scrapy +pygments==2.21.0 + # via sphinx +requests==2.34.2 + # via + # sphinx + # sphinxcontrib-youtube +roman-numerals==4.1.0 + # via sphinx +snowballstemmer==3.1.1 + # via sphinx +sphinx==9.1.0 + # via + # sphinx-copybutton + # sphinx-design + # sphinx-last-updated-by-git + # sphinx-llm-friendly + # sphinx-scrapy + # sphinxcontrib-youtube +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-design==0.7.0 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llm-friendly==0.7.0 + # via sphinx-scrapy sphinx-scrapy==0.13.0 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +sphinxcontrib-youtube==1.5.0 + # via sphinx-scrapy +tabulate==0.10.0 + # via sphinx-llm-friendly +urllib3==2.8.0 + # via requests From d29d9ab1c4fcda82633f4519dd13d1ee5dbde9c9 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Wed, 14 Oct 2026 15:48:58 +0530 Subject: [PATCH 4/7] Drop Python 3.9, add 3.14 and 3.15, bump tool versions - Require Python 3.10+; CI and classifiers now cover 3.10-3.15. This also fixes the 3.9 CI jobs, as sphinx-scrapy requires 3.10+. - Replace the nonexistent "min" tox env in CI with min-1x and min-2x, and pin scrapy==2.6.0 and w3lib<2 there, since Scrapy 1.0.0 cannot be imported on Python 3.10. - Bump ruff (hook id is now ruff-check), black, mypy, pytest, twine and build, and ignore PLC0415 in tests/test_params.py. - Add the "Typing :: Typed" classifier. --- .github/workflows/test.yml | 22 ++++++++-------------- .pre-commit-config.yaml | 6 +++--- .readthedocs.yml | 2 +- README.rst | 2 +- pyproject.toml | 11 +++++++---- tests/test_params.py | 8 ++++---- tox.ini | 18 ++++++++++-------- 7 files changed, 34 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80576ce..54e3c48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,18 +12,16 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - toxenv: "min" - - python-version: "3.9" - toxenv: "py39" - python-version: "3.10" - toxenv: "py310" + toxenv: "min-1x" + - python-version: "3.10" + toxenv: "min-2x" + - python-version: "3.10" - python-version: "3.11" - toxenv: "py311" - python-version: "3.12" - toxenv: "py312" - python-version: "3.13" - toxenv: "py313" + - python-version: "3.14" + - python-version: "3.15" steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: @@ -31,18 +29,14 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - # The first version runs tox itself (sphinx-scrapy requires 3.10+), - # the second one is used by the test environment. - python-version: | - 3.13 - ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install tox - name: tox run: | - tox -e ${{ matrix.toxenv }} + tox -e ${{ matrix.toxenv || 'py' }} - name: coverage uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aae73b5..525a19d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.2 + rev: v0.17.0 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs @@ -10,7 +10,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==26.5.1 + - black==26.10.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/.readthedocs.yml b/.readthedocs.yml index afcc1db..fa9aa07 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,7 +2,7 @@ version: 2 build: os: ubuntu-24.04 tools: - python: "3.13" + python: "3.14" commands: - pip install tox - tox -e docs -- $READTHEDOCS_OUTPUT/html diff --git a/README.rst b/README.rst index 6b397b2..c75c2c9 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ scrapy-spider-metadata .. description starts -``scrapy-spider-metadata`` is a Python 3.9+ library of utilities to extend +``scrapy-spider-metadata`` is a Python 3.10+ library of utilities to extend Scrapy spiders with usable metadata. In particular, it provides a nice way for Scrapy spiders to define, validate, diff --git a/pyproject.toml b/pyproject.toml index 95e4534..e56dc32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,15 +15,17 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", + "Typing :: Typed", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "packaging>=14.1", "pydantic>=1.9", @@ -185,10 +187,11 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel"] [tool.ruff.lint.per-file-ignores] # `from __future__ import annotations` breaks Pydantic 1.x -"tests/test_params.py" = ["FA100"] +# Pydantic version-specific imports are done inside the tests +"tests/test_params.py" = ["FA100", "PLC0415"] [tool.ruff.lint.pydocstyle] convention = "pep257" [tool.sphinx-scrapy] -python-version = "3.13" +python-version = "3.14" diff --git a/tests/test_params.py b/tests/test_params.py index 8b1ca01..38b9319 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -1,5 +1,5 @@ from enum import Enum, IntEnum -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast import pytest from packaging import version @@ -581,8 +581,8 @@ class Params(BaseModel): default=0, ) int_with_default: int = 1 - int_optional: Optional[int] = None - int_optional_without_default: Optional[int] + int_optional: int | None = None + int_optional_without_default: int | None number_without_default: float phone: str = Field( min_length=3, @@ -628,7 +628,7 @@ class Params(BaseModel): # https://github.com/pydantic/pydantic/issues/3753#issuecomment-1060850457 default=..., ) - dessert_optional: Optional[DessertEnum] = None + dessert_optional: DessertEnum | None = None dessert_required: DessertEnum class ParamSpider(Args[Params], Spider): diff --git a/tox.ini b/tox.ini index 9ad62c6..ceac010 100644 --- a/tox.ini +++ b/tox.ini @@ -27,20 +27,22 @@ deps = pydantic<2 [testenv:min-1x] -base_python = python3.9 +base_python = python3.10 deps = {[testenv]deps} packaging==14.1 pydantic==1.9.0 - scrapy==1.0.0 + scrapy==2.6.0 + w3lib<2 [testenv:min-2x] -base_python = python3.9 +base_python = python3.10 deps = {[testenv]deps} packaging==14.1 pydantic==2.0 - scrapy==1.0.0 + scrapy==2.6.0 + w3lib<2 [testenv:pre-commit] deps = @@ -49,8 +51,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:mypy] deps = - mypy==1.15.0 - pytest==8.3.3 + mypy==2.4.0 + pytest==9.1.1 commands = mypy --strict {posargs:scrapy_spider_metadata tests} [testenv:types] @@ -64,8 +66,8 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==6.1.0 - build==1.2.2.post1 + twine==7.0.0 + build==1.6.1 commands = python -m build --sdist twine check dist/* From 568c9f85e2b2f34782f97d181ae7ca33089b2e50 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Wed, 14 Oct 2026 19:05:01 +0530 Subject: [PATCH 5/7] Switch PyPI publishing to trusted publishing Grant the deploy job id-token: write and drop the PYPI_TOKEN secret and the zizmor ignore for use-trusted-publishing. --- .github/workflows/publish.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fbc8a17..b51e3c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,6 +8,9 @@ permissions: jobs: deploy: runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # required for PyPI trusted publishing steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: @@ -21,6 +24,5 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2 # zizmor: ignore[use-trusted-publishing] - with: - password: ${{ secrets.PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2 + From 7359fdf245afbc86b6dca097509957766f1cec72 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Thu, 15 Oct 2026 11:47:22 +0530 Subject: [PATCH 6/7] Fix CI: docs on Python 3.14, pytest-mypy-testing 0.2.0, trailing newline - Run the check jobs on 3.14, as the docs env requires it (matches .readthedocs.yml). - Bump pytest-mypy-testing to 0.2.0; 0.1.3 requires pytest<9. - Remove the extra blank line at the end of publish.yml. --- .github/workflows/publish.yml | 1 - .github/workflows/test.yml | 2 +- tox.ini | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b51e3c7..64badb7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,4 +25,3 @@ jobs: python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2 - diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54e3c48..112a8fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.13'] # Keep in sync with .readthedocs.yml + python-version: ['3.14'] # Keep in sync with .readthedocs.yml tox-job: ["pre-commit", "mypy", "types", "docs", "twinecheck"] steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 diff --git a/tox.ini b/tox.ini index ceac010..893f383 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ commands = mypy --strict {posargs:scrapy_spider_metadata tests} deps = {[testenv]deps} {[testenv:mypy]deps} - pytest-mypy-testing==0.1.3 + pytest-mypy-testing==0.2.0 commands = pytest {posargs:tests_typing} From 702e8786f417dd04bf300b3ed477abfa61b25130 Mon Sep 17 00:00:00 2001 From: Kishan Mehta Date: Thu, 15 Oct 2026 13:07:27 +0530 Subject: [PATCH 7/7] Add actionlint hook, compile docs requirements for Python 3.14 --- .pre-commit-config.yaml | 4 ++++ docs/requirements.txt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 525a19d..1f3090b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,10 @@ repos: rev: 0.13.0 hooks: - id: sphinx-scrapy +- repo: https://github.com/rhysd/actionlint + rev: v1.7.12 + hooks: + - id: actionlint - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: v1.30.1 hooks: diff --git a/docs/requirements.txt b/docs/requirements.txt index 7d91ca6..2a1ea52 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile docs/requirements.in -o docs/requirements.txt --python-version 3.13 +# uv pip compile docs/requirements.in -o docs/requirements.txt --python-version 3.14 alabaster==1.0.0 # via sphinx babel==2.18.0