diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..623d48c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a0e099..64badb7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,13 +3,20 @@ on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" +permissions: + contents: read jobs: deploy: runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # required for PyPI trusted publishing steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.13' - name: Install dependencies @@ -17,6 +24,4 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.10.2 - with: - password: ${{ secrets.PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4dc2b11..112a8fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,8 @@ on: pull_request: push: branches: [ main ] +permissions: + contents: read jobs: test: runs-on: ubuntu-latest @@ -10,17 +12,22 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - toxenv: "min" - - python-version: "3.9" + - python-version: "3.10" + toxenv: "min-1x" + - python-version: "3.10" + toxenv: "min-2x" - python-version: "3.10" - python-version: "3.11" - python-version: "3.12" - python-version: "3.13" + - python-version: "3.14" + - python-version: "3.15" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -31,7 +38,7 @@ jobs: run: | tox -e ${{ matrix.toxenv || 'py' }} - name: coverage - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: token: ${{ secrets.CODECOV_TOKEN }} check: @@ -39,12 +46,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.13'] # Keep in sync with .readthedocs.yml + python-version: ['3.14'] # Keep in sync with .readthedocs.yml tox-job: ["pre-commit", "mypy", "types", "docs", "twinecheck"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c92c4d..1f3090b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,35 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.2 + rev: v0.17.0 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] - id: ruff-format +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.20.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==26.10.1 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.13.0 + hooks: + - id: sphinx-scrapy +- repo: https://github.com/rhysd/actionlint + rev: v1.7.12 + hooks: + - id: actionlint +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/.readthedocs.yml b/.readthedocs.yml index 5eff096..fa9aa07 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,12 +1,8 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py build: os: ubuntu-24.04 tools: - python: "3.13" # Keep in sync with .github/workflows/tests.yml -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs -- $READTHEDOCS_OUTPUT/html diff --git a/MANIFEST.in b/MANIFEST.in index f424d6d..37f630a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include tests * \ No newline at end of file +recursive-include tests * diff --git a/README.rst b/README.rst index 6b397b2..c75c2c9 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ scrapy-spider-metadata .. description starts -``scrapy-spider-metadata`` is a Python 3.9+ library of utilities to extend +``scrapy-spider-metadata`` is a Python 3.10+ library of utilities to extend Scrapy spiders with usable metadata. In particular, it provides a nice way for Scrapy spiders to define, validate, diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..ba65b13 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/docs/_ext/__init__.py b/docs/_ext/__init__.py deleted file mode 100644 index 5a3839e..0000000 --- a/docs/_ext/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -def setup(app): - # https://stackoverflow.com/a/13663325 - # - # Scrapy’s - # https://github.com/scrapy/scrapy/blob/dba37674e6eaa6c2030c8eb35ebf8127cd488062/docs/_ext/scrapydocs.py#L90C16-L110C6 - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) diff --git a/docs/conf.py b/docs/conf.py index df09da8..290b1d0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,30 +1,10 @@ -import sys -from pathlib import Path - project = "scrapy-spider-metadata" copyright = "2023, Zyte Group Ltd" author = "Zyte Group Ltd" release = "0.0.0" -sys.path.insert(0, str(Path(__file__).parent.absolute())) # _ext extensions = [ - "_ext", - "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", + "sphinx_scrapy", ] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -html_theme = "sphinx_rtd_theme" - -intersphinx_mapping = { - "python": ( - "https://docs.python.org/3", - None, - ), - "scrapy": ( - "https://docs.scrapy.org/en/latest", - None, - ), -} diff --git a/docs/metadata.rst b/docs/metadata.rst index 3152182..cae9d49 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -17,6 +17,7 @@ attribute named ``metadata``: from scrapy import Spider + class MySpider(Spider): name = "my_spider" metadata = { @@ -32,18 +33,21 @@ add or change some values in a subclass: from scrapy import Spider + class BaseSpider(Spider): metadata = { "description": "Base spider.", "category": "Base spiders", } + class BaseNewsSpider(BaseSpider): metadata = { **BaseSpider.metadata, "description": "Base news spider.", } + class CNNSpider(BaseNewsSpider): metadata = { **BaseNewsSpider.metadata, diff --git a/docs/params.rst b/docs/params.rst index c01b271..70f74ea 100644 --- a/docs/params.rst +++ b/docs/params.rst @@ -37,9 +37,11 @@ as its parameter: from scrapy import Request, Spider from scrapy_spider_metadata import Args + class MyParams(BaseModel): pages: int + class MySpider(Args[MyParams], Spider): name = "my_spider" diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 0000000..fcc27ae --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1 @@ +sphinx-scrapy==0.13.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index 7633954..2a1ea52 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,75 @@ -Sphinx==7.2.4 # https://github.com/sphinx-doc/sphinx/issues/11662#issuecomment-1699989128 -sphinx-rtd-theme==1.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile docs/requirements.in -o docs/requirements.txt --python-version 3.14 +alabaster==1.0.0 + # via sphinx +babel==2.18.0 + # via sphinx +certifi==2026.7.22 + # via requests +charset-normalizer==3.5.2 + # via requests +docutils==0.22.4 + # via + # sphinx + # sphinx-llm-friendly + # sphinx-scrapy +idna==3.20 + # via requests +imagesize==2.0.1 + # via sphinx +jinja2==3.1.6 + # via sphinx +markupsafe==3.0.4 + # via jinja2 +packaging==26.3 + # via + # sphinx + # sphinx-scrapy +pygments==2.21.0 + # via sphinx +requests==2.34.2 + # via + # sphinx + # sphinxcontrib-youtube +roman-numerals==4.1.0 + # via sphinx +snowballstemmer==3.1.1 + # via sphinx +sphinx==9.1.0 + # via + # sphinx-copybutton + # sphinx-design + # sphinx-last-updated-by-git + # sphinx-llm-friendly + # sphinx-scrapy + # sphinxcontrib-youtube +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-design==0.7.0 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llm-friendly==0.7.0 + # via sphinx-scrapy +sphinx-scrapy==0.13.0 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +sphinxcontrib-youtube==1.5.0 + # via sphinx-scrapy +tabulate==0.10.0 + # via sphinx-llm-friendly +urllib3==2.8.0 + # via requests diff --git a/pyproject.toml b/pyproject.toml index 582288b..e56dc32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,15 +15,17 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", + "Typing :: Typed", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "packaging>=14.1", "pydantic>=1.9", @@ -185,7 +187,11 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel"] [tool.ruff.lint.per-file-ignores] # `from __future__ import annotations` breaks Pydantic 1.x -"tests/test_params.py" = ["FA100"] +# Pydantic version-specific imports are done inside the tests +"tests/test_params.py" = ["FA100", "PLC0415"] [tool.ruff.lint.pydocstyle] convention = "pep257" + +[tool.sphinx-scrapy] +python-version = "3.14" diff --git a/tests/test_params.py b/tests/test_params.py index 8b1ca01..38b9319 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -1,5 +1,5 @@ from enum import Enum, IntEnum -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast import pytest from packaging import version @@ -581,8 +581,8 @@ class Params(BaseModel): default=0, ) int_with_default: int = 1 - int_optional: Optional[int] = None - int_optional_without_default: Optional[int] + int_optional: int | None = None + int_optional_without_default: int | None number_without_default: float phone: str = Field( min_length=3, @@ -628,7 +628,7 @@ class Params(BaseModel): # https://github.com/pydantic/pydantic/issues/3753#issuecomment-1060850457 default=..., ) - dessert_optional: Optional[DessertEnum] = None + dessert_optional: DessertEnum | None = None dessert_required: DessertEnum class ParamSpider(Args[Params], Spider): diff --git a/tox.ini b/tox.ini index 1a1b718..893f383 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,6 @@ [tox] +requires = + sphinx-scrapy[tox]==0.13.0 envlist = py,pre-commit,mypy,types,docs,twinecheck [pytest] @@ -25,20 +27,22 @@ deps = pydantic<2 [testenv:min-1x] -base_python = python3.9 +base_python = python3.10 deps = {[testenv]deps} packaging==14.1 pydantic==1.9.0 - scrapy==1.0.0 + scrapy==2.6.0 + w3lib<2 [testenv:min-2x] -base_python = python3.9 +base_python = python3.10 deps = {[testenv]deps} packaging==14.1 pydantic==2.0 - scrapy==1.0.0 + scrapy==2.6.0 + w3lib<2 [testenv:pre-commit] deps = @@ -47,31 +51,23 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:mypy] deps = - mypy==1.15.0 - pytest==8.3.3 + mypy==2.4.0 + pytest==9.1.1 commands = mypy --strict {posargs:scrapy_spider_metadata tests} [testenv:types] deps = {[testenv]deps} {[testenv:mypy]deps} - pytest-mypy-testing==0.1.3 + pytest-mypy-testing==0.2.0 commands = pytest {posargs:tests_typing} -[testenv:docs] -basepython = python3 -changedir = docs -deps = - -rdocs/requirements.txt -commands = - sphinx-build -W -b html . {envtmpdir}/html - [testenv:twinecheck] basepython = python3 deps = - twine==6.1.0 - build==1.2.2.post1 + twine==7.0.0 + build==1.6.1 commands = python -m build --sdist twine check dist/*