Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7
15 changes: 10 additions & 5 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,25 @@ on:
push:
tags:
- "[0-9]+.[0-9]+.[0-9]+"
permissions:
contents: read
jobs:
deploy:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # required for PyPI trusted publishing
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: '3.13'
- name: Install dependencies
run: |
pip install --upgrade build twine
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.10.2
with:
password: ${{ secrets.PYPI_TOKEN }}
uses: pypa/gh-action-pypi-publish@897895f1e160c830e369f9779632ebc134688e1b # v1.10.2
27 changes: 18 additions & 9 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,31 @@ on:
pull_request:
push:
branches: [ main ]
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.9"
toxenv: "min"
- python-version: "3.9"
- python-version: "3.10"
toxenv: "min-1x"
- python-version: "3.10"
toxenv: "min-2x"
- python-version: "3.10"
- python-version: "3.11"
- python-version: "3.12"
- python-version: "3.13"
- python-version: "3.14"
- python-version: "3.15"
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
Expand All @@ -31,20 +38,22 @@ jobs:
run: |
tox -e ${{ matrix.toxenv || 'py' }}
- name: coverage
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
check:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ['3.13'] # Keep in sync with .readthedocs.yml
python-version: ['3.14'] # Keep in sync with .readthedocs.yml
tox-job: ["pre-commit", "mypy", "types", "docs", "twinecheck"]
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
Expand Down
32 changes: 30 additions & 2 deletions .pre-commit-config.yaml
Comment thread
wRAR marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -1,7 +1,35 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.2
rev: v0.17.0
hooks:
- id: ruff
- id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.20.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==26.10.1
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/sphinx-contrib/sphinx-lint
rev: v1.0.2
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.13.0
hooks:
- id: sphinx-scrapy
- repo: https://github.com/rhysd/actionlint
rev: v1.7.12
hooks:
- id: actionlint
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.30.1
hooks:
- id: zizmor
args: [--no-progress, --fix]
Comment thread
wRAR marked this conversation as resolved.
Resolved
12 changes: 4 additions & 8 deletions .readthedocs.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
version: 2
formats: all
sphinx:
configuration: docs/conf.py
build:
os: ubuntu-24.04
tools:
python: "3.13" # Keep in sync with .github/workflows/tests.yml
python:
install:
- requirements: docs/requirements.txt
- path: .
python: "3.14"
commands:
- pip install tox
- tox -e docs -- $READTHEDOCS_OUTPUT/html
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1 +1 @@
recursive-include tests *
recursive-include tests *
2 changes: 1 addition & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ scrapy-spider-metadata

.. description starts

``scrapy-spider-metadata`` is a Python 3.9+ library of utilities to extend
``scrapy-spider-metadata`` is a Python 3.10+ library of utilities to extend
Scrapy spiders with usable metadata.

In particular, it provides a nice way for Scrapy spiders to define, validate,
Expand Down
1 change: 1 addition & 0 deletions docs/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/_build/
25 changes: 0 additions & 25 deletions docs/_ext/__init__.py

This file was deleted.

22 changes: 1 addition & 21 deletions docs/conf.py
Original file line number Diff line number Diff line change
@@ -1,30 +1,10 @@
import sys
from pathlib import Path

project = "scrapy-spider-metadata"
copyright = "2023, Zyte Group Ltd"
author = "Zyte Group Ltd"
release = "0.0.0"

sys.path.insert(0, str(Path(__file__).parent.absolute())) # _ext
extensions = [
"_ext",
"sphinx.ext.autodoc",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
"sphinx_scrapy",
]

exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]

html_theme = "sphinx_rtd_theme"

intersphinx_mapping = {
"python": (
"https://docs.python.org/3",
None,
),
"scrapy": (
"https://docs.scrapy.org/en/latest",
None,
),
}
4 changes: 4 additions & 0 deletions docs/metadata.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ attribute named ``metadata``:

from scrapy import Spider


class MySpider(Spider):
name = "my_spider"
metadata = {
Expand All @@ -32,18 +33,21 @@ add or change some values in a subclass:

from scrapy import Spider


class BaseSpider(Spider):
metadata = {
"description": "Base spider.",
"category": "Base spiders",
}


class BaseNewsSpider(BaseSpider):
metadata = {
**BaseSpider.metadata,
"description": "Base news spider.",
}


class CNNSpider(BaseNewsSpider):
metadata = {
**BaseNewsSpider.metadata,
Expand Down
2 changes: 2 additions & 0 deletions docs/params.rst
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@ as its parameter:
from scrapy import Request, Spider
from scrapy_spider_metadata import Args


class MyParams(BaseModel):
pages: int


class MySpider(Args[MyParams], Spider):
name = "my_spider"

Expand Down
1 change: 1 addition & 0 deletions docs/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sphinx-scrapy==0.13.0
77 changes: 75 additions & 2 deletions docs/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,75 @@
Sphinx==7.2.4 # https://github.com/sphinx-doc/sphinx/issues/11662#issuecomment-1699989128
sphinx-rtd-theme==1.3.0
# This file was autogenerated by uv via the following command:
# uv pip compile docs/requirements.in -o docs/requirements.txt --python-version 3.14
alabaster==1.0.0
# via sphinx
babel==2.18.0
# via sphinx
certifi==2026.7.22
# via requests
charset-normalizer==3.5.2
# via requests
docutils==0.22.4
# via
# sphinx
# sphinx-llm-friendly
# sphinx-scrapy
idna==3.20
# via requests
imagesize==2.0.1
# via sphinx
jinja2==3.1.6
# via sphinx
markupsafe==3.0.4
# via jinja2
packaging==26.3
# via
# sphinx
# sphinx-scrapy
pygments==2.21.0
# via sphinx
requests==2.34.2
# via
# sphinx
# sphinxcontrib-youtube
roman-numerals==4.1.0
# via sphinx
snowballstemmer==3.1.1
# via sphinx
sphinx==9.1.0
# via
# sphinx-copybutton
# sphinx-design
# sphinx-last-updated-by-git
# sphinx-llm-friendly
# sphinx-scrapy
# sphinxcontrib-youtube
sphinx-copybutton==0.5.2
# via sphinx-scrapy
sphinx-design==0.7.0
# via sphinx-scrapy
sphinx-last-updated-by-git==0.3.8
# via sphinx-sitemap
sphinx-llm-friendly==0.7.0
# via sphinx-scrapy
sphinx-scrapy==0.13.0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you switch the project to use requirements.in / requirements.txt?

# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy
sphinxcontrib-applehelp==2.0.0
# via sphinx
sphinxcontrib-devhelp==2.0.0
# via sphinx
sphinxcontrib-htmlhelp==2.1.0
# via sphinx
sphinxcontrib-jsmath==1.0.1
# via sphinx
sphinxcontrib-qthelp==2.0.0
# via sphinx
sphinxcontrib-serializinghtml==2.0.0
# via sphinx
sphinxcontrib-youtube==1.5.0
# via sphinx-scrapy
tabulate==0.10.0
# via sphinx-llm-friendly
urllib3==2.8.0
# via requests
12 changes: 9 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,17 @@ classifiers = [
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Python :: 3.15",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Typing :: Typed",
]
requires-python = ">=3.9"
requires-python = ">=3.10"
dependencies = [
"packaging>=14.1",
"pydantic>=1.9",
Expand Down Expand Up @@ -185,7 +187,11 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel"]

[tool.ruff.lint.per-file-ignores]
# `from __future__ import annotations` breaks Pydantic 1.x
"tests/test_params.py" = ["FA100"]
# Pydantic version-specific imports are done inside the tests
"tests/test_params.py" = ["FA100", "PLC0415"]

[tool.ruff.lint.pydocstyle]
convention = "pep257"

[tool.sphinx-scrapy]
python-version = "3.14"
Loading
Loading