Major rewrite to v3

.cruft.json

@@ -0,0 +1,18 @@
+{
+  "template": "https://github.com/woltapp/wolt-python-package-cookiecutter",
+  "commit": "b25684a2c63387153b83a1cc03ea332be8c8279a",
+  "checkout": null,
+  "context": {
+    "cookiecutter": {
+      "author_name": "Markus Shepherd",
+      "author_email": "markus@recommend.games",
+      "github_username": "recommend-games",
+      "project_name": "Board Game Scraper",
+      "project_slug": "board-game-scraper",
+      "package_name": "board_game_scraper",
+      "project_short_description": "Scraping data about board games from the web",
+      "_template": "https://github.com/woltapp/wolt-python-package-cookiecutter"
+    }
+  },
+  "directory": null
+}

.dockerignore

@@ -105,6 +105,9 @@
 # mypy
 **/.mypy*
 
+# ruff
+**/.ruff_cache/
+
 # other
 **/*.bk
 **/*backup*/
@@ -122,13 +125,15 @@
 **/LICENSE*
 
 **/Dockerfile
+**/compose.yaml
 **/docker-compose.yaml
 **/requirements*
 **/tox.ini
 **/_config.*
 analytics
 cluster
 feeds
+feeds_v3
 jobs
 results
 test

.env.example

@@ -1,32 +1,12 @@
-# the following line will be updated automatically from board_game_scraper.__version__
-LIBRARY_VERSION=
 # environment variables
 LOG_LEVEL=DEBUG
 LOG_SCRAPED_ITEMS=0
-ROBOTSTXT_OBEY=1
-SCRAPER_FILE_TAG=<hostname>
-# you only need these settings if you want to prioritise certain BGG users
-GOOGLE_APPLICATION_CREDENTIALS=/path/to/gs.json
-# PubSub queue settings
-PULL_QUEUE_PROJECT=<pubsub-project>
-PULL_QUEUE_SUBSCRIPTION_LOGS=<pubsub-subscription-logs>
-PULL_QUEUE_SUBSCRIPTION_RESPONSES=<pubsub-subscription-responses>
-PULL_QUEUE_INTERVAL=300
+FEED_EXPORT_FILE_SUFFIX=<hostname>
 # Scrape premium users
 SCRAPE_PREMIUM_USERS_LIST=
 SCRAPE_PREMIUM_USERS_CONFIG_DIR=
 SCRAPE_PREMIUM_USERS_INTERVAL=1800
 SCRAPE_PREMIUM_USERS_PREVENT_RESCRAPE_FOR=10800
-# AWS credentials if you need access to S3
-AWS_ACCESS_KEY_ID=<access-key>
-AWS_SECRET_ACCESS_KEY=<secret-access-key>
-# limit downloaded images per game and spider
-# use 0 to disable download; -1 for all images
-LIMIT_IMAGES_TO_DOWNLOAD_BGG=0
-LIMIT_IMAGES_TO_DOWNLOAD_DBPEDIA=0
-LIMIT_IMAGES_TO_DOWNLOAD_LUDING=0
-LIMIT_IMAGES_TO_DOWNLOAD_SPIELEN=0
-LIMIT_IMAGES_TO_DOWNLOAD_WIKIDATA=0
 # BGG username and password
 BGG_USERNAME=
 BGG_PASSWORD=

.github/actions/python-poetry-env/action.yml

@@ -0,0 +1,21 @@
+name: 'Setup Python + Poetry environment'
+description: 'Setup Python + Poetry environment'
+
+inputs:
+  python-version:
+    required: false
+    description: 'Python version'
+    default: '3.13'
+outputs: {}
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{inputs.python-version}}
+    - name: Install poetry
+      run: python -m pip install poetry --all-extras --all-groups
+      shell: bash
+    - name: Create virtual environment
+      run: poetry install
+      shell: bash

.github/pull_request_template.md

@@ -0,0 +1,10 @@
+## Description
+
+A short description about the changes in this pull request. If the pull request is related to some issue, mention it
+ here.
+
+## Checklist
+
+- [ ] Tests covering the new functionality have been added
+- [ ] Documentation has been updated OR the change is too minor to be documented
+- [ ] Changes are listed in the `CHANGELOG.md` OR changes are insignificant

.github/workflows/cookiecutter.yml

@@ -0,0 +1,79 @@
+name: Autoupdate project structure
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"  # at the end of every day
+
+jobs:
+  auto-update-project:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: python -m pip install cruft poetry jello tabulate
+
+      - name: Update project structure
+        run: |
+          cruft update -y
+
+      - name: Check if there are changes
+        id: changes
+        run: echo "::set-output name=changed::$(git status --porcelain | wc -l)"
+
+      - name: apply additional changes and fixes
+        if: steps.changes.outputs.changed > 0
+        run: |
+          poetry lock --no-update  # add new dependencies
+          poetry install
+          poetry run pre-commit run -a || true  # we have to fix other issues manually
+
+      - name: Get template versions
+        id: get_versions
+        if: steps.changes.outputs.changed > 0
+        shell: bash
+        run: |
+          CURRENT_VERSION=$(git show HEAD:.cruft.json | jello -r "_['commit'][:8]")
+          NEXT_VERSION=$(jello -r "_['commit'][:8]" < .cruft.json)
+          echo ::set-output name="current_version::$CURRENT_VERSION"
+          echo ::set-output name="next_version::$NEXT_VERSION"
+
+      - name: Get changelog
+        id: get_changelog
+        if: steps.changes.outputs.changed > 0
+        shell: bash
+        run: |
+          TEMPLATE=$(jello -r "_['template']" < .cruft.json)
+          git clone "$TEMPLATE" /tmp/template
+          cd /tmp/template
+          body=$( (echo "Date;Change;Hash"; git log --pretty=format:"%as;%s;%h" ${{ steps.get_versions.outputs.current_version }}..${{ steps.get_versions.outputs.next_version }}) | tabulate --header --format github -s ';' -)
+          body=$(cat <<EOF
+          Changes from $TEMPLATE
+
+          $body
+          EOF
+          )
+          body="${body//'%'/'%25'}"
+          body="${body//$'\n'/'%0A'}"
+          body="${body//$'\r'/'%0D'}"
+          echo ::set-output name="changelog::$body"
+
+      # behaviour if PR already exists: https://github.com/marketplace/actions/create-pull-request#action-behaviour
+      - name: Create Pull Request
+        env:
+          # a PAT is required to be able to update workflows
+          GITHUB_TOKEN: ${{ secrets.AUTO_UPDATE_GITHUB_TOKEN }}
+        if: ${{ steps.changes.outputs.changed > 0 && env.GITHUB_TOKEN != 0 }}
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ env.GITHUB_TOKEN }}
+          commit-message: >-
+            chore: update project structure to ${{ steps.get_versions.outputs.next_version }}
+          title: "[Actions] Auto-Update cookiecutter template"
+          body: ${{ steps.get_changelog.outputs.changelog }}
+          branch: chore/auto-update-project-from-template
+          delete-branch: true

.github/workflows/dependencies.yml

@@ -0,0 +1,57 @@
+name: Autoupdate dependencies
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 1 * *"
+
+jobs:
+  auto-update-dependencies:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+
+      - name: Install tabulate
+        run: python -m pip install tabulate
+
+      - name: Gather outdated dependencies
+        id: check_for_outdated_dependencies
+        run: |
+          body=$(poetry show -o -n)
+          echo ::set-output name="body::$body"
+
+      - name: Format PR message
+        if: ${{ steps.check_for_outdated_dependencies.outputs.body != 0 }}
+        id: get_outdated_dependencies
+        shell: bash
+        run: |
+          body=$(poetry show -o -n | sed 's/(!)//' | awk 'BEGIN {print "Package","Used","Update"}; {print $1,$2,$3}' | tabulate --header --format github -)
+          body=$(cat <<EOF
+          The following packages are outdated
+
+          $body
+          EOF
+          )
+          body="${body//'%'/'%25'}"
+          body="${body//$'\n'/'%0A'}"
+          body="${body//$'\r'/'%0D'}"
+          echo ::set-output name="body::$body"
+
+      - name: Update outdated packages
+        if: ${{ steps.check_for_outdated_dependencies.outputs.body != 0 }}
+        run: poetry lock
+
+      # behaviour if PR already exists: https://github.com/marketplace/actions/create-pull-request#action-behaviour
+      - name: Create Pull Request
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: ${{ steps.check_for_outdated_dependencies.outputs.body != 0 }}
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ env.GITHUB_TOKEN }}
+          commit-message: >-
+            chore: update dependencies
+          title: "[Actions] Auto-Update dependencies"
+          body: ${{ steps.get_outdated_dependencies.outputs.body }}
+          branch: chore/update-dependencies
+          delete-branch: true

.github/workflows/draft_release.yml

@@ -0,0 +1,52 @@
+name: Draft a release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version number (e.g. 1.2.3) OR one of: patch|minor|major|prepatch|preminor|premajor|prerelease'
+        required: true
+        default: 'patch'
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+      - name: Update version
+        id: updated_version
+        shell: bash
+        run: |
+          poetry version ${{ github.event.inputs.version }}
+          version=$(poetry version --short)
+          echo ::set-output name="version::$version"
+      - name: Update changelog
+        id: changelog
+        shell: bash
+        run: |
+          poetry run kacl-cli release ${{ steps.updated_version.outputs.version }} --modify --auto-link
+          echo "" >> CHANGELOG.md
+          body=$(poetry run kacl-cli get ${{ steps.updated_version.outputs.version }})
+          body="${body//'%'/'%25'}"
+          body="${body//$'\n'/'%0A'}"
+          body="${body//$'\r'/'%0D'}"
+          echo ::set-output name="body::$body"
+      - name: Commit changes
+        uses: EndBug/add-and-commit@v7
+        with:
+          add: 'CHANGELOG.md pyproject.toml'
+          message: 'Release ${{ steps.updated_version.outputs.version }}'
+      - name: Create tag
+        run: |
+          git tag ${{ steps.updated_version.outputs.version }}
+          git push origin ${{ steps.updated_version.outputs.version }}
+      - name: Create a draft release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.updated_version.outputs.version }}
+          release_name: Release ${{ steps.updated_version.outputs.version }}
+          body: ${{ steps.changelog.outputs.body }}
+          draft: true

.github/workflows/release.yml

@@ -0,0 +1,18 @@
+name: Release
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+      - name: Publish to pypi
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
+          poetry publish --build --no-interaction
+      - name: Deploy docs
+        run: poetry run mkdocs gh-deploy --force

.github/workflows/test.yml

@@ -0,0 +1,54 @@
+name: Test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  actionlint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download actionlint
+        run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) 1.6.21
+        shell: bash
+      - name: Check workflow files
+        run: ./actionlint -color
+        shell: bash
+
+  lint-cruft:
+    name: Check if automatic project update was successful
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fail if .rej files exist as structure update was not successful
+        run: test -z "$(find . -iname '*.rej')"
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+      - run: poetry run pre-commit run --all-files
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: poetry run pytest
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/python-poetry-env
+      - run: poetry run mkdocs build