From 6366395e46c1ce9dd6e9e0cae7c0798d274e60cb Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:06:22 +0000
Subject: [PATCH 01/11] Fetch existing HACS data once via a preflight job

The generate workflow fetched the existing published data from R2 inside
every category leg: each leg called HacsDataClient.get_data(<category>) for
the category's data.json, and get_repositories("removed") for the removed
list. Since the six legs run serially, the removed list was fetched once per
category, and a republish of R2 mid-run could leave later legs working off a
different baseline than earlier ones.

Fetch that data exactly once in a new lightweight preflight job (curl only)
and share it with every category leg via an artifact, so the whole run uses
one consistent snapshot:

- generate_category_data.py: add get_stored_data()/get_removed_repositories()
  helpers that read each category's data.json and the removed list from
  $HACS_EXISTING_DATA_DIR when set, and fall back to fetching from the data
  client when unset (tests, local dev, single-repo validate.yml). Re-point the
  two existing fetches at these helpers.
- generate-hacs-data.yml: add a preflight job that fetches every category's
  data.json plus the removed list into outputdata/existing (retrying 5 times
  then failing the run), uploads it as the existing-data artifact, and has
  category-data depend on it, download it, and run with
  HACS_EXISTING_DATA_DIR=outputdata/existing. Add preflight to
  notify_on_failure.needs.
- Add unit tests covering both helper paths (snapshot dir vs. fetch fallback).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml      | 55 ++++++++++++++-
 scripts/data/generate_category_data.py        | 39 ++++++++++-
 .../data/test_generate_category_data.py       | 68 ++++++++++++++++++-
 3 files changed, 157 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index 4117d1c5a5a..a3abf2fab5e 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -52,10 +52,54 @@ jobs:
             echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
           fi
 
-  category-data:
+  preflight:
+    name: Fetch existing data
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
+    steps:
+      # Fetch the existing published data once, so every category leg works off
+      # the same consistent baseline (no skew if R2 is republished mid-run) and
+      # the removed list is fetched a single time instead of once per category.
+      # URLs mirror custom_components/hacs/data_client.py::_do_request.
+      - name: Fetch existing published data
+        env:
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
+        run: |
+          mkdir -p outputdata/existing
+
+          # generate-matrix emits categories single-quoted (['a','b',...]),
+          # which fromJSON accepts but jq does not; normalise to bare names.
+          categories=$(printf '%s' "$CATEGORIES" | tr -d "[]\"' " | tr ',' '\n')
+
+          fetch() {
+            curl \
+              --silent --show-error --fail --retry 5 --retry-all-errors \
+              --header "User-Agent: HACS/Generator" \
+              "https://data-v2.hacs.xyz/$1" \
+              --output "$2"
+          }
+
+          for category in $categories; do
+            echo "Fetching existing data for $category"
+            fetch "${category}/data.json" "outputdata/existing/${category}.json"
+          done
+
+          echo "Fetching removed repositories"
+          fetch "removed/repositories.json" "outputdata/existing/removed.json"
+
+      - name: Upload existing data
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+          if-no-files-found: error
+          retention-days: 1
+
+  category-data:
+    runs-on: ubuntu-latest
+    needs: [generate-matrix, preflight]
+    if: github.repository == 'hacs/integration'
     name: Generate ${{ matrix.category }} data
     strategy:
       fail-fast: false
@@ -83,11 +127,18 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
+      - name: Download existing data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: existing-data
+          path: outputdata/existing
+
       - name: Generate ${{ matrix.category }} data
         run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
+          HACS_EXISTING_DATA_DIR: outputdata/existing
 
       - name: Validate output with JQ
         run: |
@@ -287,7 +338,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "preflight", "category-data", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification
diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index fe04c61df21..41084c19aba 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -58,6 +58,41 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
+# When set, the existing published data (each category's data.json and the
+# removed list) is read from this directory instead of being fetched from R2.
+# A preflight job fetches that snapshot once and shares it with every category
+# leg, so the whole run works off one consistent baseline. When unset (tests,
+# local dev, single-repo validate.yml) the data is fetched as before.
+EXISTING_DATA_DIR_ENV = "HACS_EXISTING_DATA_DIR"
+
+
+async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
+    """Return the existing published data for a category.
+
+    Read from the ``$HACS_EXISTING_DATA_DIR`` snapshot when set, otherwise
+    fetched from the data client.
+    """
+    if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
+        with open(
+            os.path.join(existing_dir, f"{category}.json"), encoding="utf-8"
+        ) as file:
+            return json.load(file)
+    return await hacs.data_client.get_data(category, validate=False)
+
+
+async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
+    """Return the list of repositories removed from HACS.
+
+    Read from the ``$HACS_EXISTING_DATA_DIR`` snapshot when set, otherwise
+    fetched from the data client.
+    """
+    if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
+        with open(
+            os.path.join(existing_dir, "removed.json"), encoding="utf-8"
+        ) as file:
+            return json.load(file)
+    return await hacs.data_client.get_repositories("removed")
+
 
 def jsonprint(data: any):
     print(
@@ -309,7 +344,7 @@ async def generate_data_for_category(
         removed = (
             []
             if repository_name is not None
-            else await self.data_client.get_repositories("removed")
+            else await get_removed_repositories(self)
         )
         await self.data.register_base_data(
             category,
@@ -461,7 +496,7 @@ async def generate_category_data(category: str, repository_name: str = None):
         os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
         os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
         force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
-        stored_data = await hacs.data_client.get_data(category, validate=False)
+        stored_data = await get_stored_data(hacs, category)
         current_data = (
             next(
                 (
diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py
index 6b602d9feb8..12e343344bd 100644
--- a/tests/scripts/data/test_generate_category_data.py
+++ b/tests/scripts/data/test_generate_category_data.py
@@ -8,7 +8,13 @@
 from homeassistant.core import HomeAssistant
 import pytest
 
-from scripts.data.generate_category_data import OUTPUT_DIR, generate_category_data
+from scripts.data.generate_category_data import (
+    EXISTING_DATA_DIR_ENV,
+    OUTPUT_DIR,
+    generate_category_data,
+    get_removed_repositories,
+    get_stored_data,
+)
 
 from tests.common import (
     FIXTURES_PATH,
@@ -311,3 +317,63 @@ async def test_generate_category_data_with_30plus_prereleases(
         f"scripts/data/test_generate_category_data_with_30plus_prereleases/{
             category_test_data['category']}.json",
     )
+
+
+class _StubDataClient:
+    """Minimal data client recording that a fetch happened."""
+
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str]] = []
+
+    async def get_data(self, section: str, *, validate: bool) -> dict[str, Any]:
+        self.calls.append(("get_data", section))
+        return {"fetched": section}
+
+    async def get_repositories(self, section: str) -> list[str]:
+        self.calls.append(("get_repositories", section))
+        return [f"fetched/{section}"]
+
+
+class _StubHacs:
+    """Minimal HACS stand-in exposing only a data client."""
+
+    def __init__(self) -> None:
+        self.data_client = _StubDataClient()
+
+
+async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, stored data is read from it, not fetched."""
+    monkeypatch.setenv(EXISTING_DATA_DIR_ENV, str(tmp_path))
+    payload = {"1": {"full_name": "octocat/Hello-World"}}
+    (tmp_path / "integration.json").write_text(json.dumps(payload))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "integration") == payload
+    assert hacs.data_client.calls == []
+
+
+async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
+    """When the snapshot dir is set, the removed list is read from it, not fetched."""
+    monkeypatch.setenv(EXISTING_DATA_DIR_ENV, str(tmp_path))
+    removed = ["octocat/Hello-World", "hacs/integration"]
+    (tmp_path / "removed.json").write_text(json.dumps(removed))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == removed
+    assert hacs.data_client.calls == []
+
+
+async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, stored data is fetched from the data client."""
+    monkeypatch.delenv(EXISTING_DATA_DIR_ENV, raising=False)
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == {"fetched": "plugin"}
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
+    """Without the snapshot dir, the removed list is fetched from the data client."""
+    monkeypatch.delenv(EXISTING_DATA_DIR_ENV, raising=False)
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == ["fetched/removed"]
+    assert hacs.data_client.calls == [("get_repositories", "removed")]

From 53ee4661c67e3e822bee4f7cb51b94cdce1ece38 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:11:06 +0000
Subject: [PATCH 02/11] Trim comments

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml |  7 +------
 scripts/data/generate_category_data.py   | 17 ++---------------
 2 files changed, 3 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index a3abf2fab5e..09a70e84264 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -58,18 +58,13 @@ jobs:
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
     steps:
-      # Fetch the existing published data once, so every category leg works off
-      # the same consistent baseline (no skew if R2 is republished mid-run) and
-      # the removed list is fetched a single time instead of once per category.
-      # URLs mirror custom_components/hacs/data_client.py::_do_request.
       - name: Fetch existing published data
         env:
           CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
         run: |
           mkdir -p outputdata/existing
 
-          # generate-matrix emits categories single-quoted (['a','b',...]),
-          # which fromJSON accepts but jq does not; normalise to bare names.
+          # categories is single-quoted (['a',...]); strip to bare names for the loop.
           categories=$(printf '%s' "$CATEGORIES" | tr -d "[]\"' " | tr ',' '\n')
 
           fetch() {
diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index 41084c19aba..d7f7dc83f22 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -58,20 +58,11 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
-# When set, the existing published data (each category's data.json and the
-# removed list) is read from this directory instead of being fetched from R2.
-# A preflight job fetches that snapshot once and shares it with every category
-# leg, so the whole run works off one consistent baseline. When unset (tests,
-# local dev, single-repo validate.yml) the data is fetched as before.
 EXISTING_DATA_DIR_ENV = "HACS_EXISTING_DATA_DIR"
 
 
 async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
-    """Return the existing published data for a category.
-
-    Read from the ``$HACS_EXISTING_DATA_DIR`` snapshot when set, otherwise
-    fetched from the data client.
-    """
+    """Return existing category data from the snapshot dir when set, else fetch."""
     if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
         with open(
             os.path.join(existing_dir, f"{category}.json"), encoding="utf-8"
@@ -81,11 +72,7 @@ async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[s
 
 
 async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
-    """Return the list of repositories removed from HACS.
-
-    Read from the ``$HACS_EXISTING_DATA_DIR`` snapshot when set, otherwise
-    fetched from the data client.
-    """
+    """Return the removed-repositories list from the snapshot dir when set, else fetch."""
     if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
         with open(
             os.path.join(existing_dir, "removed.json"), encoding="utf-8"

From 8f7d3bcf23f628dc853400ece25569ec09739a1d Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:13:00 +0000
Subject: [PATCH 03/11] Emit categories as valid JSON and parse with jq

Build the generate-matrix categories output with jq so it is valid JSON
(double-quoted) instead of single-quoted, letting both the matrix (fromJSON)
and the preflight loop (jq -r) parse it robustly.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index 09a70e84264..e32372c72d5 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -45,12 +45,18 @@ jobs:
       categories: ${{ steps.set-matrix.outputs.categories }}
     steps:
       - id: set-matrix
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_CATEGORY: ${{ inputs.category }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch"  ]] && [[ "${{ inputs.category }}" != "None"  ]] && [[ "${{ inputs.category }}" != ""  ]]; then
-            echo "categories=['${{ inputs.category }}']" >> $GITHUB_OUTPUT
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]] && [[ "$INPUT_CATEGORY" != "None" ]] && [[ -n "$INPUT_CATEGORY" ]]; then
+            categories=("$INPUT_CATEGORY")
           else
-            echo "categories=['appdaemon','integration','plugin','python_script','template','theme']" >> $GITHUB_OUTPUT
+            categories=(appdaemon integration plugin python_script template theme)
           fi
+          categories_json=$(printf '%s\n' "${categories[@]}" | jq -R . | jq -cs .)
+          echo "categories=$categories_json" >> $GITHUB_OUTPUT
+          echo "categories: $categories_json"
 
   preflight:
     name: Fetch existing data
@@ -64,9 +70,6 @@ jobs:
         run: |
           mkdir -p outputdata/existing
 
-          # categories is single-quoted (['a',...]); strip to bare names for the loop.
-          categories=$(printf '%s' "$CATEGORIES" | tr -d "[]\"' " | tr ',' '\n')
-
           fetch() {
             curl \
               --silent --show-error --fail --retry 5 --retry-all-errors \
@@ -75,7 +78,7 @@ jobs:
               --output "$2"
           }
 
-          for category in $categories; do
+          for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
             echo "Fetching existing data for $category"
             fetch "${category}/data.json" "outputdata/existing/${category}.json"
           done

From 42c3324222dcaac316f58896e3c893b777ac2de4 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:17:20 +0000
Subject: [PATCH 04/11] Store the resolved existing-data dir directly

Read HACS_EXISTING_DATA_DIR once at module level instead of keeping the env
var name in a constant and resolving it on every call.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 scripts/data/generate_category_data.py            | 10 +++++-----
 tests/scripts/data/test_generate_category_data.py | 12 +++++++-----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index d7f7dc83f22..9b2bc07c79b 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -58,14 +58,14 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
-EXISTING_DATA_DIR_ENV = "HACS_EXISTING_DATA_DIR"
+EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
 
 
 async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
     """Return existing category data from the snapshot dir when set, else fetch."""
-    if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
+    if EXISTING_DATA_DIR:
         with open(
-            os.path.join(existing_dir, f"{category}.json"), encoding="utf-8"
+            os.path.join(EXISTING_DATA_DIR, f"{category}.json"), encoding="utf-8"
         ) as file:
             return json.load(file)
     return await hacs.data_client.get_data(category, validate=False)
@@ -73,9 +73,9 @@ async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[s
 
 async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
     """Return the removed-repositories list from the snapshot dir when set, else fetch."""
-    if existing_dir := os.getenv(EXISTING_DATA_DIR_ENV):
+    if EXISTING_DATA_DIR:
         with open(
-            os.path.join(existing_dir, "removed.json"), encoding="utf-8"
+            os.path.join(EXISTING_DATA_DIR, "removed.json"), encoding="utf-8"
         ) as file:
             return json.load(file)
     return await hacs.data_client.get_repositories("removed")
diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py
index 12e343344bd..675cfc0cc01 100644
--- a/tests/scripts/data/test_generate_category_data.py
+++ b/tests/scripts/data/test_generate_category_data.py
@@ -9,7 +9,6 @@
 import pytest
 
 from scripts.data.generate_category_data import (
-    EXISTING_DATA_DIR_ENV,
     OUTPUT_DIR,
     generate_category_data,
     get_removed_repositories,
@@ -341,9 +340,12 @@ def __init__(self) -> None:
         self.data_client = _StubDataClient()
 
 
+_MODULE = "scripts.data.generate_category_data"
+
+
 async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
     """When the snapshot dir is set, stored data is read from it, not fetched."""
-    monkeypatch.setenv(EXISTING_DATA_DIR_ENV, str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
     payload = {"1": {"full_name": "octocat/Hello-World"}}
     (tmp_path / "integration.json").write_text(json.dumps(payload))
 
@@ -354,7 +356,7 @@ async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
 
 async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
     """When the snapshot dir is set, the removed list is read from it, not fetched."""
-    monkeypatch.setenv(EXISTING_DATA_DIR_ENV, str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
     removed = ["octocat/Hello-World", "hacs/integration"]
     (tmp_path / "removed.json").write_text(json.dumps(removed))
 
@@ -365,7 +367,7 @@ async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkey
 
 async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, stored data is fetched from the data client."""
-    monkeypatch.delenv(EXISTING_DATA_DIR_ENV, raising=False)
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
     hacs = _StubHacs()
     assert await get_stored_data(hacs, "plugin") == {"fetched": "plugin"}
     assert hacs.data_client.calls == [("get_data", "plugin")]
@@ -373,7 +375,7 @@ async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
 
 async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, the removed list is fetched from the data client."""
-    monkeypatch.delenv(EXISTING_DATA_DIR_ENV, raising=False)
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
     hacs = _StubHacs()
     assert await get_removed_repositories(hacs) == ["fetched/removed"]
     assert hacs.data_client.calls == [("get_repositories", "removed")]

From 8adff4b735a87ef04569944d605a0067707f312e Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:28:46 +0000
Subject: [PATCH 05/11] Fall back to fetching when a snapshot file is missing
 or invalid

Reading the existing-data snapshot now degrades gracefully: a missing file or
invalid JSON logs a warning and falls back to the data client instead of
aborting generation, so a partial snapshot cannot fail the run. Make the test
stub return realistic data-client shapes and add regression tests for the
missing/invalid-file fallback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 scripts/data/generate_category_data.py        | 35 ++++++++++-----
 .../data/test_generate_category_data.py       | 45 ++++++++++++++++---
 2 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index 9b2bc07c79b..0bc5e5e26dd 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -61,23 +61,34 @@
 EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
 
 
-async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
-    """Return existing category data from the snapshot dir when set, else fetch."""
-    if EXISTING_DATA_DIR:
-        with open(
-            os.path.join(EXISTING_DATA_DIR, f"{category}.json"), encoding="utf-8"
-        ) as file:
+def _read_snapshot(hacs: AdjustedHacs, filename: str) -> Any | None:
+    """Read a snapshot file from ``EXISTING_DATA_DIR``.
+
+    Returns the parsed JSON when available, or ``None`` (so the caller fetches
+    instead) when the dir is unset or the file is missing or unreadable.
+    """
+    if not EXISTING_DATA_DIR:
+        return None
+    try:
+        with open(os.path.join(EXISTING_DATA_DIR, filename), encoding="utf-8") as file:
             return json.load(file)
+    except (OSError, json.JSONDecodeError) as err:
+        hacs.log.warning(
+            "Could not read snapshot %s (%s), fetching instead", filename, err)
+        return None
+
+
+async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
+    """Return existing category data from the snapshot dir when available, else fetch."""
+    if (data := _read_snapshot(hacs, f"{category}.json")) is not None:
+        return data
     return await hacs.data_client.get_data(category, validate=False)
 
 
 async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
-    """Return the removed-repositories list from the snapshot dir when set, else fetch."""
-    if EXISTING_DATA_DIR:
-        with open(
-            os.path.join(EXISTING_DATA_DIR, "removed.json"), encoding="utf-8"
-        ) as file:
-            return json.load(file)
+    """Return the removed-repositories list from the snapshot dir when available, else fetch."""
+    if (removed := _read_snapshot(hacs, "removed.json")) is not None:
+        return removed
     return await hacs.data_client.get_repositories("removed")
 
 
diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py
index 675cfc0cc01..da5f5ccea10 100644
--- a/tests/scripts/data/test_generate_category_data.py
+++ b/tests/scripts/data/test_generate_category_data.py
@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import logging
 import os
 from typing import Any
 
@@ -318,26 +319,32 @@ async def test_generate_category_data_with_30plus_prereleases(
     )
 
 
+# Shapes mirror the real data client: get_data -> {repo-id: {...}}, removed -> [full_name].
+_FETCHED_STORED = {"1296269": {"full_name": "octocat/Hello-World", "category": "plugin"}}
+_FETCHED_REMOVED = ["octocat/removed-repo"]
+
+
 class _StubDataClient:
     """Minimal data client recording that a fetch happened."""
 
     def __init__(self) -> None:
         self.calls: list[tuple[str, str]] = []
 
-    async def get_data(self, section: str, *, validate: bool) -> dict[str, Any]:
+    async def get_data(self, section: str, *, validate: bool) -> dict[str, dict[str, Any]]:
         self.calls.append(("get_data", section))
-        return {"fetched": section}
+        return _FETCHED_STORED
 
     async def get_repositories(self, section: str) -> list[str]:
         self.calls.append(("get_repositories", section))
-        return [f"fetched/{section}"]
+        return _FETCHED_REMOVED
 
 
 class _StubHacs:
-    """Minimal HACS stand-in exposing only a data client."""
+    """Minimal HACS stand-in exposing only a data client and a logger."""
 
     def __init__(self) -> None:
         self.data_client = _StubDataClient()
+        self.log = logging.getLogger("test.generate_category_data")
 
 
 _MODULE = "scripts.data.generate_category_data"
@@ -369,7 +376,7 @@ async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, stored data is fetched from the data client."""
     monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
     hacs = _StubHacs()
-    assert await get_stored_data(hacs, "plugin") == {"fetched": "plugin"}
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
     assert hacs.data_client.calls == [("get_data", "plugin")]
 
 
@@ -377,5 +384,31 @@ async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, the removed list is fetched from the data client."""
     monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
     hacs = _StubHacs()
-    assert await get_removed_repositories(hacs) == ["fetched/removed"]
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkeypatch):
+    """An invalid-JSON snapshot file falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text("{ not valid json")
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
+    """A missing removed snapshot falls back to fetching instead of raising."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
     assert hacs.data_client.calls == [("get_repositories", "removed")]

From 3e112064892caaa0036ecaaa841803e017a85ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= <joasoe@proton.me>
Date: Thu, 23 Jul 2026 19:33:16 +0200
Subject: [PATCH 06/11] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .github/workflows/generate-hacs-data.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index e32372c72d5..f8a142842ca 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -73,6 +73,7 @@ jobs:
           fetch() {
             curl \
               --silent --show-error --fail --retry 5 --retry-all-errors \
+              --connect-timeout 10 --max-time 60 \
               --header "User-Agent: HACS/Generator" \
               "https://data-v2.hacs.xyz/$1" \
               --output "$2"

From d7cce99386842cd1fbfe39749436646e7e9be5cd Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:35:55 +0000
Subject: [PATCH 07/11] Name the preflight fetch helper

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index f8a142842ca..bc19d27f814 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -70,7 +70,7 @@ jobs:
         run: |
           mkdir -p outputdata/existing
 
-          fetch() {
+          fetch_stored_hacs_content() {
             curl \
               --silent --show-error --fail --retry 5 --retry-all-errors \
               --connect-timeout 10 --max-time 60 \
@@ -81,11 +81,11 @@ jobs:
 
           for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
             echo "Fetching existing data for $category"
-            fetch "${category}/data.json" "outputdata/existing/${category}.json"
+            fetch_stored_hacs_content "${category}/data.json" "outputdata/existing/${category}.json"
           done
 
           echo "Fetching removed repositories"
-          fetch "removed/repositories.json" "outputdata/existing/removed.json"
+          fetch_stored_hacs_content "removed/repositories.json" "outputdata/existing/removed.json"
 
       - name: Upload existing data
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

From 2ff5483a89badfa616d1391d6cbcee16e8908dec Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Thu, 23 Jul 2026 17:40:58 +0000
Subject: [PATCH 08/11] Rename the preflight job to fetch-stored-data

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index bc19d27f814..2dbbbd77acf 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -58,8 +58,8 @@ jobs:
           echo "categories=$categories_json" >> $GITHUB_OUTPUT
           echo "categories: $categories_json"
 
-  preflight:
-    name: Fetch existing data
+  fetch-stored-data:
+    name: Fetch stored data
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
@@ -97,7 +97,7 @@ jobs:
 
   category-data:
     runs-on: ubuntu-latest
-    needs: [generate-matrix, preflight]
+    needs: [generate-matrix, fetch-stored-data]
     if: github.repository == 'hacs/integration'
     name: Generate ${{ matrix.category }} data
     strategy:
@@ -337,7 +337,7 @@ jobs:
   notify_on_failure:
     runs-on: ubuntu-latest
     name: Trigger Discord notification when jobs fail
-    needs: ["generate-matrix", "preflight", "category-data", "summarize", "publish"]
+    needs: ["generate-matrix", "fetch-stored-data", "category-data", "summarize", "publish"]
     if: ${{ always() && github.repository == 'hacs/integration' && contains(join(needs.*.result, ','), 'failure') && github.event_name == 'schedule' }}
     steps:
       - name: Send notification

From 2418380d97721d7cf53fce21185157d8249564c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= <joasoe@proton.me>
Date: Fri, 24 Jul 2026 21:17:43 +0200
Subject: [PATCH 09/11] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 scripts/data/generate_category_data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index 0bc5e5e26dd..54c95be837f 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -61,7 +61,7 @@
 EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
 
 
-def _read_snapshot(hacs: AdjustedHacs, filename: str) -> Any | None:
+def _read_snapshot(hacs: AdjustedHacs, filename: str) -> object | None:
     """Read a snapshot file from ``EXISTING_DATA_DIR``.
 
     Returns the parsed JSON when available, or ``None`` (so the caller fetches

From 741433d39f5a69995ac40d236688f2e051f0a924 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 24 Jul 2026 19:20:59 +0000
Subject: [PATCH 10/11] Validate snapshot type and fall back on mismatch

A snapshot file with valid JSON but the wrong shape (e.g. a list where a dict
is expected) would previously be returned as-is and crash later. _read_snapshot
now takes the expected type and falls back to fetching when the parsed content
does not match. Add regression tests for both helpers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 scripts/data/generate_category_data.py        | 23 +++++++++++++++----
 .../data/test_generate_category_data.py       | 20 ++++++++++++++++
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index 54c95be837f..12c039f7009 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -61,33 +61,46 @@
 EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
 
 
-def _read_snapshot(hacs: AdjustedHacs, filename: str) -> object | None:
+def _read_snapshot(
+    hacs: AdjustedHacs,
+    filename: str,
+    expected_type: type,
+) -> object | None:
     """Read a snapshot file from ``EXISTING_DATA_DIR``.
 
     Returns the parsed JSON when available, or ``None`` (so the caller fetches
-    instead) when the dir is unset or the file is missing or unreadable.
+    instead) when the dir is unset, the file is missing or unreadable, or the
+    content is not of ``expected_type``.
     """
     if not EXISTING_DATA_DIR:
         return None
     try:
         with open(os.path.join(EXISTING_DATA_DIR, filename), encoding="utf-8") as file:
-            return json.load(file)
+            data = json.load(file)
     except (OSError, json.JSONDecodeError) as err:
         hacs.log.warning(
             "Could not read snapshot %s (%s), fetching instead", filename, err)
         return None
+    if not isinstance(data, expected_type):
+        hacs.log.warning(
+            "Snapshot %s has unexpected type %s, fetching instead",
+            filename,
+            type(data).__name__,
+        )
+        return None
+    return data
 
 
 async def get_stored_data(hacs: AdjustedHacs, category: str) -> dict[str, dict[str, Any]]:
     """Return existing category data from the snapshot dir when available, else fetch."""
-    if (data := _read_snapshot(hacs, f"{category}.json")) is not None:
+    if (data := _read_snapshot(hacs, f"{category}.json", dict)) is not None:
         return data
     return await hacs.data_client.get_data(category, validate=False)
 
 
 async def get_removed_repositories(hacs: AdjustedHacs) -> list[str]:
     """Return the removed-repositories list from the snapshot dir when available, else fetch."""
-    if (removed := _read_snapshot(hacs, "removed.json")) is not None:
+    if (removed := _read_snapshot(hacs, "removed.json", list)) is not None:
         return removed
     return await hacs.data_client.get_repositories("removed")
 
diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py
index da5f5ccea10..c43f54bc5f4 100644
--- a/tests/scripts/data/test_generate_category_data.py
+++ b/tests/scripts/data/test_generate_category_data.py
@@ -412,3 +412,23 @@ async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_pat
     hacs = _StubHacs()
     assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
     assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (list, not dict) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "plugin.json").write_text(json.dumps(["not", "a", "dict"]))
+
+    hacs = _StubHacs()
+    assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
+    assert hacs.data_client.calls == [("get_data", "plugin")]
+
+
+async def test_get_removed_repositories_falls_back_on_wrong_shape(tmp_path, monkeypatch):
+    """Valid JSON of the wrong type (dict, not list) falls back to fetching."""
+    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    (tmp_path / "removed.json").write_text(json.dumps({"not": "a list"}))
+
+    hacs = _StubHacs()
+    assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
+    assert hacs.data_client.calls == [("get_repositories", "removed")]

From e2bc91ade40687c7ffd421be4641e2955c1c26bf Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 24 Jul 2026 20:23:38 +0000
Subject: [PATCH 11/11] Address review: fix summarize, rename to stored, e2e
 test

- summarize: iterate only the expected category directories (needs
  generate-matrix + CATEGORIES), so the new stored-data artifact (and any
  non-category dir) can't crash the JSON.parse(summary.json) loop.
- Rename the existing-data snapshot to stored-data across the workflow, the
  HACS_STORED_DATA_DIR env var, and the script constant, for consistency with
  fetch-stored-data / get_stored_data.
- Comment the fetch-stored-data failure-isolation trade-off (one failed fetch
  fails the whole run, by design).
- _read_snapshot: generic return type so callers keep their dict/list types.
- Add an end-to-end test proving the stored-snapshot path reproduces the
  fetch-path output byte-for-byte; add timeout-minutes to the fetch job.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L22jYdNGF4hB4fvTYPTzP4
---
 .github/workflows/generate-hacs-data.yml      | 45 +++++++++-----
 scripts/data/generate_category_data.py        | 19 +++---
 .../data/test_generate_category_data.py       | 60 ++++++++++++++++---
 ...pshot-hacs-test-org-integration-basic.json | 19 ++++++
 4 files changed, 109 insertions(+), 34 deletions(-)
 create mode 100644 tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json

diff --git a/.github/workflows/generate-hacs-data.yml b/.github/workflows/generate-hacs-data.yml
index 2dbbbd77acf..5e8dbf12954 100644
--- a/.github/workflows/generate-hacs-data.yml
+++ b/.github/workflows/generate-hacs-data.yml
@@ -58,17 +58,25 @@ jobs:
           echo "categories=$categories_json" >> $GITHUB_OUTPUT
           echo "categories: $categories_json"
 
+  # Fetches the currently published data once and shares it with every
+  # category leg, so the whole run works off one consistent baseline.
+  # Footgun / trade-off: any single fetch failing (after the retries) fails
+  # this job and therefore the whole run -- so one category's data being
+  # unavailable blocks publishing for all of them. This is intentional (one
+  # consistent snapshot), unlike the previous behaviour where a fetch failure
+  # was isolated to that category's own leg.
   fetch-stored-data:
     name: Fetch stored data
     runs-on: ubuntu-latest
     needs: generate-matrix
     if: github.repository == 'hacs/integration'
+    timeout-minutes: 10
     steps:
-      - name: Fetch existing published data
+      - name: Fetch stored data
         env:
           CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
         run: |
-          mkdir -p outputdata/existing
+          mkdir -p outputdata/stored
 
           fetch_stored_hacs_content() {
             curl \
@@ -80,18 +88,18 @@ jobs:
           }
 
           for category in $(echo "$CATEGORIES" | jq -r '.[]'); do
-            echo "Fetching existing data for $category"
-            fetch_stored_hacs_content "${category}/data.json" "outputdata/existing/${category}.json"
+            echo "Fetching stored data for $category"
+            fetch_stored_hacs_content "${category}/data.json" "outputdata/stored/${category}.json"
           done
 
           echo "Fetching removed repositories"
-          fetch_stored_hacs_content "removed/repositories.json" "outputdata/existing/removed.json"
+          fetch_stored_hacs_content "removed/repositories.json" "outputdata/stored/removed.json"
 
-      - name: Upload existing data
+      - name: Upload stored data
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: existing-data
-          path: outputdata/existing
+          name: stored-data
+          path: outputdata/stored
           if-no-files-found: error
           retention-days: 1
 
@@ -126,18 +134,18 @@ jobs:
           scripts/install/frontend
           scripts/install/pip_packages --requirement requirements_generate_data.txt
 
-      - name: Download existing data
+      - name: Download stored data
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: existing-data
-          path: outputdata/existing
+          name: stored-data
+          path: outputdata/stored
 
       - name: Generate ${{ matrix.category }} data
         run: python3 -m scripts.data.generate_category_data ${{ matrix.category }}
         env:
           DATA_GENERATOR_TOKEN: ${{ secrets.DATA_GENERATOR_TOKEN }}
           FORCE_REPOSITORY_UPDATE: ${{ inputs.forceRepositoryUpdate }}
-          HACS_EXISTING_DATA_DIR: outputdata/existing
+          HACS_STORED_DATA_DIR: outputdata/stored
 
       - name: Validate output with JQ
         run: |
@@ -179,7 +187,7 @@ jobs:
   summarize:
     name: Summarize
     runs-on: ubuntu-latest
-    needs: category-data
+    needs: [generate-matrix, category-data]
     if: ${{ always() && github.repository == 'hacs/integration' }}
     outputs:
       changedCategories: ${{ steps.combined.outputs.changedCategories }}
@@ -196,6 +204,7 @@ jobs:
         env:
           HACS_CHANGED_PCT_TARGET: ${{ vars.HACS_CHANGED_PCT_TARGET }}
           HACS_DIFF_TARGET: ${{ vars.HACS_DIFF_TARGET }}
+          CATEGORIES: ${{ needs.generate-matrix.outputs.categories }}
         with:
           script: |
             const fs = require('fs');
@@ -206,9 +215,13 @@ jobs:
 
             core.info(`[global] diffTarget: ${diffTarget}`);
 
-            const subDirectories = fs.readdirSync("outputdata", { withFileTypes: true })
-                .filter(entry => entry.isDirectory())
-                .map(entry => entry.name)
+            // Only consider the expected category directories that produced a
+            // summary -- ignores the stored-data artifact and any category
+            // whose leg failed to write one.
+            const categories = JSON.parse(process.env.CATEGORIES);
+            const subDirectories = categories.filter(
+                category => fs.existsSync(`outputdata/${category}/summary.json`)
+            )
 
             for (const directory of subDirectories) {
               let changedPctTarget = Number(process.env.HACS_CHANGED_PCT_TARGET)
diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py
index 12c039f7009..ca04f65f252 100644
--- a/scripts/data/generate_category_data.py
+++ b/scripts/data/generate_category_data.py
@@ -58,28 +58,29 @@
 OUTPUT_DIR = os.path.join(os.getcwd(), "outputdata")
 COMPARE_IGNORE = {"etag_releases", "etag_repository", "last_fetched"}
 
-EXISTING_DATA_DIR = os.getenv("HACS_EXISTING_DATA_DIR")
+# Directory holding the preflight snapshot of the existing published data. When
+# set, snapshots are read from here instead of fetched; unset outside the workflow.
+STORED_DATA_DIR = os.getenv("HACS_STORED_DATA_DIR")
 
 
-def _read_snapshot(
+def _read_snapshot[T](
     hacs: AdjustedHacs,
     filename: str,
-    expected_type: type,
-) -> object | None:
-    """Read a snapshot file from ``EXISTING_DATA_DIR``.
+    expected_type: type[T],
+) -> T | None:
+    """Read a snapshot file from ``STORED_DATA_DIR``.
 
     Returns the parsed JSON when available, or ``None`` (so the caller fetches
     instead) when the dir is unset, the file is missing or unreadable, or the
     content is not of ``expected_type``.
     """
-    if not EXISTING_DATA_DIR:
+    if not STORED_DATA_DIR:
         return None
     try:
-        with open(os.path.join(EXISTING_DATA_DIR, filename), encoding="utf-8") as file:
+        with open(os.path.join(STORED_DATA_DIR, filename), encoding="utf-8") as file:
             data = json.load(file)
     except (OSError, json.JSONDecodeError) as err:
-        hacs.log.warning(
-            "Could not read snapshot %s (%s), fetching instead", filename, err)
+        hacs.log.warning("Could not read snapshot %s (%s), fetching instead", filename, err)
         return None
     if not isinstance(data, expected_type):
         hacs.log.warning(
diff --git a/tests/scripts/data/test_generate_category_data.py b/tests/scripts/data/test_generate_category_data.py
index c43f54bc5f4..c72f9d791c5 100644
--- a/tests/scripts/data/test_generate_category_data.py
+++ b/tests/scripts/data/test_generate_category_data.py
@@ -352,7 +352,7 @@ def __init__(self) -> None:
 
 async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
     """When the snapshot dir is set, stored data is read from it, not fetched."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     payload = {"1": {"full_name": "octocat/Hello-World"}}
     (tmp_path / "integration.json").write_text(json.dumps(payload))
 
@@ -363,7 +363,7 @@ async def test_get_stored_data_reads_from_existing_dir(tmp_path, monkeypatch):
 
 async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkeypatch):
     """When the snapshot dir is set, the removed list is read from it, not fetched."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     removed = ["octocat/Hello-World", "hacs/integration"]
     (tmp_path / "removed.json").write_text(json.dumps(removed))
 
@@ -374,7 +374,7 @@ async def test_get_removed_repositories_reads_from_existing_dir(tmp_path, monkey
 
 async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, stored data is fetched from the data client."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None)
     hacs = _StubHacs()
     assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
     assert hacs.data_client.calls == [("get_data", "plugin")]
@@ -382,7 +382,7 @@ async def test_get_stored_data_falls_back_to_fetch(monkeypatch):
 
 async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
     """Without the snapshot dir, the removed list is fetched from the data client."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", None)
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", None)
     hacs = _StubHacs()
     assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
     assert hacs.data_client.calls == [("get_repositories", "removed")]
@@ -390,7 +390,7 @@ async def test_get_removed_repositories_falls_back_to_fetch(monkeypatch):
 
 async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
     """A missing snapshot file falls back to fetching instead of raising."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     hacs = _StubHacs()
     assert await get_stored_data(hacs, "plugin") == _FETCHED_STORED
     assert hacs.data_client.calls == [("get_data", "plugin")]
@@ -398,7 +398,7 @@ async def test_get_stored_data_falls_back_when_snapshot_missing(tmp_path, monkey
 
 async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkeypatch):
     """An invalid-JSON snapshot file falls back to fetching instead of raising."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     (tmp_path / "plugin.json").write_text("{ not valid json")
 
     hacs = _StubHacs()
@@ -408,7 +408,7 @@ async def test_get_stored_data_falls_back_when_snapshot_invalid(tmp_path, monkey
 
 async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_path, monkeypatch):
     """A missing removed snapshot falls back to fetching instead of raising."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     hacs = _StubHacs()
     assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
     assert hacs.data_client.calls == [("get_repositories", "removed")]
@@ -416,7 +416,7 @@ async def test_get_removed_repositories_falls_back_when_snapshot_missing(tmp_pat
 
 async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch):
     """Valid JSON of the wrong type (list, not dict) falls back to fetching."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     (tmp_path / "plugin.json").write_text(json.dumps(["not", "a", "dict"]))
 
     hacs = _StubHacs()
@@ -426,9 +426,51 @@ async def test_get_stored_data_falls_back_on_wrong_shape(tmp_path, monkeypatch):
 
 async def test_get_removed_repositories_falls_back_on_wrong_shape(tmp_path, monkeypatch):
     """Valid JSON of the wrong type (dict, not list) falls back to fetching."""
-    monkeypatch.setattr(f"{_MODULE}.EXISTING_DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(tmp_path))
     (tmp_path / "removed.json").write_text(json.dumps({"not": "a list"}))
 
     hacs = _StubHacs()
     assert await get_removed_repositories(hacs) == _FETCHED_REMOVED
     assert hacs.data_client.calls == [("get_repositories", "removed")]
+
+
+@pytest.mark.parametrize(
+    "category_test_data", category_test_data_parametrized(categories=["integration"])
+)
+async def test_generate_category_data_from_stored_snapshot(
+    hass: HomeAssistant,
+    response_mocker: ResponseMocker,
+    snapshots: SnapshotFixture,
+    category_test_data: CategoryTestData,
+    tmp_path,
+    monkeypatch,
+):
+    """Generating from the stored snapshot reproduces the fetch-path output.
+
+    Feeds the same inputs the fetch path uses (empty category data + the removed
+    fixture) through STORED_DATA_DIR, and asserts the generated data.json matches
+    the fetch-path snapshot -- i.e. sourcing the baseline from disk is equivalent
+    to fetching it.
+    """
+    category = category_test_data["category"]
+
+    stored_dir = tmp_path / "stored"
+    stored_dir.mkdir()
+    (stored_dir / f"{category}.json").write_text("{}")
+    with open(
+        os.path.join(
+            FIXTURES_PATH, "proxy", "data-v2.hacs.xyz", "removed", "repositories.json"
+        ),
+        encoding="utf-8",
+    ) as file:
+        (stored_dir / "removed.json").write_text(file.read())
+    monkeypatch.setattr(f"{_MODULE}.STORED_DATA_DIR", str(stored_dir))
+
+    await generate_category_data(category)
+
+    with open(f"{OUTPUT_DIR}/{category}/data.json", encoding="utf-8") as file:
+        snapshots.assert_match(
+            safe_json_dumps(recursive_remove_key(
+                json.loads(file.read()), ("last_fetched",))),
+            f"scripts/data/generate_category_data/{category}//data.json",
+        )
diff --git a/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json b/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json
new file mode 100644
index 00000000000..8c0b57fadc8
--- /dev/null
+++ b/tests/snapshots/api-usage/tests/scripts/data/test_generate_category_datatest-generate-category-data-from-stored-snapshot-hacs-test-org-integration-basic.json
@@ -0,0 +1,19 @@
+{
+    "tests/scripts/data/test_generate_category_data.py::test_generate_category_data_from_stored_snapshot[hacs-test-org/integration-basic]": {
+        "https://api.github.com/rate_limit": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/custom_components/example/manifest.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/hacs.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/git/trees/1.0.0": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/releases": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/contents/custom_components/example/manifest.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/contents/hacs.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/git/trees/1.0.0": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/releases": 1,
+        "https://api.github.com/repos/hacs/default/contents/integration": 1,
+        "https://api.github.com/repos/hacs/integration": 1,
+        "https://api.github.com/repos/hacs/integration/branches/main": 1,
+        "https://api.github.com/repos/hacs/integration/releases": 1
+    }
+}
\ No newline at end of file