build(scraper): migrate to Bun runtime and dynamic image versioning

Bun executes TypeScript natively, eliminating the ncc compilation step
and the pnpm dependency. The Containerfile is now a lean two-stage
build (deps + production) that runs src/main.ts directly via bun,
and uses --filter + --production to avoid pulling in the ~100 AWS SDK
packages that belong to packages/cli.

The scraper image version was previously a static Terraform variable
requiring a manual commit to bump. It is now built fresh in CI at the
same git_ref as the website, so the scraper always runs at the exact
commit being deployed. The scrape-and-index Argo task receives the
image tag as a runtime parameter instead of having it baked into the
Terraform config, removing the risk of version drift between site
content and the scraper that indexes it.

Author: fullykubed

.dockerignore

@@ -17,8 +17,6 @@
 !packages/reference-typescript/src
 !packages/reference-typescript/package.json
 !packages/reference-typescript/tsconfig.json
-!packages/eslint/package.json
-!packages/eslint/index.js
 !packages/bastion
 !.npmrc
 !package.json

bun.lock

@@ -38,7 +38,6 @@ inputs = {
   mongodb_atlas_public_key  = local.secrets.mongodb_atlas_public_key
   mongodb_atlas_private_key = local.secrets.mongodb_atlas_private_key
   site_url                  = "https://panfactum.com"
-  scraper_image_version     = "586d06cee96633a26ffe0ce318d85f26c3f7c27d"
   module_bucket             = dependency.module_bucket.outputs.bucket_name
   installer_bucket          = dependency.installer_bucket.outputs.installer_bucket_name
 }

environments/production/us-east-2/demo_cicd/terragrunt.hcl

@@ -33,6 +33,11 @@ module "run_scraper_workflow_spec" {
       {
         name        = "algolia_index_name"
         description = "The index name in algolia to update"
+      },
+
+      {
+        name        = "image_version"
+        description = "The scraper image tag to run (typically the git commit hash built by scraper-builder)"
       }
     ]
   }
@@ -53,19 +58,42 @@ module "run_scraper_workflow_spec" {
   templates = [
     {
       name = "scrape-and-index",
+      inputs = {
+        parameters = [
+          {
+            name = "image_version"
+          }
+        ]
+      }
       container = {
-        image   = "891377197483.dkr.ecr.us-east-2.amazonaws.com/scraper:${var.scraper_image_version}"
-        command = ["node"]
-        args    = ["index.js", "{{workflow.parameters.sitemap_url}}", "{{workflow.parameters.algolia_index_name}}"]
+        image   = "891377197483.dkr.ecr.us-east-2.amazonaws.com/scraper:{{inputs.parameters.image_version}}"
+        command = ["bun"]
+        args    = ["run", "src/main.ts", "{{workflow.parameters.sitemap_url}}", "{{workflow.parameters.algolia_index_name}}"]
       }
     },
     {
       name = "entry",
+      inputs = {
+        parameters = [
+          {
+            name    = "image_version"
+            default = "{{workflow.parameters.image_version}}"
+          }
+        ]
+      }
       dag = {
         tasks = [
           {
-            name = "scrape-and-index"
-            template : "scrape-and-index"
+            name     = "scrape-and-index"
+            template = "scrape-and-index"
+            arguments = {
+              parameters = [
+                {
+                  name  = "image_version"
+                  value = "{{inputs.parameters.image_version}}"
+                }
+              ]
+            }
           }
         ]
       }

infrastructure/demo_cicd/scraper_runner.tf

@@ -36,11 +36,6 @@ variable "pull_through_cache_enabled" {
   default     = false
 }
 
-variable "scraper_image_version" {
-  description = "The version of the image to use for the scraper"
-  type        = string
-}
-
 variable "algolia_app_id" {
   description = "The Algolia App ID for the search index"
   type        = string

infrastructure/demo_cicd/vars.tf

@@ -161,6 +161,28 @@ module "astro_builder_workflow" {
             template = "build-images"
           },
 
+          # Build the scraper image at the same commit as the website so the
+          # scrape-and-index step always runs the latest code. The
+          # scraper-builder's clone.sh resolves git_ref to a commit hash and
+          # tags the ECR image with it, so git_ref must be a commit hash for
+          # the downstream scrape-and-index task to find the matching tag.
+          # The sensor trigger passes body.after (a commit hash) for this.
+          {
+            name = "build-scraper"
+            templateRef = {
+              name     = module.scraper_builder.name
+              template = module.scraper_builder.entrypoint
+            }
+            arguments = {
+              parameters = [
+                {
+                  name  = "git_ref"
+                  value = "{{workflow.parameters.git_ref}}"
+                }
+              ]
+            }
+          },
+
           {
             name = "scrape-and-index"
             templateRef = {
@@ -176,10 +198,14 @@ module "astro_builder_workflow" {
                 {
                   name  = "algolia_index_name"
                   value = "{{workflow.parameters.algolia_index_name}}"
+                },
+                {
+                  name  = "image_version"
+                  value = "{{workflow.parameters.git_ref}}"
                 }
               ]
             }
-            depends = "build-images"
+            depends = "build-images && build-scraper"
           }
         ]
       }

infrastructure/demo_cicd/website_astro_builder.tf

@@ -12,5 +12,5 @@
     "typescript": "5.6.2",
     "yaml": "2.7.1"
   },
-  "workspaces": ["packages/cli"]
+  "workspaces": ["packages/cli", "packages/scraper"]
 }

package.json

@@ -1,47 +1,43 @@
 #################################################
 ## Base Image
 #################################################
-FROM node:20-bullseye-slim AS base
+FROM oven/bun:1.2.5-slim AS base
 WORKDIR /code
 
 #################################################
 ## Dependencies Image
 #################################################
 FROM base AS deps
 
-# Install package manager
-RUN npm i -g pnpm@9.14.2
-
-# Install Dependencies
-COPY package.json pnpm-lock.yaml .npmrc pnpm-workspace.yaml .
+# Workspace lockfile and members live at the repo root.
+# Copy the least-changing files first so layer caching survives source edits.
+# bun requires every workspace member's package.json to be present at install
+# time even when filtering, otherwise it errors with "Workspace not found".
+COPY package.json bun.lock ./
+COPY packages/cli/package.json packages/cli/
 COPY packages/scraper/package.json packages/scraper/
-COPY packages/eslint/ packages/eslint/
-RUN --mount=type=cache,id=pnpm,target=/code/.pnpm pnpm install --frozen-lockfile
-
-#################################################
-## Development Image
-#################################################
-
-FROM deps AS development
-WORKDIR /code/packages/scraper
-COPY packages/scraper/ .
-RUN npx ncc build src/main.ts -o dist
-
-#################################################
-## Builder Image
-#################################################
 
-FROM deps AS builder
-ENV NODE_ENV=production
-WORKDIR /code/packages/scraper
-COPY packages/scraper/ .
-RUN npx ncc build src/main.ts -o dist
+# --filter scrape installs only the scraper workspace's deps (skips the
+# 100+ AWS SDK packages that belong to packages/cli). --production drops
+# devDependencies (typescript, @types/node) since bun runs TypeScript natively.
+# The cache mount preserves bun's tarball cache across builds via the BuildKit
+# S3 cache backend, so reinstalls are essentially free.
+RUN --mount=type=cache,id=bun,target=/root/.bun/install/cache \
+    bun install --frozen-lockfile --filter scrape --production
 
 #################################################
 ## Production Image
 #################################################
-
 FROM base AS production
 ENV NODE_ENV=production
+
+# COPY --link makes each layer content-addressable so they can be cached
+# independently of preceding layers. Layers are listed most-stable first
+# so source edits only invalidate the final layer.
+COPY --link --from=deps /code/node_modules ./node_modules
+COPY --link package.json ./
+COPY --link packages/scraper/package.json packages/scraper/tsconfig.json packages/scraper/
+COPY --link packages/scraper/src packages/scraper/src
+
 WORKDIR /code/packages/scraper
-COPY --from=builder /code/packages/scraper/dist ./
+CMD ["bun", "run", "src/main.ts"]

packages/scraper/Containerfile

@@ -0,0 +1,25 @@
+# Per-Dockerfile dockerignore for the scraper image build.
+#
+# BuildKit loads this file from the --local dockerfile (packages/scraper/)
+# and applies it to the --local context (repo root). It takes precedence
+# over the root .dockerignore for this Containerfile only.
+#
+# We use an allowlist (deny-everything-then-allow) to keep the build
+# context as small as possible. Without this, the context includes the
+# 77MB packages/website/ tree which the scraper does not need.
+
+*
+
+# Root files needed for the bun workspace install
+!bun.lock
+!package.json
+
+# All workspace members listed in root package.json must have a
+# package.json present at install time, even when filtering, otherwise
+# bun errors with "Workspace not found".
+!packages/cli/package.json
+
+# The actual scraper sources
+!packages/scraper/package.json
+!packages/scraper/tsconfig.json
+!packages/scraper/src

packages/scraper/Containerfile.dockerignore

@@ -3,17 +3,15 @@
   "type": "module",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "main": "src/main.ts",
   "keywords": [],
   "author": "",
   "license": "AGPL",
   "scripts": {
-    "dev": "tsx src/main.ts"
+    "dev": "bun run src/main.ts"
   },
   "devDependencies": {
     "@types/node": "18.13.0",
-    "@vercel/ncc": "0.38.2",
-    "tsx": "4.19.1",
     "typescript": "5.6.2"
   },
   "dependencies": {