ci: enforce schema migration and Web quality gates

.github/workflows/ci.yml

@@ -64,6 +64,10 @@ jobs:
           key: retrom-postgresql-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('scripts/prepare-postgres.sh') }}
       - name: Check workspace, structure, API and data contracts
         run: make ci-contracts
+      - name: Check migrations against the latest published stable release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: make migration-check-released
 
   backend:
     runs-on: ubuntu-latest
@@ -207,6 +211,14 @@ jobs:
           key: retrom-node-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.node-version', 'scripts/prepare-node.sh') }}
       - name: Check Web styles, lint, types, tests and build
         run: make web-check
+      - name: Upload Web duplication report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: web-duplication
+          path: .artifacts/duplicates/web/
+          include-hidden-files: true
+          if-no-files-found: ignore
 
   browser_ui:
     name: browser-ui (${{ matrix.project }})
@@ -309,9 +321,20 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  pfb_gateway:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - name: Verify isolated PFB gateway routing
+        run: make pfb-gateway-test
+
   quality:
     runs-on: ubuntu-latest
-    needs: [changes, contracts, backend, integration, web, browser_ui]
+    needs: [changes, contracts, backend, integration, web, browser_ui, pfb_gateway]
     if: ${{ always() }}
     steps:
       - name: Require every quality check to pass
@@ -323,12 +346,13 @@ jobs:
           INTEGRATION_RESULT: ${{ needs.integration.result }}
           WEB_RESULT: ${{ needs.web.result }}
           BROWSER_UI_RESULT: ${{ needs.browser_ui.result }}
+          PFB_GATEWAY_RESULT: ${{ needs.pfb_gateway.result }}
         run: |
           # Only a documentation-only change may skip the product browser.
           if [[ "$PRODUCT_CHANGED" == false && "$BROWSER_UI_RESULT" == skipped ]]; then
             BROWSER_UI_RESULT=success
           fi
-          for result in "$CHANGES_RESULT" "$CONTRACTS_RESULT" "$BACKEND_RESULT" "$INTEGRATION_RESULT" "$WEB_RESULT" "$BROWSER_UI_RESULT"; do
+          for result in "$CHANGES_RESULT" "$CONTRACTS_RESULT" "$BACKEND_RESULT" "$INTEGRATION_RESULT" "$WEB_RESULT" "$BROWSER_UI_RESULT" "$PFB_GATEWAY_RESULT"; do
             if [[ "$result" != success ]]; then
               echo "Quality check did not pass: $result" >&2
               exit 1

.github/workflows/docker-image.yml

@@ -22,6 +22,8 @@ jobs:
     steps:
       - name: Check out the repository
         uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
       - name: Validate the image tag
         id: release
         env:
@@ -61,6 +63,15 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.13"
+      - name: Prepare the release migration manifest
+        env:
+          IMAGE_TAG: ${{ steps.release.outputs.image_tag }}
+        run: make migration-manifest TAG="$IMAGE_TAG"
+      - name: Require stable image sources to be on master
+        if: steps.release.outputs.publish_latest == 'true'
+        run: |
+          git fetch --no-tags origin '+refs/heads/master:refs/remotes/origin/master'
+          git merge-base --is-ancestor "$GITHUB_SHA" origin/master
       - name: Prepare pinned password blocklist
         run: python3 scripts/dependencies.py prepare-auth --versions 4.2.3,4.3.0-pre
       - name: Restore verified Provider chunks
@@ -101,3 +112,10 @@ jobs:
           docker tag "$WEB_IMAGE:$IMAGE_TAG" "$WEB_IMAGE:latest"
           docker push "$BACKEND_IMAGE:latest"
           docker push "$WEB_IMAGE:latest"
+      - name: Record the released migration manifest
+        uses: actions/upload-artifact@v4
+        with:
+          name: released-migrations-${{ steps.release.outputs.image_tag }}
+          path: .artifacts/release/migrations.json
+          include-hidden-files: true
+          if-no-files-found: error

Makefile

@@ -113,6 +113,8 @@ test: prepare-go api-generate-go
 
 lint-go: api-generate-go install-golangci-lint
 	@python3 scripts/test_architecture_rules.py
+	@go test ./scripts/sql-boundary ./scripts/lint-policy
+	@go run ./scripts/sql-boundary
 	@bin/golangci-lint run --allow-serial-runners --build-tags integration $(GO_PACKAGES)
 
 backend-check: database-isolation-check quality-structure-check fmt-check build test lint-go
@@ -147,6 +149,8 @@ web-ui-check: prepare-node
 	@cd web && "$(NODE_HOME)/bin/node" scripts/check-ui-styles.mjs
 
 web-lint: prepare-node web-ui-check
+	@cd web && "$(NODE_HOME)/bin/node" --test scripts/lint-policy.test.mjs scripts/duplicates.test.mjs
+	@cd web && $(NPM) run check:duplicates
 	@cd web && $(NPM) run lint
 
 web-typecheck: prepare-node
@@ -200,6 +204,9 @@ data-check:
 	@python3 -m unittest scripts.test_pfb_upstream_sync
 	@python3 -m unittest scripts.test_pfb_init
 	@python3 -m unittest scripts.test_pfb_data_reset
+	@python3 scripts/test_pfb_runtime_tool.py
+	@python3 scripts/test_pfb_server_data.py
+	@python3 scripts/test_runtime_origin_topology.py
 	@python3 scripts/test_prepare_toolchains.py
 	@python3 scripts/test_makefile.py
 	@python3 scripts/test_workflows.py
@@ -257,9 +264,24 @@ workspace-check:
 	@python3 workspace/catalog.py
 	@python3 -m unittest discover -s workspace -p 'test_*.py'
 
-ci-contracts: workspace-check database-isolation-check quality-structure-check api-check data-check test-local-postgres
+.PHONY: migration-check migration-check-released migration-manifest pfb-gateway-test
+migration-check:
+	@python3 scripts/test_migrations.py
+	@python3 scripts/migrations.py check
 
-ci: workspace-check database-isolation-check quality-structure-check api-check backend-check web-check integration-test data-check test-local-postgres
+migration-check-released:
+	@python3 scripts/migrations.py check --latest-release
+
+migration-manifest:
+	@test -n "$(TAG)" || { echo 'TAG is required' >&2; exit 2; }
+	@python3 scripts/migrations.py snapshot --release-tag "$(TAG)" --output .artifacts/release/migrations.json
+
+pfb-gateway-test:
+	@python3 scripts/test_pfb_gateway.py
+
+ci-contracts: workspace-check database-isolation-check quality-structure-check migration-check api-check data-check test-local-postgres
+
+ci: workspace-check database-isolation-check quality-structure-check migration-check api-check backend-check web-check integration-test data-check test-local-postgres
 
 require-local-user:
 	@python3 scripts/local_user.py

docs/data-model.md

@@ -2,6 +2,10 @@
 
 目标 schema 由 migrations 按编号顺序叠加得到：001_schema.sql 是完整基线（含项目包成员表、标题字母序列和活动槽列），后续变化从 002 起追加。新库和既有库执行同一序列，新库不跳过后续 migration。已发布的 migration 文件固化不再修改，schema 变化只能追加新编号文件。迁移不读取或重打包游戏字节。应用数据库包含以下20张业务表和schema_migrations_tab技术账本，没有FK、CHECK、用户TRIGGER、VIEW、materialized VIEW、DOMAIN或业务函数，也没有 partial index（索引不带 WHERE 谓词，新增状态值不需要重建索引）。
 
+`migrations.All()` 自动嵌入并按文件名排列全部顶层 SQL，要求 `NNN_lowercase.sql` 从 001 连续、唯一且非空；启动时在同一加锁事务内逐项应用，已登记的 ID 跳过。新库集成测试核对每个嵌入 ID 已登记，账本数量随实际序列变化。
+
+`quality/released-migrations.json` 提供离线已发布基线；首次基线来自 v0.0.158 的真实 tag commit。运行 `make migration-check` 检查发布文件的字节摘要、删除和重命名，未发布的新文件可继续原位编辑。PR CI 通过 `make migration-check-released` 重新读取最新已发布 GitHub stable release 的 tag 源码，不以分支提交当作发布。tag 镜像流程用 `make migration-manifest TAG=<tag>` 生成 `.artifacts/release/migrations.json` 并保存为 Actions artifact；历史比对和回归测试在 PR 阶段完成。正式发布后可用 `python3 scripts/migrations.py snapshot --ref <tag> --release-tag <tag> --output quality/released-migrations.json` 在后续 PR 更新离线基线；PR 联网门禁独立核对最新发布源码。
+
 “同一键只有一条活动记录”由可空槽列加普通唯一索引表达：活动行的 active_slot 为 1，退休行为 NULL；PostgreSQL 唯一索引中 NULL 互不相等，因此退休行可以任意多条。唯一索引为 game_file_tab(game_id,logical_key,active_slot)、game_media_tab(game_id,kind,ordinal,active_slot)、bios_file_tab(requirement_key,active_slot)、tag_tab(name_key,active_slot)；项目包的 package_slot 仅在活动 project_package 行为 1，唯一索引为 game_file_tab(game_id,package_slot)。插入活动行和改为 deleted 的同一条语句同步写入槽列，BIOS 扫描安装以 ON CONFLICT(requirement_key,active_slot) 保持手工安装不被覆盖。
 
 | 表 | 当前事实 |

docs/engineering-quality-and-testing.md

@@ -14,6 +14,16 @@
 
 make backend-check执行数据库隔离上限、结构门禁、格式、build、普通tests和全部Go lint。depguard检查HTTP不能越过service读取持久层、基础层不反依赖业务、runtime/存档不依赖扫描，真实可编译的违规样例必须被门禁拒绝。make integration-test需要显式RETROM_TEST_DATABASE_URL，每例建独立DB迁移并销毁，不skip、不污染产品库。make api-check重生成同一authority对比TS；make web-check与runtime自身lint/type/test保留原强度。
 
+`lint-go` 还执行 service SQL 类型边界：按实际 `persistence.Repository` 方法识别 `Count`/`Execute`，覆盖别名、方法值和方法表达式，允许命名领域操作及其他类型的同名方法。Go/Web 配置回归核对现有 lint、复杂度、行数及忽略范围；规则调整与相应回归通过普通 PR 审阅。
+
+Web 的 `make web-lint` 运行 `cd web && npm run check:duplicates`，使用锁定的 jscpd 5.4.2 检查 JS/JSX/TS/TSX，覆盖整个 web 目录及新路径，排除测试、声明文件、生成 API 和构建/依赖目录。检测门槛为 120 token 且至少 10 行，忽略注释和空白，按一致的标识符重命名识别复制，跨 JS/TS 格式比较。`.jscpd-baseline.json` 首次为空；新增重复或对已记录重复再增加副本都会使检查失败。配置和探测器回归验证正例、改名/同文件/跨文件反例、生成与测试豁免及空扫描拒绝。
+
+PR web job 在成功/失败后上传 `.artifacts/duplicates/web/` 的 JSON 报告，含文件与行号。检测不新增 CI job 或 release 步骤。jscpd 的[原生基线](https://github.com/kucherenko/jscpd/blob/master/docs/rust.md#baseline)按内容指纹和副本数量记录已有重复；确需接受新重复时，在明确说明原因的 PR 中运行 `cd web && npm run check:duplicates -- --update-baseline`，将基线差异一起审阅，日常检查不会自动更新基线。
+
+PG integration 在完整应用 migration 后检查应用 schema、精确业务表集合、技术账本、禁用对象和目录/核心关系空种子，并逐项注入违规对象验证拒绝。它检查最终数据库结构。`make migration-check` 离线检查已发布 SQL 摘要和连续编号；PR CI contracts 另执行 `make migration-check-released`，从 GitHub 最新已发布 stable release 的 tag 源码计算摘要，避免本地清单落后漏检。migration 的历史比对及其回归测试在 PR 阶段执行，tag 镜像流程生成当前 SQL 摘要作为发布产物记录。详见 [数据模型](data-model.md)。
+
+PR 的 `quality` 聚合 contracts、backend、integration、web、browser-ui 和独立 `pfb_gateway`。网关使用临时 Docker 网络和容器验证分流及隔离，结束后清理自有资源；`data-check` 同时包含 PFB runtime tool、server-data、runtime origin topology 回归。`branch-image` 的 `build` 验证双应用镜像及部署连接。镜像 tag 发布保留原有构建与部署验证，stable tag 额外要求源码已在 master，并记录 migration 摘要为 Actions artifact。
+
 风险集成用真实PostgreSQL覆盖：Game/文件写入后失败与progress一起回滚、commit响应丢失按分配ID核对；两save同版本竞争仅一胜、同commit重试不增版本、Game删除后保存拒绝；BIOS自动安装遇并发手动安装只skip；真实文件移除失败保留deleted，active引用保护字节。纯回归覆盖可信代理链/伪造XFF、严格JSON、UUID和编码斜杠、目录有界迭代与并发消失、共享常驻runtime并发关联、单请求取消、异常退出后的恢复。
 
 Playwright通过正常HTTP建立合法MIT NES来源、扫描、共享审核、运行、保存和新实例恢复，页面无error/溢出；保留桌面/移动/4K视口。浏览器CI必须提供同版runtime工具/Provider明确输入，不能用旧正式release冒充新契约。私有素材验收只在授权目录读取并复制，证据和素材不提交。

internal/persistence/database.go

@@ -47,6 +47,10 @@ func (r *Repository) Close() {
 }
 
 func (r *Repository) Migrate(ctx context.Context) error {
+	ordered, err := migrations.All()
+	if err != nil {
+		return fmt.Errorf("load migrations: %w", err)
+	}
 	tx, err := r.pool.Begin(ctx)
 	if err != nil {
 		return fmt.Errorf("begin migration: %w", err)
@@ -58,10 +62,8 @@ func (r *Repository) Migrate(ctx context.Context) error {
 	if _, err = tx.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations_tab (id TEXT PRIMARY KEY)"); err != nil {
 		return fmt.Errorf("create migration bookkeeping: %w", err)
 	}
-	for _, migration := range []struct{ id, schema string }{
-		{"001_schema", migrations.Schema},
-	} {
-		if err = applyMigration(ctx, tx, migration.id, migration.schema); err != nil {
+	for _, migration := range ordered {
+		if err = applyMigration(ctx, tx, migration.ID, migration.SQL); err != nil {
 			return err
 		}
 	}

internal/persistence/schema_integration_test.go

@@ -21,41 +21,8 @@ func TestSchemaAndSessionLifecycle(t *testing.T) {
 
 func assertEmptyApplicationSchema(t *testing.T, repository *persistence.Repository) {
 	t.Helper()
-	ctx := t.Context()
-	tableCount, err := repository.Count(ctx, `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
- WHERE n.nspname='public' AND c.relkind='r' AND c.relname LIKE '%\_tab' ESCAPE '\'
- AND c.relname <> 'schema_migrations_tab'`)
-	if err != nil || tableCount != 20 {
-		t.Fatalf("tables=%d err=%v", tableCount, err)
-	}
-	unsuffixed, err := repository.Count(ctx, `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
- WHERE n.nspname='public' AND c.relkind='r' AND c.relname NOT LIKE '%\_tab' ESCAPE '\'`)
-	if err != nil || unsuffixed != 0 {
-		t.Fatalf("unsuffixed tables=%d err=%v", unsuffixed, err)
-	}
-	ledger, err := repository.Count(ctx, "SELECT count(*) FROM schema_migrations_tab")
-	if err != nil || ledger != 1 {
-		t.Fatalf("applied migrations=%d err=%v", ledger, err)
-	}
-	prohibited,
-		err := repository.Count(ctx,
-		`SELECT count(*) FROM pg_constraint k JOIN pg_namespace n ON n.oid=k.connamespace
- WHERE n.nspname='public' AND k.contype IN ('c','f')`)
-	if err != nil || prohibited != 0 {
-		t.Fatalf("constraints=%d err=%v", prohibited, err)
-	}
-	// Partial indexes bake a status list into the schema; one-active rules use active_slot columns instead.
-	partial, err := repository.Count(ctx, `SELECT count(*) FROM pg_index x JOIN pg_class c ON c.oid=x.indexrelid
- JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND x.indpred IS NOT NULL`)
-	if err != nil || partial != 0 {
-		t.Fatalf("partial indexes=%d err=%v", partial, err)
-	}
-	views, err := repository.Count(ctx, `SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
- WHERE n.nspname='public' AND c.relkind IN ('v','m')`)
-	if err != nil || views != 0 {
-		t.Fatalf("views=%d err=%v", views, err)
-	}
-	directories, err := repository.Directories(ctx, true)
+	assertSchemaPolicy(t, repository)
+	directories, err := repository.Directories(t.Context(), true)
 	if err != nil || len(directories) != 0 {
 		t.Fatalf("directories=%v err=%v", directories, err)
 	}