diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 0000000..d02619c --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,112 @@ +name: Biweekly changelog + +# Rolls merged PRs into CHANGELOG.md on a ~biweekly cadence and opens a PR — never pushes to +# main, never merges itself. A human reviews and merges it like any other PR (see +# CONTRIBUTING.md / SCAN.md). Purely mechanical: reads `gh pr list`, no model calls. + +on: + schedule: + - cron: "0 14 1,15 * *" # ~biweekly: 1st and 15th of each month, 14:00 UTC + workflow_dispatch: + inputs: + since: + description: "Override the start date (YYYY-MM-DD) instead of reading it from CHANGELOG.md" + required: false + default: "" + +permissions: + contents: write + pull-requests: write + +concurrency: + group: awesome-evals-changelog + cancel-in-progress: false + +jobs: + changelog: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Roll up merged PRs since the last entry + id: rollup + env: + GH_TOKEN: ${{ github.token }} + SINCE_OVERRIDE: ${{ github.event.inputs.since }} + run: | + set -euo pipefail + + # The cursor is the most recent dated header already in CHANGELOG.md (## [YYYY-MM-DD]), + # skipping the standing ## [Unreleased] section — so this is idempotent even if the repo + # went quiet for a cycle (a rerun just produces an empty-but-valid window, see below). + since="${SINCE_OVERRIDE:-$(grep -oE '^## \[[0-9]{4}-[0-9]{2}-[0-9]{2}\]' CHANGELOG.md | head -1 | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}')}" + if [ -z "$since" ]; then + echo "::error::No prior dated entry found in CHANGELOG.md and no --since override given." + exit 1 + fi + today="$(date -u +%F)" + echo "since=$since" >> "$GITHUB_OUTPUT" + echo "today=$today" >> "$GITHUB_OUTPUT" + + prs=$(gh pr list -R "$GITHUB_REPOSITORY" --state merged --limit 200 \ + --json number,title,mergedAt,url \ + --jq "[.[] | select(.mergedAt > \"${since}T00:00:00Z\")] | sort_by(.mergedAt)") + count=$(echo "$prs" | jq 'length') + echo "count=$count" >> "$GITHUB_OUTPUT" + if [ "$count" -eq 0 ]; then + echo "No PRs merged since $since — nothing to roll up." + exit 0 + fi + + # Cheap keyword classification from PR titles (this repo's own commit-message convention + # makes "Correct"/"Fix"/"Remove"/"Prune" reliable signals). Anything else defaults to Added, + # which is correct for the overwhelming majority of PRs here (new list entries). + added=$(echo "$prs" | jq -r '.[] | select(.title | test("^(Correct|Fix|Remove|Prune|Bump|Bypass|Harden)"; "i") | not) | "- \(.title) (#\(.number))"') + changed=$(echo "$prs" | jq -r '.[] | select(.title | test("^(Correct|Bump|Bypass|Harden)"; "i")) | "- \(.title) (#\(.number))"') + fixed=$(echo "$prs" | jq -r '.[] | select(.title | test("^Fix"; "i")) | "- \(.title) (#\(.number))"') + removed=$(echo "$prs" | jq -r '.[] | select(.title | test("^(Remove|Prune)"; "i")) | "- \(.title) (#\(.number))"') + + { + echo "## [$today]" + [ -n "$added" ] && { echo "### Added"; echo "$added"; } + [ -n "$changed" ] && { echo "### Changed"; echo "$changed"; } + [ -n "$fixed" ] && { echo "### Fixed"; echo "$fixed"; } + [ -n "$removed" ] && { echo "### Removed"; echo "$removed"; } + echo + } > /tmp/new-entry.md + + cat /tmp/new-entry.md + + - name: Insert entry into CHANGELOG.md + if: steps.rollup.outputs.count != '0' + run: | + set -euo pipefail + # Insert right after the "## [Unreleased]" line (which always stays empty/at the top). + awk ' + /^## \[Unreleased\]/ { print; getline; print; while ((getline line < "/tmp/new-entry.md") > 0) print line; next } + { print } + ' CHANGELOG.md > /tmp/CHANGELOG.md.new + mv /tmp/CHANGELOG.md.new CHANGELOG.md + + - name: Open the release PR + if: steps.rollup.outputs.count != '0' + env: + GH_TOKEN: ${{ github.token }} + SINCE: ${{ steps.rollup.outputs.since }} + TODAY: ${{ steps.rollup.outputs.today }} + COUNT: ${{ steps.rollup.outputs.count }} + run: | + set -euo pipefail + branch="changelog/${TODAY}" + git config user.name "awesome-evals-changelog" + git config user.email "actions@github.com" + git checkout -b "$branch" + git add CHANGELOG.md + git commit -m "Changelog: ${SINCE}..${TODAY} (${COUNT} merged PRs)" + git push -u origin "$branch" + gh pr create -R "$GITHUB_REPOSITORY" --base main --head "$branch" \ + --title "Changelog: ${SINCE}..${TODAY}" \ + --body "Mechanical rollup of the ${COUNT} PR(s) merged since ${SINCE}. Review the categorization (title-keyword heuristic — reclassify by hand if a PR was miscategorized) and merge like any other PR." diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..514dc33 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ +# Changelog + +All notable additions, corrections, and removals to this list. Format loosely follows +[Keep a Changelog](https://keepachangelog.com/); since this is a curated list rather than +software, entries are grouped by **date of the release PR**, not semver. + +Cut roughly **biweekly** by [`.github/workflows/changelog.yml`](.github/workflows/changelog.yml), +which opens a PR rolling up everything merged since the last entry — a human reviews and merges +it like any other PR (see [CONTRIBUTING.md](CONTRIBUTING.md); the bot never pushes to `main`). + +## [Unreleased] + +## [2026-08-20] +### Added +- Petri (Meridian Labs / UK AISI / Anthropic) — automated behavioral-auditing agent, §10 +- Caliper — pass@k reliability harness for agent skills, §5a +- Coder Eval (UiPath) — §5a +- PerspectiveGap, Open Multi-Agent evaluation, Dr. Bench, RewardHarness, ClawBench — §9 benchmarks +- TAC (Travel Agent Compassion), Manager Coercion Benchmark (MCB) — §9 multi-agent benchmarks +- Scan 2026-07-31: 8 new eval finds (#69) +### Changed +- Corrected stale annotations (Harbor star count, note count) and wrote down the enforced + curation bar explicitly into CONTRIBUTING.md (#67) +- Made the daily scan's "no new items" result independently verifiable via a tool-call audit + trail and run-start-gated PR lookup (#66) + +## [2026-07-30] +### Added +- Doubleword open-source-LLM release-forecasting blog — §6 (benchmark vs. eval) +- whatbroke — §5f (observability + eval platforms) +- truescore, Tura coding-agent cost study +- Two benchmark-auditing resources ("How We Broke Top AI Agent Benchmarks", RewardHackingAgents) — §6/§10 +- Scan 2026-06-29: 6 new eval finds (#33) + +## [2026-06-26] +### Added +- Awesome-Evals Responder — comment-only PR assistant for this repo (#7) +- Daily scan → Slack thread notifications, distributed into themed sections by topic (#8, #10, #32) +- Scan 2026-06-26: 18 verified eval finds (#8) diff --git a/README.md b/README.md index 8567f45..709abfb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Most "awesome" lists are link dumps. This one is **annotated and verified**: eve - [Notes on provenance & gaps](#notes-on-provenance-gaps) - [Deep notes](#deep-notes) - [Contributing](#contributing) +- [Changelog](CHANGELOG.md) - [License](#license) --- @@ -581,6 +582,8 @@ This repo ships **143 deep reading notes** in [`notes/`](notes/) — structured PRs welcome. Keep the bar high: **show your work** (real data/code/war-stories beat hot takes), give every entry a one-line *why*, verify the URL, and flag caveats. See [CONTRIBUTING.md](CONTRIBUTING.md). Quality over quantity — a great list is as much about what it *excludes*. +Additions land continuously via reviewed PRs (manual + [the daily scan](SCAN.md)); a [biweekly workflow](.github/workflows/changelog.yml) rolls them up into [CHANGELOG.md](CHANGELOG.md) as its own reviewed PR — see [CHANGELOG.md](CHANGELOG.md) for what shipped when. + ## License [![CC0](https://licensebuttons.net/p/zero/1.0/88x31.png)](LICENSE)