diff --git a/.github/workflows/oss-guardrails.yml b/.github/workflows/oss-guardrails.yml index 44501dd..1a888e7 100644 --- a/.github/workflows/oss-guardrails.yml +++ b/.github/workflows/oss-guardrails.yml @@ -138,45 +138,15 @@ jobs: # check never reports and leaves the PR perpetually pending). sync-parity: name: Plugin skill sync parity - permissions: - contents: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Refresh sanctioned Grok fallback mirror (master only) - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - # R5 fallback: Grok's installer needs real files under - # plugins/last9/skills/. Automation keeps it in sync so - # contributors only ever touch skills/. Regenerate from scratch so - # orphans cannot survive, then verify. - rm -rf plugins/last9/skills - mkdir -p plugins/last9/skills - for d in skills/*/; do - n=$(basename "$d") - mkdir -p "plugins/last9/skills/$n" - cp "${d}SKILL.md" "plugins/last9/skills/$n/SKILL.md" - done - if ! git diff --quiet -- plugins/last9/skills; then - git config user.name "last9-bot" - git config user.email "bot@last9.io" - git add plugins/last9/skills - git commit -m "chore(grok): refresh sanctioned fallback mirror from canonical skills/" - git push - fi - - name: Verify skill distribution run: | # Repo-as-hub model: skills/ is the single source of truth. This # check validates distribution correctness (frontmatter naming, - # declared marketplace paths, opencode tarball completeness, and - # the sanctioned Grok fallback mirror). On PRs the mirror-parity - # leg is skipped: contributors touch only skills/, and the - # master-push step above refreshes the mirror after merge. The - # gate runs at full strength on master pushes, after that refresh. - if [ "${{ github.event_name }}" = "pull_request" ]; then - export SKIP_MIRROR_PARITY=1 - fi + # declared marketplace paths, opencode tarball completeness) and + # forbids committed plugin skill copies outright. scripts/check-skill-pack.sh scripts/check-skill-pack-selftest.sh diff --git a/.grok-plugin/marketplace.json b/.grok-plugin/marketplace.json deleted file mode 100644 index e96e375..0000000 --- a/.grok-plugin/marketplace.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "last9-ai-toolkit", - "description": "Last9 AI Toolkit plugin marketplace for Grok Build.", - "owner": { "name": "Last9", "url": "https://last9.io" }, - "plugins": [ - { - "name": "last9", - "description": "Use Last9 observability workflows from Grok Build through the hosted Last9 MCP server.", - "source": "./plugins/last9", - "category": "coding" - } - ] -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2961f11..cc426e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thanks for contributing! This repo ships AI-agent skills and plugin packages for ## The canonical-source model -The top-level `skills/` directory is the **single source of truth**. Marketplace manifests for Claude Code and Codex reference it directly, and the OpenCode plugin assembles it into its npm tarball at pack time. One exception: Grok Build’s installer cannot follow root sources, so a mirror under `plugins/last9/skills/` is refreshed automatically by CI after merge — never edit or commit it by hand. Never edit the generated copy under `plugins/opencode-last9/skills/` either (gitignored, regenerated by `prepack`). +The top-level `skills/` directory is the **single source of truth**. Marketplace manifests for Claude Code and Codex reference it directly, Grok Build installs from the `plugins/last9` subdirectory via `grok plugin install --trust "last9/ai-toolkit#plugins/last9"`, and the OpenCode plugin assembles it into its npm tarball at pack time — no skill copies are committed anywhere else. Never edit generated copies under `plugins/opencode-last9/skills/` (gitignored, regenerated by `prepack`). When you change or add a skill: diff --git a/README.md b/README.md index c962245..d5189e9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,11 @@ npx skills add last9/ai-toolkit --skill last9-logs npx skills add last9/ai-toolkit -a claude-code ``` -Claude Code, Codex, and Grok Build users can install through their plugin marketplaces instead — all three read the same root-level manifests (`.claude-plugin/marketplace.json`, `.agents/plugins/marketplace.json`) and resolve skills directly from this repository's canonical `skills/` tree; there are no packaged copies to keep in sync. +Claude Code and Codex install through their marketplaces, which resolve skills directly from this repository's canonical `skills/` tree. Grok Build installs straight from the plugin subdirectory: + +```shell +grok plugin install --trust "last9/ai-toolkit#plugins/last9" +``` OpenCode users get MCP registration plus skills from a single plugin: diff --git a/plugins/last9/CHANGELOG.md b/plugins/last9/CHANGELOG.md deleted file mode 100644 index b2440ae..0000000 --- a/plugins/last9/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -## [0.2.0](https://github.com/last9/ai-toolkit/compare/v0.1.0...v0.2.0) (2026-06-17) - - -### Features - -* **go-agent-install:** add Go instrumentation install skill ([#4](https://github.com/last9/ai-toolkit/issues/4)) ([4c83639](https://github.com/last9/ai-toolkit/commit/4c836391692f86a68676b6b859e796198ac4048c)) -* **skills:** add correctness disciplines to last9-logs and last9-traces ([#1](https://github.com/last9/ai-toolkit/issues/1)) ([aed78dc](https://github.com/last9/ai-toolkit/commit/aed78dca25860e525a23c04cdf9ee9ae3394d076)) diff --git a/plugins/last9/README.md b/plugins/last9/README.md deleted file mode 100644 index 7f120ac..0000000 --- a/plugins/last9/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Last9 Agent Plugin - -Public Codex, Claude Code, and Grok Build plugin package for Last9 observability workflows. - -This plugin packages the Last9 skills for agent marketplaces: - -- `last9-logs` — log-first investigation and logjson guardrails. -- `last9-traces` — trace-first investigation and tracejson guardrails. - -Claude Code and Codex read root-level marketplace manifests that point at the canonical repository-root `skills/` directory directly — adding a skill is a single commit under `skills//`. Grok Build’s installer cannot follow root sources, so it installs from this package directory: its skills mirror is refreshed automatically by CI after merge, so contributors still only touch `skills/`. - -If you are using the Agent Skills CLI or skills.sh directly, install from the repository root instead: - -```shell -npx skills add last9/ai-toolkit -npx skills add last9/ai-toolkit --skill last9-logs -npx skills add last9/ai-toolkit --skill last9-traces -``` - -The plugin does not bundle or launch a local MCP server. Last9 MCP is hosted; configure your agent to connect to your organization's hosted endpoint. - -## Hosted MCP Configuration - -Add the hosted MCP endpoint to your local agent MCP config. For Codex: - -```toml -[mcp_servers.last9] -url = "https://app.last9.io/api/v4/organizations//mcp" -``` - -For Claude Code, add the same hosted endpoint through Claude Code's MCP configuration flow. For Grok Build, add the same hosted endpoint through its MCP configuration (a `.mcp.json` entry or the in-terminal MCP flow). - -Then start your agent and run `/mcp` to authenticate. - -Do not commit local `.codex/config.toml`, `.claude/settings.local.json`, or equivalent MCP config files. They are user- and organization-specific. - -## Skill Boundaries - -- Use `last9-logs` for log-first investigation: service/env surfacing, attribute discovery, aggregate-then-drill, body search only after narrowing. -- Use `last9-traces` for trace-first investigation: guided service/env/time/symptom/scope interview plus tracejson reference. diff --git a/plugins/last9/skills/go-agent-install/SKILL.md b/plugins/last9/skills/go-agent-install/SKILL.md deleted file mode 100644 index 373fbb1..0000000 --- a/plugins/last9/skills/go-agent-install/SKILL.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -name: go-agent-install -description: Install Last9 go-agent instrumentation into a Go service — base setup, chi router, database/sql tracing, and opt-in features like HTTP request/response body capture, with build and telemetry verification. Use when the user wants to add Last9 tracing to a Go app, "install go-agent", "instrument my Go service with Last9", "add Last9 tracing to chi", or "capture request/response bodies". -metadata: - author: last9 ---- - -# go-agent-install — add Last9 go-agent to a Go service - -This skill **edits the customer's repository**. It detects the stack, shows a diff plan, applies minimal edits, promotes the features worth turning on, then verifies telemetry reaches Last9. v1 auto-wires base + chi + `database/sql`; other frameworks fall back to base + db with the manual snippet. - -`go-agent` is the SDK path — one `agent.Start()` replaces the OpenTelemetry SDK setup, and each integration is a drop-in replacement for the standard constructor. - -## Prerequisites - -Run only inside a Go module. If there is no `go.mod` at or above the working directory, **stop** and tell the user this isn't a Go module — there is nothing to instrument. - -## Workflow - -### 1. Detect the stack - -Read `go.mod` and the `main` package. Determine: - -- **HTTP framework** — inspect the `require` block and the router construction: - - `github.com/go-chi/chi` → chi. **Auto-wire it** (step 4). - - gin / echo / gorilla / grpc / fasthttp / iris / beego → **not auto-wired in v1**. Wire base + db only, tell the user chi is the only auto-wired framework today, and hand them the manual snippet from [Other frameworks](#other-frameworks-manual). -- **SQL driver** — map the imported driver to a `DriverName`: `github.com/lib/pq` → `postgres`, `github.com/jackc/pgx` → `pgx`, `github.com/go-sql-driver/mysql` → `mysql`, `github.com/mattn/go-sqlite3` → `sqlite3`, `modernc.org/sqlite` → `sqlite`. -- **Existing instrumentation** — if a `go.opentelemetry.io/...` SDK init, an `otelgin`/`otelchi`-style middleware, or a prior `github.com/last9/go-agent` import is present, this is a **conflict**. Do not double-instrument. Report what you found and **stop** for user direction. - -### 2. Confirm the plan - -Show the detected stack and the **exact edits** (as a diff preview) before touching any file. Get one explicit confirmation. A single yes/no — not an interview. - -### 3. Wire base - -1. Add the dependency: `go get github.com/last9/go-agent` -2. In `main`, add the first two lines (and the import): - ```go - import agent "github.com/last9/go-agent" - - func main() { - agent.Start() - defer agent.Shutdown() - - // existing code, unchanged - } - ``` -3. Surface the three environment variables. Read existing values from the shell or a `.env`; if absent, prompt the user. **Never write a token into a tracked file.** - ``` - OTEL_EXPORTER_OTLP_ENDPOINT="" - OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic " - OTEL_SERVICE_NAME="" - ``` - -### 4. Wire chi - -`chiagent.New()` alone does **not** instrument — Chi captures the route pattern only when the middleware is applied *after* routes are defined. Always finish with `chiagent.Use(r)` and serve the handler it returns: - -```go -import chiagent "github.com/last9/go-agent/instrumentation/chi" - -r := chiagent.New() // or keep an existing chi.NewRouter() -r.Get("/users/{id}", handler) // define ALL routes first -handler := chiagent.Use(r) // REQUIRED — returns the instrumented http.Handler -http.ListenAndServe(":8080", handler) -``` - -Without the `chiagent.Use(r)` call, no server spans are produced. Do not instead call `r.Use(...)` here — Chi v5 panics when middleware is added after routes. - -### 5. Wire database - -Replace `sql.Open(...)` with `database.Open(...)`. Every query gets a span; connection-pool metrics and DSN attributes (host, port, user, db name) come automatically. - -```go -import "github.com/last9/go-agent/integrations/database" - -db, err := database.Open(database.Config{ - DriverName: "postgres", // from detection: postgres | pgx | mysql | sqlite3 - DSN: dsn, // reuse the existing DSN expression - DatabaseName: "", -}) -``` - -Supported drivers: `postgres`, `pgx`, `mysql`, `sqlite`, `sqlite3`. A `database.MustOpen` panic-on-error variant exists for quick init. **GORM:** don't replace it — use the official `gorm.io/plugin/opentelemetry` plugin *alongside* `database.Open` for two-layer (ORM span + wire-level SQL span) tracing; go-agent does not wrap GORM. - -### 6. Promote features - -After the core wiring builds, tell the user what go-agent gives them beyond basic spans, and offer to turn on the opt-in ones. Don't silently enable anything that changes data captured. - -- **Already on, no action** — `agent.Start()` automatically stamps **`code.*` call-site attributes** (`code.function`, `code.filepath`, `code.lineno`) onto Client/Producer/Consumer spans, so you see *where* an outbound call originated. No config, no hot-path cost on Server/Internal spans. - -- **HTTP request/response body capture** — the `httpcapture` middleware records bodies onto the span as `http.request.body` / `http.response.body`. **Opt-in, off by default.** Offer it, with the trade-off: enable only after confirming payloads carry no PII or credentials; for production prefer redacting at the collector. It must run **inside** the tracing middleware so the span is already active. Canonical wiring (net/http, from go-agent docs): - ```go - import ( - nethttpagent "github.com/last9/go-agent/instrumentation/nethttp" - "github.com/last9/go-agent/instrumentation/httpcapture" - ) - http.ListenAndServe(":8080", nethttpagent.WrapHandler(httpcapture.Middleware(mux))) - ``` - Placement differs per framework (for chi, the `chiagent.Use(r)` span wrapper must stay outermost). Follow go-agent's HTTP Body Capture docs for the framework in use, and **verify** that `http.request.body`/`http.response.body` actually land on a span before relying on it. - ```bash - export LAST9_BODY_CAPTURE_ENABLED=true # default false - export LAST9_BODY_CAPTURE_MAX_BYTES=4096 # default 8192 - export LAST9_BODY_CAPTURE_ON_ERROR_ONLY=true # only status >= 400; no alloc on success - export LAST9_BODY_CAPTURE_CONTENT_TYPES="application/json,application/xml,text/plain" # this is the default - ``` - -- **Log-trace correlation** — if the app uses `log/slog` or Uber `zap`, offer to swap in `github.com/last9/go-agent/instrumentation/slog` (or the zap equivalent) so `trace_id`/`span_id` get injected into log lines — jump from a log straight to its trace. - -Mention these; wire the ones the user accepts. Note: don't add per-operation metric instrumentation — Last9 derives RED metrics from spans server-side. - -### 7. Verify - -See [Verification](#verification). - -## Other frameworks (manual) - -v1 auto-wires chi only. For a detected non-chi framework, wire base + db automatically and hand the user the matching drop-in (each replaces the standard constructor): - -| Framework | Drop-in | -|-----------|---------| -| net/http | `nethttpagent.NewServeMux()` | -| Gin | `ginagent.Default()` / `ginagent.New()` | -| Echo | `echoagent.New()` | -| Gorilla Mux | `gorillaagent.NewRouter()` | -| gRPC | `grpcagent.NewServer()` + `grpcagent.NewClientDialOption()` | -| fasthttp | `fasthttpagent.Middleware(handler)` | -| Iris | `irisagent.New()` | -| Beego | `beegoagent.New()` | - -Import path pattern: `github.com/last9/go-agent/instrumentation/`. The package name is the bare framework (`gin`, `echo`, …), so alias it to avoid colliding with the upstream import — e.g. `import ginagent "github.com/last9/go-agent/instrumentation/gin"`. - -## Edge cases — stop or ask, don't guess - -These break a naive run. Handle each explicitly; never paper over with a guess: - -- **No `main`, or several `main` packages** (e.g. `cmd/api/main.go` + `cmd/worker/main.go`) — `agent.Start()` goes in the entrypoint that runs the server. If there's more than one, **ask which** binary to instrument; don't edit all of them. -- **Router built in a helper, not `main`** — the framework constructor may live in a `setupRouter()` / `NewServer()` function. Wire it where the router is *constructed*, not where it's used. Find the construction site before editing. -- **More than one HTTP framework present** (e.g. chi *and* gin) — ambiguous. **Ask** which serves the traffic you're instrumenting; wire one. -- **No SQL driver at all** — skip the database step entirely. Do not fabricate a `database.Open` for a service that has no SQL. -- **Dynamic / non-literal DSN** — reuse the existing DSN variable or expression in `database.Config{DSN: ...}`. Never inline a hardcoded connection string, and never a password. -- **Vendored dependencies** (`vendor/` + `-mod=vendor`) — `go get` won't update vendored modules; run `go mod vendor` after adding the dependency, or tell the user to. -- **`go get` fails** (proxy, private module, network) — report the exact error and **stop**; don't guess a workaround. -- **`golangci-lint` not installed** — say so, fall back to `go vet ./...`, and report that lint was skipped because the tool is missing. Don't silently skip. -- **App can't run locally** (needs a DB, secrets, or external config you don't have) — the build can still pass, but say plainly that end-to-end telemetry is **unverified** and what's needed to verify it. Never claim spans arrived when you didn't see them. - -## Edit guardrails - -This skill mutates the customer's repo. Hold these lines: - -- **Dry-run first** — show every edit as a diff; apply only after the step-2 confirmation. -- **Never break the build** — run `go build ./...` *before* editing. If it already fails, report that and **stop** — don't own pre-existing breakage. -- **No secrets in code** — environment variables via the shell or `.env` only; never a token in a tracked file. -- **Conflict-aware / idempotent** — if go-agent is already wired, or another OTel SDK is present, **stop** (see Detect). Never double-instrument. -- **Body capture is consent-gated** — never enable `httpcapture` without the user explicitly accepting the PII trade-off. -- **Minimal diff** — change the import, the constructor, and the db open only. Touch nothing else. - -## Verification - -1. **Build** — `go build ./...` must pass after the edits. -2. **The repo's own checks, if present** — run them and **report** the result (never skip silently): - - `golangci-lint run` when a `.golangci.y*ml` exists. - - `make build` / `make lint` / `make test` when the `Makefile` defines them. - - `go vet ./...` as a fallback. -3. **Telemetry, end-to-end** — run the app, hit one instrumented endpoint, then hand off to the `last9-traces` skill to confirm the spans arrive in Last9. `agent.Shutdown()` flushes on exit, but a blocking server never returns — rely on the batch processor's periodic flush (tunable via `OTEL_BSP_SCHEDULE_DELAY`) or stop the process cleanly. Use the real run path — not a simulated curl against a mock. - - **No prod creds yet?** go-agent has no stdout exporter, but you can point `OTEL_EXPORTER_OTLP_ENDPOINT` at a local OTLP/HTTP sink (an OpenTelemetry Collector with a `debug` exporter, or any listener on `/v1/traces`) and confirm a span appears — same export path, no production token. Seeing a `SPAN_KIND_SERVER` span named by route pattern (e.g. `/users/{id}`) confirms the HTTP layer is wired correctly. - -### If something is wrong - -- **Build breaks** — missing import (`agent`, `chiagent`, `database`), or `go get`/`go mod tidy` not run. -- **No spans in Last9** — env vars not exported into the process; bad endpoint (connection error) or bad token (401/403); process exited before `agent.Shutdown()` flushed; or no request was sent to an instrumented route. -- **Duplicate spans** — a prior OTel SDK/middleware is still wired alongside go-agent. Remove one. (This is why detection stops on a conflict.) - -## Related skills - -- `last9-traces` — confirm the spans you just produced are landing in Last9. -- `last9-logs` — verify log-trace correlation once traces flow. diff --git a/plugins/last9/skills/last9-logs/SKILL.md b/plugins/last9/skills/last9-logs/SKILL.md deleted file mode 100644 index 9c02642..0000000 --- a/plugins/last9/skills/last9-logs/SKILL.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -name: last9-logs -description: Query Last9 logs effectively — service-first scoping, attribute filters over body search, aggregate-then-drill. Use when querying or searching Last9 logs, finding log errors, building a get_logs logjson pipeline, or debugging a symptom through logs ("query logs", "find logs", "search logs", "log errors"). -compatibility: Requires the Last9 MCP server (last9-mcp-server) connected to the session -metadata: - author: last9 ---- - -# last9-logs — query Last9 logs effectively - -Extends the `get_logs`, `get_service_logs`, and `get_log_attributes` tool instructions — read those first. This skill carries the navigation judgment: which tool when, in what order, and the mistakes that make log queries fail or lie. - -**Operating principle: progressive narrowing, assisted.** At every step, surface the narrowing handles the data offers — services, environments, resource attributes — and let the user pick. Never guess scope; never go broad when a narrowing option exists. - -## Prerequisites - -Requires the [Last9 MCP server](https://github.com/last9/last9-mcp-server) connected to this session. If `get_logs`, `get_service_logs`, or `get_log_attributes` are not available as tools, **stop** — tell the user to install and authenticate the Last9 MCP server first, and link them to the README above. Never fabricate or simulate these tool calls. - -## Tool parameters at a glance - -For agents consuming this skill outside Claude Code (copy-pasted into a custom system prompt), this is the minimum parameter surface; the MCP tool descriptions remain authoritative. - -| Tool | Purpose | Key params | -|------|---------|-----------| -| `get_log_attributes` | Discover the global label catalog (names that exist somewhere in the window) | time params; optional `index` | -| `get_log_attributes_for_pipeline` | Discover the fields that actually exist within a filtered scope, each with its exact `filter_field` | `pipeline` (filter stages), time params, optional `index` | -| `get_logs` | Filter / aggregate via logjson pipeline | `logjson` (array of stages), time params, optional `index` | -| `get_service_logs` | Quick service-scoped raw lines | `service`, `severity_filters`, `body_filters`, `limit`, time params | - -**Time params (every tool):** `lookback_minutes` for relative windows, or `start_time_iso` + `end_time_iso` for explicit ranges — see Time discipline below. - -## Before any query — anti-pattern checklist - -Check every query against these seven. Each one names the redirect, not just the mistake. - -| # | Anti-pattern | Do this instead | -|---|--------------|-----------------| -| 1 | Global search across all logs | Scope to a service first (`ServiceName` filter, or `get_service_logs`) | -| 2 | Full-text body search for something an attribute covers | Call `get_log_attributes`, filter on the attribute | -| 3 | Pulling raw log lines for a broad symptom | Aggregate first (count by severity / attribute), then drill into the dominant pattern | -| 4 | Pipeline starting with an `aggregate` stage | First stage must be a `filter` — a missing filter is silently treated as match-all, widening scope to every log in the window. Lead with an explicit filter so scope is deliberate | -| 5 | Bare dotted field refs (`service.name`, `k8s.namespace.name`) | `ServiceName` / `attributes['field.name']` / `resources['field.name']` — a few refs (`service.name`, `k8s.*`) are silently normalized as aliases; everything else is rejected. Always write the canonical form | -| 6 | Assuming an attribute name carries over between services (`attributes['http.status_code']` on one service, `attributes['status_code']` on another) | Attribute names vary per scope with no error on a wrong guess — the filter silently matches nothing, and the **global** catalog can list keys that are empty for your scope. Confirm via `get_log_attributes_for_pipeline` with your scoping filter, and use only the `filter_field` it returns | -| 7 | Fabricating fixed timestamps for a relative request ("last 30 mins") | `lookback_minutes` — see Time discipline below | - -## Methodology — service first, aggregate first, drill last - -Work through these steps in order. Skip a step only when the conversation already answered it. - -1. **Establish scope — surface, don't interrogate** — if the user named a service loosely or you suspect a typo, resolve it with `did_you_mean` (`type: service`). If no service is named, don't ask blind: surface the inventory — run a `filter` → `aggregate` `$count` grouped by `ServiceName` (pre-filtered by the symptom when there is one) and present the top candidates for the user to pick from. When multiple environments exist, surface `resources['deployment.environment']` values the same way and confirm which one — include the empty-env bucket explicitly when present, since unset environments often dominate. (Use the aggregate to surface environments; `did_you_mean` coverage for environment names is unreliable.) If the service dimension doesn't discriminate (catch-all service, or none), scope by the next-best dimension attribute discovery surfaces — namespace, host, environment. Never fire an unscoped body search. -2. **Discover attributes — and offer them as narrowing handles** — call `get_log_attributes` to see available log and resource attributes. It is global (time-window scoped, no service param) — service scoping happens in your filter stage, not here. Use it to learn the exact field names before filtering; never guess whether a field lives in `attributes[...]` or `resources[...]`. Discovery and query are strictly sequential: discover, wait for results, then build the query from the names returned — never fire discovery and `get_logs` in the same response. Discovery is two-stage when filtering on a structured attribute (an HTTP/gRPC status, `user_id`, latency, region — any field-level value, not a free-text body match): the global catalog first, then **confirm within your scope** — build the scoping filter stage(s) (commonly `{"$eq": ["ServiceName", ""]}`, but possibly namespace, environment, host, or a combination), call `get_log_attributes_for_pipeline` with that pipeline, and filter using only the `filter_field` values it returns. The global catalog can list keys that are empty for your scope, and near-duplicate names coexist (`status_code` on some sources, `http.status_code` on others — neither is a safe default). If `get_log_attributes_for_pipeline` is unavailable (older MCP server), fall back to verifying the candidate field with an aggregate `$count` grouped by it inside your scope before trusting a filter on it. When scope is still broad, present the discriminating resource attributes (`resources['k8s.namespace.name']`, `resources['k8s.deployment.name']`, `resources['deployment.environment']`, host) as narrowing options the user can pick — an aggregate `$count` grouped by a candidate attribute shows them where their logs actually concentrate. A handle whose buckets split evenly has no narrowing power — pick a different one. -3. **Aggregate to find the pattern** — call `get_logs` with a pipeline: `filter` stage (service + symptom conditions) followed by an `aggregate` stage (e.g., `$count` grouped by `SeverityText` or a suspect attribute). Present the narrowed pattern before pulling raw lines. -4. **Narrow with attribute filters** — tighten the filter stage using the discovered attributes (`$eq`, `$gte` on `attributes['http.status_code']`, etc.). -5. **Drill into raw lines** — call `get_service_logs` (simple: `service`, `severity_filters`, `body_filters`, `limit`) or `get_logs` with the narrowed filter. -6. **Body search — last resort** — `$containsWords` on `Body` only inside the already-narrowed scope, and only for text no attribute covers. - -## Time discipline - -`lookback_minutes` (default 5) or `start_time_iso` + `end_time_iso` are **top-level tool params** — never write `Timestamp` conditions inside the pipeline for time ranges. - -- **Relative requests** ("last 30 mins", "past hour") → `lookback_minutes`. Never fabricate fixed timestamps for a relative request — a generated "now" is wrong by the time the query runs and silently shifts the window. -- **Explicit dates** ("on 2026-06-08 between 10:00 and 11:00 UTC") → `start_time_iso` / `end_time_iso` in RFC3339/ISO8601 UTC, e.g. `2026-06-08T10:00:00Z`. Legacy `YYYY-MM-DD HH:MM:SS` is compatibility-only — do not generate it. -- **Both present** (explicit range plus a relative phrase) → explicit timestamps win; drop `lookback_minutes`. - -## Index targeting - -Include `index` in `get_logs` / `get_log_attributes` / `get_log_attributes_for_pipeline` **only when the user explicitly names one**. Never guess or invent an index — omit the parameter entirely otherwise. - -| User says | `index` value | -|-----------|---------------| -| "rehydration index X" | `rehydration_index:X` | -| "physical index X" or "index X" | `physical_index:X` | -| no index named | omit `index` | - -## logjson essentials - -A `get_logs` query is a JSON **array of stages**, executed in order: `filter` → `parse` → `aggregate` / `window_aggregate`. Conditions take the form `{"$op": [field, value]}` — field is always the first element. - -**Operators:** - -| Category | Operators | -|----------|-----------| -| Equality | `$eq` `$neq` `$ieq` `$ineq` | -| Numeric (value is a **string**) | `$gt` `$lt` `$gte` `$lte` | -| Substring | `$contains` `$notcontains` `$icontains` `$inotcontains` | -| Word-boundary (prefer for `Body`) | `$containsWords` `$icontainsWords` | -| Regex | `$regex` `$notregex` `$iregex` `$inotregex` | -| Logical | `$and` `$or` `$not` | - -**Fields:** `Body`, `ServiceName` (always prefer over similar attributes), `SeverityText` (DEBUG/INFO/WARN/ERROR/FATAL — may be empty for services that don't set it; aggregate by `SeverityText` before filtering on it, or a severity filter silently returns nothing; when it is empty and bodies are JSON, add a `parse` stage and aggregate by the extracted level field, e.g. `attributes['level']`), `Timestamp`, `attributes['field.name']`, `resources['field.name']`. - -**Canonical shapes:** - -Service-scoped error search (filter only — no aggregation unless the user asks "how many"): - -```json -[{ - "type": "filter", - "query": { - "$and": [ - {"$eq": ["ServiceName", "auth"]}, - {"$containsWords": ["Body", "error"]} - ] - } -}] -``` - -Error count by service (aggregate ALWAYS preceded by filter): - -```json -[{ - "type": "filter", - "query": {"$and": [{"$containsWords": ["Body", "error"]}]} -}, { - "type": "aggregate", - "aggregates": [{"function": {"$count": []}, "as": "error_count"}], - "groupby": {"ServiceName": "service"} -}] -``` - -Rate over time windows: - -```json -[{ - "type": "filter", - "query": {"$and": [{"$neq": ["attributes['endpoint']", ""]}]} -}, { - "type": "window_aggregate", - "function": {"$count": []}, - "as": "request_rate", - "window": ["5", "minutes"], - "groupby": {"attributes['endpoint']": "endpoint"} -}] -``` - -**Common mistakes:** - -| Wrong | Correct | -|-------|---------| -| `{"$eq": ["service.name", "auth"]}` | `{"$eq": ["ServiceName", "auth"]}` | -| `{"$eq": ["k8s.namespace.name", "prod"]}` | `{"$eq": ["resources['k8s.namespace.name']", "prod"]}` | -| `[{"type": "aggregate", ...}]` as first stage | `filter` stage first, then `aggregate` | -| `{"$contains": ["Body", "error"]}` for word search | `{"$containsWords": ["Body", "error"]}` | -| `{"$gt": ["attributes['http.status_code']", 500]}` | `{"$gt": ["attributes['http.status_code']", "500"]}` (the reference specifies string values; raw numbers are often tolerated, but strings are the guaranteed form) | -| `{"$gte": ["Timestamp", "2026-06-04T00:00:00Z"]}` in pipeline | `start_time_iso` / `end_time_iso` as top-level params | -| `{"ServiceName": "auth"}` | `{"$eq": ["ServiceName", "auth"]}` | -| `"group_by": {...}` / `"aggregations": [...]` | `"groupby": {...}` / `"aggregates": [...]` | - -**Silent-failure warning:** misspelled pipeline keys are NOT rejected — they are silently ignored. A `group_by` typo returns an ungrouped total with no error, which reads like a valid answer. Check key spelling against this table before trusting aggregate output. - -## Which tool when - -| Need | Tool | -|------|------| -| Resolve a fuzzy service name | `did_you_mean` (`type: service`) | -| What attributes exist anywhere in the window? | `get_log_attributes` | -| Which fields actually exist in my filtered scope, and their exact `filter_field`? | `get_log_attributes_for_pipeline` | -| Counts, grouping, rates, complex filters | `get_logs` (logjson pipeline) | -| Quick service-scoped raw lines | `get_service_logs` | - -## Related skills - -This skill owns **log-first** investigation. For trace-first work (latency breakdowns, span analysis, trace IDs, tracejson queries), use `last9-traces` — guided 5-question trace investigation plus the tracejson reference card. On an ambiguous debugging prompt ("why is checkout failing"), take the logs path here and point the user to `last9-traces` for the trace side; do not attempt trace queries from this skill. - - diff --git a/plugins/last9/skills/last9-traces/SKILL.md b/plugins/last9/skills/last9-traces/SKILL.md deleted file mode 100644 index cf9f906..0000000 --- a/plugins/last9/skills/last9-traces/SKILL.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -name: last9-traces -description: Guided trace investigation and tracejson query reference for Last9. Interviews the user with 5 targeted questions (service, environment, time window, symptom, scope), fires the right tool call, and carries the reference card for get_traces tracejson pipelines. Use when investigating a trace issue, debugging a service, building or fixing a tracejson query, or when user says "diagnose" / "investigate" / "what's wrong with". -compatibility: Requires the Last9 MCP server (last9-mcp-server) connected to the session -metadata: - author: last9 ---- - -# last9-traces — trace investigation + tracejson reference - -Extends the `get_traces`, `get_service_traces`, and `get_trace_attributes` tool instructions — read those first. This skill carries the investigation flow (which question, which tool, in what order) and the query reference (operators, field syntax, common mistakes). - -**Operating principle: progressive narrowing, assisted.** At every step, surface the narrowing handles the data offers — services, environments, trace attributes — and let the user pick. Never guess scope; never go broad when a narrowing option exists. - -## Prerequisites - -Requires the [Last9 MCP server](https://github.com/last9/last9-mcp-server) connected to this session. If `get_traces`, `get_service_traces`, or `get_trace_attributes` are not available as tools, **stop** — tell the user to install and authenticate the Last9 MCP server first, and link them to the README above. Never fabricate or simulate these tool calls. - -## Tool parameters at a glance - -For agents consuming this skill outside Claude Code (copy-pasted into a custom system prompt), this is the minimum parameter surface; the MCP tool descriptions remain authoritative. - -| Tool | Purpose | Key params | -|------|---------|-----------| -| `get_trace_attributes` | Discover available trace field names (`filter_field`) | time params | -| `get_traces` | Filter / aggregate via tracejson pipeline | tracejson pipeline, time params | -| `get_service_traces` | Service-scoped traces or a specific trace | `service`, optional `trace_id`, time params | - -**Time params (every tool):** `lookback_minutes` for relative windows, or `start_time_iso` + `end_time_iso` for explicit ranges — see Time discipline below. - -## Attributes first — strict sequence - -Discovery and query are strictly sequential: call `get_trace_attributes`, wait for its results, then build the query from the `filter_field` values it returned — never fire discovery and `get_traces` in the same response. Attribute names vary per service with no error on a wrong guess (`attributes['http.status_code']` on one service, `attributes['status_code']` on another) — the filter silently returns zero results. Verify the name via discovery before filtering on it. - -## Time discipline - -`lookback_minutes` or `start_time_iso` + `end_time_iso` are **top-level tool params**. - -- **Relative requests** ("last 15 minutes", "past hour") → `lookback_minutes`. Never fabricate fixed timestamps for a relative request — a generated "now" is wrong by the time the query runs and silently shifts the window. -- **Explicit dates** → `start_time_iso` / `end_time_iso` in RFC3339/ISO8601 UTC, e.g. `2026-06-08T10:00:00Z`. Legacy `YYYY-MM-DD HH:MM:SS` is compatibility-only — do not generate it. -- **Both present** (explicit range plus a relative phrase) → explicit timestamps win; drop `lookback_minutes`. - -## Guided investigation — 5 questions - -Ask one at a time. For each question, provide your recommended answer based on context already in the conversation; if the conversation already answers a question, skip it and state what you inferred. - -**Q1 — Service** -"Which service are you investigating?" -→ If the user named a service loosely, resolve it with `did_you_mean` (`type: service`). If they can't name one, don't ask blind — surface candidates: a `get_traces` aggregate counting traces by `ServiceName` (symptom-filtered when there is one) and let them pick. - -**Q2 — Environment** -"Which environment? (e.g. production, staging) — I'll look up the exact attribute name." -→ Call `get_trace_attributes` to find the deployment environment field. Use the returned `filter_field` verbatim for the environment condition. - -**Q3 — Time window** -"When did you first notice the issue? Give a relative window ('last 15 minutes') or absolute start/end times." -→ Map the answer per Time discipline above: relative → `lookback_minutes`; absolute → `start_time_iso` / `end_time_iso`. - -**Q4 — Symptom** -"What are you seeing — errors, high latency, a specific operation failing, or something else?" -→ Map to: `StatusCode = STATUS_CODE_ERROR` for errors; `Duration > ` for latency; `SpanName $contains ` for a specific operation. - -**Q5 — Scope** -"Do you have a specific trace ID to look at, or should we search broadly?" -→ Specific trace ID → use `get_service_traces` with `trace_id`. -→ Broad search → use `get_traces` with a tracejson pipeline built from Q1–Q4 answers. - -After Q5, immediately execute the tool call — no confirmation step. - -## tracejson reference - -## Operator quick reference - -**Comparison** (value must be a string): -`$eq` `$neq` `$gt` `$gte` `$lt` `$lte` - -**String match**: -`$contains` `$notcontains` `$icontains` `$inotcontains` -`$containsWords` `$notcontainsWords` `$icontainsWords` `$inotcontainsWords` -`$regex` `$notregex` `$iregex` `$inotregex` -`$ieq` `$ineq` - -**Existence**: `$exists` `$notnull` - -**Logical**: `$and` `$or` `$not` - -Form: `{"$op": [field, value]}` — field is always the first element. - -## Field mapping priority - -Call `get_trace_attributes` to get the exact `filter_field` for any resource or span attribute. Use `filter_field` verbatim — never transform it. - -| Situation | filter_field to use | -|---------------------------------|--------------------------------------------| -| `resource_service.name` | `ServiceName` | -| `resource_` | `resources['']` | -| `event_` | `events['']` | -| known top-level field | field name as-is (no brackets) | -| `grpc.status_code` | `attributes['rpc.grpc.status_code']` | -| anything else | `attributes['']` | - -Top-level fields: `TraceId` `SpanId` `ServiceName` `SpanName` `SpanKind` `StatusCode` `StatusMessage` `Duration` `Timestamp` `ParentSpanId` `TraceState` - -## Common mistakes - -| Wrong | Correct | -|--------------------------------------------|------------------------------------------------| -| `"resource_department"` | `"resources['department']"` | -| `"ResourceAttributes.department"` | `"resources['department']"` | -| `"SpanAttributes.http.method"` | `"attributes['http.method']"` | -| `resources["key"]` (double quotes) | `resources['key']` (single quotes) | -| `{"ServiceName": "checkout"}` | `{"$eq": ["ServiceName", "checkout"]}` | -| `{"function": "count", "alias": "n"}` | `{"function": {"$count": []}, "as": "n"}` | -| `"aggregations": [...]` | `"aggregates": [...]` | -| `"group_by": {...}` | `"groupby": {...}` | -| `{"$eq": ["StatusCode", "ERROR"]}` | `{"$eq": ["StatusCode", "STATUS_CODE_ERROR"]}` | -| `{"$eq": ["SpanKind", "SERVER"]}` | `{"$eq": ["SpanKind", "SPAN_KIND_SERVER"]}` | - -**Failure modes differ by mistake class.** Field-ref, quoting, and pipeline-key mistakes are rejected with corrective errors (the API tells you the right form). Enum-value mistakes are NOT: a wrong `StatusCode` or `SpanKind` value silently returns zero results with no error. If a StatusCode/SpanKind filter comes back empty, check the value against the `STATUS_CODE_*` / `SPAN_KIND_*` forms before trusting the empty result. Numeric values: strings are the guaranteed form; raw numbers are tolerated. - -## Related skills - -This skill owns **trace-first** investigation and tracejson query craft. For log-first work (log search, log aggregation, logjson pipelines), use `last9-logs`. On an ambiguous debugging prompt, the logs skill takes the logs path and refers trace work here — symmetrically, refer log questions there; do not attempt logjson queries from this skill. diff --git a/plugins/last9/skills/sidekiq-debugging/SKILL.md b/plugins/last9/skills/sidekiq-debugging/SKILL.md deleted file mode 100644 index caf5e59..0000000 --- a/plugins/last9/skills/sidekiq-debugging/SKILL.md +++ /dev/null @@ -1,245 +0,0 @@ ---- -name: sidekiq-debugging -description: Debug Sidekiq/ActiveJob background jobs through Last9 telemetry — failing jobs, retry storms, slow jobs, queue backlog, and alerting on job health. Carries the Rails job span anatomy (dual ActiveJob+Sidekiq instrumentation, job identity/dedup keys) and named, validated tracejson/PromQL recipes. Use when investigating background job issues or when user says "jobs failing", "sidekiq", "background jobs", "queue backed up", "retry storm", or wants alerts on job failures. -compatibility: Requires the Last9 MCP server (last9-mcp-server) connected to the session -metadata: - author: last9 ---- - -# sidekiq-debugging — background job investigation via Last9 - -Debugs Rails background jobs (Sidekiq, via ActiveJob or bare) through telemetry alone — traces first, logs and trace-derived metrics where they fit. No application or Redis access assumed. - -**Operating principle: identity before counting.** Pick the identity level first — job class, job instance, or execution attempt — and the query follows. Counts at the wrong level inflate or deflate with no error to warn you. - -## Prerequisites - -Requires the [Last9 MCP server](https://github.com/last9/last9-mcp-server) connected to this session. If `get_traces`, `get_trace_attributes`, or `prometheus_instant_query` are not available as tools, **stop** — tell the user to install and authenticate the Last9 MCP server first. Never fabricate or simulate these tool calls. - -For tracejson operator syntax, field mapping, and common mistakes, defer to the `last9-traces` skill — this skill only carries Sidekiq-specific knowledge on top of it. - -## Applicability check — run this FIRST - -This skill triggers on generic phrases ("jobs failing", "queue backed up") that also describe Kafka, SQS, Celery, and other job systems. Before using any recipe, verify the target service actually runs Sidekiq: - -```json -[ - {"type": "filter", "query": {"$and": [ - {"$eq": ["ServiceName", ""]}, - {"$eq": ["SpanKind", "SPAN_KIND_CONSUMER"]} - ]}}, - {"type": "aggregate", - "groupby": {"attributes['messaging.system']": "messaging_system"}, - "aggregates": [{"function": {"$count": []}, "as": "spans"}]} -] -``` - -- `sidekiq` and/or `active_job` present → proceed with this skill. -- Neither present (e.g. only `kafka`, `aws_sqs`, or empty) → **stop using this skill.** Tell the user the service does not emit Sidekiq/ActiveJob spans and route the investigation through `last9-traces` instead. Do not adapt these recipes to other messaging systems — the anatomy (dual spans, dedup keys) is Sidekiq-specific and will mislead. -- A service can show multiple systems (e.g. `kafka` + `sidekiq`); recipes still apply to its sidekiq/active_job spans. -- The gate reads consumer spans only — a system that appears solely on the producer side (e.g. the service publishes to Kafka but never consumes it) won't show up, and doesn't affect applicability. - -## Symptom router - -| User says | Section | Recipes | -|---|---|---| -| "jobs are failing" / "alert on job failures" | Jobs failing | R1, R2, R3 | -| "same job keeps retrying" / "retry storm" | Retry storms | R4 | -| "jobs are slow" / "worker latency" | Slow jobs | R5 | -| "queue backed up" / "jobs not processing" | Queue backlog | R6 | -| "job disappeared" / "dead set" | Known gaps | — | - -## Sidekiq trace anatomy - -Read this before running any recipe. Every fact here comes from live production traces, not framework docs. - -### Dual instrumentation — every execution emits TWO consumer spans - -Rails apps instrumented with OTel emit one span from the ActiveJob layer AND one from the Sidekiq adapter layer for the *same* job execution. Both are `SPAN_KIND_CONSUMER`; both get `STATUS_CODE_ERROR` on failure. - -| Layer | `messaging.system` | Instance ID attribute | Notes | -|---|---|---|---| -| ActiveJob | `active_job` | `messaging.message.id` = ActiveJob job_id (UUID) | also carries `messaging.active_job.message.provider_job_id` = Sidekiq JID | -| Sidekiq | `sidekiq` | `messaging.message_id` = Sidekiq JID | also carries `messaging.sidekiq.job_class` | - -The cross-link: the ActiveJob span's `provider_job_id` equals the Sidekiq span's `messaging.message_id`. - -**Rule: every job query MUST pin one layer** with `{"$eq": ["attributes['messaging.system']", "active_job"]}` (or `"sidekiq"`). Unpinned queries double-count everything. - -### Job identity taxonomy — three levels - -1. **Job class** — e.g. `Billing::ChargeJob`. Lives in `SpanName`, formatted ` process ` (e.g. `default process Billing::ChargeJob`). Same class on two queues = two SpanNames. -2. **Job instance** — one enqueued job with specific arguments. Two IDs exist: - - `messaging.message.id` (ActiveJob job_id, on `active_job` spans) — survives Sidekiq native retries AND `retry_on` re-enqueues. **Strongest instance identity; prefer it.** - - `messaging.message_id` (Sidekiq JID, on `sidekiq` spans) — survives Sidekiq native retries only; `retry_on` mints a fresh JID, so re-enqueues count as new instances. -3. **Execution attempt** — each retry produces a fresh pair of consumer spans. Raw error-span counts measure attempts, not instances. - -### Other facts that bite - -- `Duration` is **nanoseconds** (`313200779` ≈ 313ms). -- `StatusCode`/`SpanKind` take enum forms (`STATUS_CODE_ERROR`, `SPAN_KIND_CONSUMER`); wrong values return empty results with **no error** — see `last9-traces` for the full table. -- Enqueue side is instrumented too: `SPAN_KIND_PRODUCER` spans with `messaging.destination` = queue name on both layers. -- Queue name is available as `attributes['messaging.destination']` on both producer and consumer `sidekiq`-layer spans, and as the `SpanName` prefix. - -## Jobs failing - -### R1: unique-job-classes-failing (PromQL, alertable directly) - -**When:** "how many *different* jobs are failing" — distinct classes, not failure volume. One class failing 100× counts as 1. - -Last9 derives `trace_endpoint_count` from spans with `span_name`, `span_kind`, `status_code` labels. It is a **per-window gauge**, not a counter — use `sum_over_time`, never `rate()`/`increase()` (see the `last9-trace-metrics` skill): - -```promql -count( - sum by (span_name) ( - sum_over_time(trace_endpoint_count{ - service_name="", - span_kind="SPAN_KIND_CONSUMER", - status_code="STATUS_CODE_ERROR" - }[5m]) - ) > 0 -) -``` - -Inner `sum by (span_name) ... > 0` keeps only classes with ≥1 failure; outer `count` counts classes. To dedupe job class across queues (SpanName carries the queue prefix): - -```promql -count( - sum by (job_class) ( - label_replace( - sum_over_time(trace_endpoint_count{service_name="", span_kind="SPAN_KIND_CONSUMER", status_code="STATUS_CODE_ERROR"}[5m]), - "job_class", "$1", "span_name", ".+ process (.+)" - ) - ) > 0 -) -``` - -**Gotcha:** the trace-derived metric path counts classes only — instance IDs are span attributes and never become metric labels (cardinality). - -### R2: unique-job-instances-failing (tracejson) - -**When:** distinct job *instances* failing — one stuck instance retrying 25× counts as 1; 100 different instances of one class count as 100. - -Chained aggregates emulate `COUNT(DISTINCT ...)`: - -```json -[ - {"type": "filter", "query": {"$and": [ - {"$eq": ["ServiceName", ""]}, - {"$eq": ["SpanKind", "SPAN_KIND_CONSUMER"]}, - {"$eq": ["StatusCode", "STATUS_CODE_ERROR"]}, - {"$eq": ["attributes['messaging.system']", "active_job"]} - ]}}, - {"type": "aggregate", - "groupby": {"attributes['messaging.message.id']": "job_instance", "SpanName": "job_class"}, - "aggregates": [{"function": {"$count": []}, "as": "attempts"}]}, - {"type": "aggregate", - "groupby": {"job_class": "job_class"}, - "aggregates": [{"function": {"$count": []}, "as": "unique_failing_instances"}]} -] -``` - -Drop `job_class` from the final `groupby` for a single org-wide number. Add a `$contains` SpanName condition to scope to one job family. - -**Gotchas:** dedup is window-scoped — an instance failing in two separate query windows counts twice. The `sidekiq` layer + `messaging.message_id` variant gives identical results until `retry_on` re-enqueues occur (then it overcounts). - -### R3: instance-failure-alert (wiring) - -R2's tracejson can't be a PromQL alert indicator directly. Path: **scheduled search → metric → alert**. - -1. Save R2 as a scheduled search rule (eval every 1m, trailing 5m window) emitting a counter metric, e.g. `unique_failing_job_instances` (labeled by `job_class` if R2 keeps the per-class groupby) -2. Alert on the emitted metric with threshold `> N`; pick the alert window in Alert Studio - -Class-level alerting (R1) needs no scheduled search — alert on the PromQL directly. Prefer R1 unless retry inflation genuinely distorts your signal (measure first: run R2's middle stage and compare attempts vs instances). - -## Retry storms - -### R4: instances-stuck-retrying - -**When:** find specific instances burning retries. Post-aggregate `filter` acts as SQL `HAVING`: - -```json -[ - {"type": "filter", "query": {"$and": [ - {"$eq": ["ServiceName", ""]}, - {"$eq": ["SpanKind", "SPAN_KIND_CONSUMER"]}, - {"$eq": ["StatusCode", "STATUS_CODE_ERROR"]}, - {"$eq": ["attributes['messaging.system']", "active_job"]} - ]}}, - {"type": "aggregate", - "groupby": {"attributes['messaging.message.id']": "job_instance", "SpanName": "job_class"}, - "aggregates": [{"function": {"$count": []}, "as": "failed_attempts"}]}, - {"type": "filter", "query": {"$gt": ["failed_attempts", "2"]}} -] -``` - -Output: each row = one stuck instance with its attempt count. Many rows for one `job_class` = that class is systemically broken; scattered classes = shared dependency failing. - -## Slow jobs - -### R5: duration-percentiles-by-class - -```json -[ - {"type": "filter", "query": {"$and": [ - {"$eq": ["ServiceName", ""]}, - {"$eq": ["SpanKind", "SPAN_KIND_CONSUMER"]}, - {"$eq": ["attributes['messaging.system']", "active_job"]} - ]}}, - {"type": "aggregate", - "groupby": {"SpanName": "job_class"}, - "aggregates": [ - {"function": {"$quantile": [0.95, "Duration"]}, "as": "p95_ns"}, - {"function": {"$count": []}, "as": "executions"} - ]} -] -``` - -**Gotcha:** `Duration` is nanoseconds — divide by 1e9 for seconds before reporting. Low-volume classes (executions < ~20) make p95 noisy; report the count alongside. - -## Queue backlog - -### R6: enqueue-vs-process-rate - -**When:** "is a queue falling behind?" Direct queue depth and enqueue→start latency are NOT derivable from traces (see Known gaps). The proxy: compare producer span rate vs consumer span rate per queue over the same window. - -Run twice with `SPAN_KIND_PRODUCER` then `SPAN_KIND_CONSUMER`: - -```json -[ - {"type": "filter", "query": {"$and": [ - {"$eq": ["ServiceName", ""]}, - {"$eq": ["SpanKind", "SPAN_KIND_PRODUCER"]}, - {"$eq": ["attributes['messaging.system']", "sidekiq"]} - ]}}, - {"type": "aggregate", - "groupby": {"attributes['messaging.destination']": "queue"}, - "aggregates": [{"function": {"$count": []}, "as": "spans"}]} -] -``` - -Enqueued ≫ processed sustained across windows = backlog growing. Roughly equal = keeping up. Use the same lookback for both runs. - -**Gotcha:** aggregate output is unordered and result limits truncate silently — a low `limit` can drop the *highest-volume* queues from the comparison (verified live: a limit of 6 hid the three busiest queues out of ~40). Leave the limit unset or set it well above the queue count. - -## Known gaps — validated as NOT answerable from telemetry - -- **Dead set / retries-exhausted jobs**: Sidekiq does not log job death by default, and no span marks it. Requires an app-side death handler (`config.death_handlers`) that logs or emits a metric. If logs exist, search via the `last9-logs` skill. -- **Enqueue→start latency (true queue latency)**: consumer spans carry no `enqueued_at` attribute, and tracejson cannot join producer and consumer spans. R6's rate comparison is the honest proxy. - -Do not improvise recipes for these from priors. Both probes ran against live production telemetry and came back empty. - -## Alerting cookbook - -| Want | Path | Recipe | -|---|---|---| -| N distinct job classes failing | PromQL on `trace_endpoint_count`, alert directly | R1 | -| N distinct job instances failing | scheduled search → metric → alert | R2+R3 | -| Specific critical job failing at all | R1 PromQL with `span_name` filter, threshold `> 0` | R1 variant | -| Queue falling behind | two scheduled searches (enqueue/process rates) → metric ratio | R6 | - -Start with R1 — zero moving pieces. Add the scheduled-search path only when retry noise provably distorts the class-level signal. - -## Related skills - -This skill owns Sidekiq/ActiveJob-specific debugging. For general tracejson query craft (operators, field mapping, common mistakes), use `last9-traces`. For log-side investigation (death handlers, exception text search), use `last9-logs`. diff --git a/release-please-config.json b/release-please-config.json index 9b99e23..2c1d9d9 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -3,10 +3,6 @@ "release-type": "simple", "include-component-in-tag": false, "packages": { - "plugins/last9": { - "package-name": "last9", - "changelog-path": "CHANGELOG.md" - }, "plugins/opencode-last9": { "package-name": "opencode-last9", "changelog-path": "CHANGELOG.md", diff --git a/scripts/check-skill-pack-selftest.sh b/scripts/check-skill-pack-selftest.sh index 9ae5cf7..be61527 100755 --- a/scripts/check-skill-pack-selftest.sh +++ b/scripts/check-skill-pack-selftest.sh @@ -13,10 +13,8 @@ setup_fixture() { "$FIX/.claude-plugin" \ "$FIX/.agents/plugins" \ "$FIX/.codex-plugin" \ - "$FIX/.grok-plugin" \ "$FIX/skills/last9-logs" \ - "$FIX/plugins/opencode-last9" \ - "$FIX/plugins/last9/skills/last9-logs" + "$FIX/plugins/opencode-last9" cp "$ROOT_DIR/scripts/check-skill-pack.sh" "$FIX/scripts/" cat > "$FIX/plugins/opencode-last9/package.json" <<'PKG' { @@ -28,11 +26,9 @@ setup_fixture() { } PKG printf -- '---\nname: last9-logs\ndescription: x\n---\nbody\n' > "$FIX/skills/last9-logs/SKILL.md" - cp "$FIX/skills/last9-logs/SKILL.md" "$FIX/plugins/last9/skills/last9-logs/SKILL.md" printf '{"plugins":[{"name":"last9","source":"./","skills":["./skills/"]}]}' > "$FIX/.claude-plugin/marketplace.json" printf '{"plugins":[{"name":"last9","source":{"source":"local","path":"./"}}]}' > "$FIX/.agents/plugins/marketplace.json" printf '{"name":"last9","skills":"./skills/"}' > "$FIX/.codex-plugin/plugin.json" - printf '{"name":"last9-ai-toolkit","plugins":[{"name":"last9","source":"./plugins/last9"}]}' > "$FIX/.grok-plugin/marketplace.json" (cd "$FIX" && git init -q && git add -A && git -c user.email=t@t -c user.name=t commit -qm base) } @@ -83,23 +79,6 @@ rm "$FIX/.claude-plugin/marketplace.json" git -C "$FIX" rm -q .claude-plugin/marketplace.json expect_fail "deleted manifest" -# Branch 6: sanctioned Grok mirror drifted from canonical. -setup_fixture mirror-drift -printf 'drift' >> "$FIX/plugins/last9/skills/last9-logs/SKILL.md" -commit_fault -expect_fail "mirror drift" - -# Branch 6b: mirror drift is skippable for contributor PRs. -if ! SKIP_MIRROR_PARITY=1 sh "$FIX/scripts/check-skill-pack.sh" >/dev/null 2>&1; then - echo "selftest FAILED: SKIP_MIRROR_PARITY did not skip drift" >&2 - exit 1 -fi -# And full strength still catches it even when the caller exports the skip. -if run_full sh "$FIX/scripts/check-skill-pack.sh" >/dev/null 2>&1; then - echo "selftest FAILED: drift passed despite env-leak guard" >&2 - exit 1 -fi - # Branch 7: codex plugin skills pointer drift. setup_fixture codex-drift printf '{"name":"last9","skills":"./wrong/"}' > "$FIX/.codex-plugin/plugin.json" @@ -113,4 +92,11 @@ if ! run_full sh "$FIX/scripts/check-skill-pack.sh" >/dev/null 2>&1; then exit 1 fi +# Committed plugin skill copies must fail (hub invariant). +setup_fixture committed-copy +mkdir -p "$FIX/plugins/acme/skills/rogue" +cp "$FIX/skills/last9-logs/SKILL.md" "$FIX/plugins/acme/skills/rogue/SKILL.md" +git -C "$FIX" add -A && git -C "$FIX" -c user.email=t@t -c user.name=t commit -qm fault +expect_fail "committed plugin skill copy" + echo "check-skill-pack selftests passed" diff --git a/scripts/check-skill-pack.sh b/scripts/check-skill-pack.sh index 5c836d2..907c5ad 100755 --- a/scripts/check-skill-pack.sh +++ b/scripts/check-skill-pack.sh @@ -11,50 +11,15 @@ set -eu ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" cd "$ROOT_DIR" -# 0. Committed plugin skill copies are forbidden except the sanctioned -# Grok fallback mirror (Grok's installer cannot follow root sources or -# symlinks, so it needs real files inside plugins/last9/). -sanctioned="plugins/last9/skills" -violations="$(git ls-files 'plugins/*/skills' | grep -v "^$sanctioned/" || true)" -if [ -n "$violations" ]; then - echo "::error::committed plugin skill copies outside the sanctioned Grok fallback:" >&2 - echo "$violations" >&2 +# 0. Committed plugin skill copies are forbidden — every consumer reads or +# assembles from the canonical tree; Grok installs via +# `grok plugin install --trust last9/ai-toolkit#plugins/last9`. +if [ -n "$(git ls-files 'plugins/*/skills/*')" ]; then + echo "::error::committed plugin skill copies are forbidden (hub model):" >&2 + git ls-files 'plugins/*/skills' >&2 exit 1 fi -# 0a. The hub must have at least one skill, or every later check is vacuous. -total_skills="$(git ls-files 'skills/*/SKILL.md' | wc -l | tr -d ' ')" -if [ "$total_skills" -lt 1 ]; then - echo "::error::no tracked skills/*/SKILL.md files found" >&2 - exit 1 -fi - -# 0b. Sanctioned Grok mirror parity. SKIP_MIRROR_PARITY=1 marks the known -# PR-time gap: contributors touch only skills/, and the master-push bot -# refreshes the mirror after merge. -for skill_md in $(git ls-files 'skills/*/SKILL.md'); do - rel="${skill_md#skills/}" - if [ ! -f "$ROOT_DIR/plugins/last9/skills/$rel" ]; then - if [ "${SKIP_MIRROR_PARITY:-0}" = "1" ]; then continue; fi - echo "::error::Grok fallback mirror missing canonical skill: $rel" >&2 - exit 1 - fi - cmp -s "$skill_md" "$ROOT_DIR/plugins/last9/skills/$rel" || { - if [ "${SKIP_MIRROR_PARITY:-0}" = "1" ]; then continue; fi - echo "::error::Grok fallback mirror drifted from canonical: $rel" >&2 - exit 1 - } -done -if [ "${SKIP_MIRROR_PARITY:-0}" != "1" ]; then - for extra in $(git ls-files 'plugins/last9/skills/*'); do - rel="${extra#plugins/last9/skills/}" - [ -f "$ROOT_DIR/skills/$rel" ] || { - echo "::error::Grok fallback mirror has orphaned file: $extra" >&2 - exit 1 - } - done -fi - # 1. Frontmatter name must equal the skill directory name. Extract from the # YAML frontmatter block only (between the first two --- delimiters), so a # body line starting "name: " cannot false-fail the gate. @@ -113,13 +78,8 @@ check_manifest() { check_manifest .claude-plugin/marketplace.json yes check_manifest .agents/plugins/marketplace.json no -check_manifest .grok-plugin/marketplace.json no -# 2b. The sanctioned Grok mirror's own manifest must point at the mirror. -jq -e '.plugins[0].source == "./plugins/last9"' .grok-plugin/marketplace.json >/dev/null || { - echo "::error::.grok-plugin/marketplace.json must source ./plugins/last9 (the sanctioned mirror)" >&2 - exit 1 -} +# 2b. The Codex root plugin manifest must keep pointing at the canonical tree. jq -e '.skills == "./skills/"' .codex-plugin/plugin.json >/dev/null || { echo "::error::.codex-plugin/plugin.json skills pointer drifted" >&2 exit 1