diff --git a/SKILL.md b/SKILL.md index 8104511..6a309a5 100644 --- a/SKILL.md +++ b/SKILL.md @@ -151,6 +151,9 @@ Template (with routing clarity for High score): --- name: skill-name description: "**WORKFLOW SKILL** — [ACTION VERB] [UNIQUE_DOMAIN]. [Clarifying sentence]. WHEN: \"[phrase1]\", \"[phrase2]\", \"[phrase3]\". INVOKES: [tools/MCP servers used]. FOR SINGLE OPERATIONS: [when to bypass this skill]." +allowed-tools: shell, read, write # optional: auto-allowed tools +user-invocable: true # optional: default true +disable-model-invocation: false # optional: default false --- ``` @@ -267,6 +270,19 @@ description: "**WORKFLOW SKILL** — Extract, rotate, merge, and split PDF files See [references/examples.md](references/examples.md) for more before/after transformations. +### Reference-Only Pattern + +Use `user-invocable: false` + `disable-model-invocation: true` to create a skill that acts as a shared reference file (loaded from disk, never invoked): + +```yaml +--- +name: team-conventions +description: "Shared coding conventions loaded as context." +user-invocable: false +disable-model-invocation: true +--- +``` + ## Commit Messages ``` diff --git a/references/scoring.md b/references/scoring.md index 6aa07ef..1b13cec 100644 --- a/references/scoring.md +++ b/references/scoring.md @@ -107,7 +107,7 @@ These checks are programmatic and run via `npm run tokens -- score`. They valida | Check | What it validates | Spec rule | |-------|-------------------|-----------| | `spec-frontmatter` | YAML frontmatter exists, `name` and `description` present | Required fields | -| `spec-allowed-fields` | No unknown fields (only `name`, `description`, `license`, `allowed-tools`, `metadata`, `compatibility`) | Field allowlist | +| `spec-allowed-fields` | No unknown fields (only `name`, `description`, `license`, `allowed-tools`, `metadata`, `compatibility`, `user-invocable`, `disable-model-invocation`) | Field allowlist | | `spec-name` | Lowercase, ≤64 chars, no leading/trailing `-`, no `--`, alphanumeric + hyphens only | Name constraints | | `spec-dir-match` | Directory name matches skill `name` field | Directory = name | | `spec-description` | Non-empty, ≤1024 characters | Description constraints | @@ -115,6 +115,10 @@ These checks are programmatic and run via `npm run tokens -- score`. They valida | `spec-license` | Recommends adding `license` field | Optional but strongly recommended | | `spec-version` | Recommends adding `metadata.version` | Optional but strongly recommended | | `spec-security` | No XML angle brackets (`< >`) in frontmatter; name does not use reserved prefixes (`claude-`, `anthropic-`) | Security restrictions ([Anthropic guide](https://resources.anthropic.com/hubfs/The-Complete-Guide-to-Building-Skill-for-Claude.pdf), p31) | +| `copilot-user-invocable` | If present, must be boolean (`true`/`false`) | Copilot CLI extension | +| `copilot-disable-model-invocation` | If present, must be boolean (`true`/`false`) | Copilot CLI extension | +| `copilot-allowed-tools` | If present, must be non-empty comma-separated list | Copilot CLI extension | +| `copilot-reference-only-pattern` | Detects `user-invocable=false` + `disable-model-invocation=true` | Advisory (Copilot CLI) | ## Rule-Based Checks @@ -196,6 +200,29 @@ Optional field documenting: - Supported frameworks - Prerequisites +### Copilot CLI Extension Fields + +These fields are part of the Copilot skill frontmatter schema (not yet documented publicly; not part of the agentskills.io spec) and validated by Sensei when present: + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `allowed-tools` | string | — | Comma-separated list of auto-allowed tools while skill is active | +| `user-invocable` | boolean | `true` | Whether users can invoke via `/skill-name` | +| `disable-model-invocation` | boolean | `false` | Prevent the model from invoking this skill | + +#### Reference-Only Pattern + +When `user-invocable: false` AND `disable-model-invocation: true`, the skill becomes a **reference-only file** — loaded from disk but never invoked by user or model. Useful for shared context, configuration, or conventions. + +```yaml +--- +name: team-conventions +description: "Shared coding conventions for the team." +user-invocable: false +disable-model-invocation: true +--- +``` + ## Scoring Algorithm ```python diff --git a/scripts/package-lock.json b/scripts/package-lock.json index 5efd25f..27aeca3 100644 --- a/scripts/package-lock.json +++ b/scripts/package-lock.json @@ -963,6 +963,7 @@ "integrity": "sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -2653,6 +2654,7 @@ "integrity": "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", diff --git a/scripts/src/tokens/commands/score.test.ts b/scripts/src/tokens/commands/score.test.ts index cbe4c92..3e8e9bc 100644 --- a/scripts/src/tokens/commands/score.test.ts +++ b/scripts/src/tokens/commands/score.test.ts @@ -21,7 +21,10 @@ import { checkDescriptionCompliance, checkCompatibilityCompliance, checkLicenseRecommendation, - checkVersionRecommendation + checkVersionRecommendation, + checkBooleanField, + checkAllowedToolsFormat, + checkReferenceOnlyPattern } from './score.js'; describe('checkModuleCount', () => { @@ -375,11 +378,155 @@ describe('checkAllowedFields', () => { expect(result.message).toContain('author'); }); - it('allows allowed-tools field', () => { - const fields = { name: 'x', description: 'y', 'allowed-tools': 'Bash Read' }; + it('allows user-invocable field', () => { + const fields = { name: 'x', description: 'y', 'user-invocable': 'false' }; const result = checkAllowedFields(fields); expect(result.status).toBe('ok'); }); + + it('allows disable-model-invocation field', () => { + const fields = { name: 'x', description: 'y', 'disable-model-invocation': 'true' }; + const result = checkAllowedFields(fields); + expect(result.status).toBe('ok'); + }); + + it('allows all Copilot CLI fields together', () => { + const fields = { + name: 'x', description: 'y', 'allowed-tools': 'Bash', + 'user-invocable': 'true', 'disable-model-invocation': 'false' + }; + const result = checkAllowedFields(fields); + expect(result.status).toBe('ok'); + }); +}); + +// --------------------------------------------------------------------------- +// Copilot CLI Extension Field Checks +// --------------------------------------------------------------------------- + +describe('checkBooleanField', () => { + it('passes when field is not present', () => { + const result = checkBooleanField('user-invocable', undefined); + expect(result.status).toBe('ok'); + expect(result.message).toContain('not present'); + }); + + it('passes for "true"', () => { + const result = checkBooleanField('user-invocable', 'true'); + expect(result.status).toBe('ok'); + expect(result.message).toContain('true'); + }); + + it('passes for "false"', () => { + const result = checkBooleanField('disable-model-invocation', 'false'); + expect(result.status).toBe('ok'); + expect(result.message).toContain('false'); + }); + + it('passes for "yes"', () => { + const result = checkBooleanField('user-invocable', 'yes'); + expect(result.status).toBe('ok'); + }); + + it('passes for "no"', () => { + const result = checkBooleanField('user-invocable', 'no'); + expect(result.status).toBe('ok'); + }); + + it('warns on non-boolean value', () => { + const result = checkBooleanField('user-invocable', 'maybe'); + expect(result.status).toBe('warning'); + expect(result.message).toContain('boolean'); + expect(result.message).toContain('maybe'); + }); + + it('warns on numeric value', () => { + const result = checkBooleanField('disable-model-invocation', '1'); + expect(result.status).toBe('warning'); + }); + + it('handles quoted boolean values', () => { + const result = checkBooleanField('user-invocable', '"true"'); + expect(result.status).toBe('ok'); + }); + + it('uses correct check name based on field', () => { + const r1 = checkBooleanField('user-invocable', 'true'); + expect(r1.name).toBe('copilot-user-invocable'); + + const r2 = checkBooleanField('disable-model-invocation', 'false'); + expect(r2.name).toBe('copilot-disable-model-invocation'); + }); +}); + +describe('checkAllowedToolsFormat', () => { + it('passes when not present', () => { + const result = checkAllowedToolsFormat(undefined); + expect(result.status).toBe('ok'); + expect(result.name).toBe('copilot-allowed-tools'); + }); + + it('passes for valid comma-separated list', () => { + const result = checkAllowedToolsFormat('Bash, Read, Write'); + expect(result.status).toBe('ok'); + expect(result.message).toContain('Bash, Read, Write'); + }); + + it('passes for single tool', () => { + const result = checkAllowedToolsFormat('Bash'); + expect(result.status).toBe('ok'); + }); + + it('warns on empty string', () => { + const result = checkAllowedToolsFormat(''); + expect(result.status).toBe('warning'); + expect(result.message).toContain('empty'); + }); + + it('warns on whitespace-only', () => { + const result = checkAllowedToolsFormat(' '); + expect(result.status).toBe('warning'); + }); +}); + +describe('checkReferenceOnlyPattern', () => { + it('returns ok when both fields absent (defaults)', () => { + const result = checkReferenceOnlyPattern(undefined, undefined); + expect(result.status).toBe('ok'); + expect(result.name).toBe('copilot-reference-only-pattern'); + expect(result.message).toContain('defaults apply'); + }); + + it('detects reference-only pattern', () => { + const result = checkReferenceOnlyPattern('false', 'true'); + expect(result.status).toBe('optimal'); + expect(result.message).toContain('Reference-only'); + }); + + it('detects reference-only with yes/no values', () => { + const result = checkReferenceOnlyPattern('no', 'yes'); + expect(result.status).toBe('optimal'); + expect(result.message).toContain('Reference-only'); + }); + + it('reports user-invocable=false only', () => { + const result = checkReferenceOnlyPattern('false', undefined); + expect(result.status).toBe('ok'); + expect(result.message).toContain('not user-invocable'); + expect(result.message).toContain('model-invoked'); + }); + + it('reports disable-model-invocation=true only', () => { + const result = checkReferenceOnlyPattern(undefined, 'true'); + expect(result.status).toBe('ok'); + expect(result.message).toContain('user-invocable only'); + }); + + it('returns ok for standard defaults (both true/false)', () => { + const result = checkReferenceOnlyPattern('true', 'false'); + expect(result.status).toBe('ok'); + expect(result.message).toContain('Standard invocable'); + }); }); describe('checkNameCompliance', () => { diff --git a/scripts/src/tokens/commands/score.ts b/scripts/src/tokens/commands/score.ts index c4ab2bc..52795ad 100644 --- a/scripts/src/tokens/commands/score.ts +++ b/scripts/src/tokens/commands/score.ts @@ -3,6 +3,7 @@ * and agentskills.io specification compliance checks. * * Spec reference: https://agentskills.io/specification + * Copilot CLI fields: not yet documented publicly (see skillFrontmatterSchema in copilot-agent-runtime) */ import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs'; @@ -18,9 +19,10 @@ const MAX_DESCRIPTION_LENGTH = 1024; /** Max compatibility field length per agentskills.io spec */ const MAX_COMPATIBILITY_LENGTH = 500; -/** Allowed frontmatter fields per agentskills.io spec */ +/** Allowed frontmatter fields per agentskills.io spec + Copilot CLI extensions */ const ALLOWED_FIELDS = new Set([ - 'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility' + 'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility', + 'user-invocable', 'disable-model-invocation' ]); /** Recursively list all files in a directory (Node 18 compatible) */ @@ -60,6 +62,9 @@ interface ParsedFrontmatter { readonly name?: string; readonly description?: string; readonly compatibility?: string; + readonly userInvocable?: string; + readonly disableModelInvocation?: string; + readonly allowedTools?: string; } /** @@ -100,6 +105,9 @@ export function parseFrontmatter(content: string): ParsedFrontmatter | null { name: typeof fields['name'] === 'string' ? fields['name'] : undefined, description: typeof fields['description'] === 'string' ? fields['description'] : undefined, compatibility: typeof fields['compatibility'] === 'string' ? fields['compatibility'] : undefined, + userInvocable: typeof fields['user-invocable'] === 'string' ? fields['user-invocable'] : undefined, + disableModelInvocation: typeof fields['disable-model-invocation'] === 'string' ? fields['disable-model-invocation'] : undefined, + allowedTools: typeof fields['allowed-tools'] === 'string' ? fields['allowed-tools'] : undefined, }; } @@ -334,6 +342,129 @@ export function checkVersionRecommendation(fields: Record): Adv }; } +/** Valid boolean string values in YAML frontmatter */ +const YAML_BOOLEAN_VALUES = new Set([ + 'true', 'false', 'yes', 'no', 'on', 'off' +]); + +/** + * Copilot CLI Check: Validate that a frontmatter field is a boolean value when present. + */ +export function checkBooleanField(fieldName: string, value: string | undefined): AdvisoryCheck { + const checkName = `copilot-${fieldName}`; + + if (value === undefined) { + return { + name: checkName, + status: 'ok', + message: `${fieldName} not present (optional, uses default)` + }; + } + + const normalized = value.trim().toLowerCase(); + // Strip surrounding quotes that the YAML parser may leave + const unquoted = normalized.replace(/^['"]|['"]$/g, ''); + + if (!YAML_BOOLEAN_VALUES.has(unquoted)) { + return { + name: checkName, + status: 'warning', + message: `${fieldName} should be a boolean (true/false), got: "${value}"`, + evidence: 'Copilot CLI skill frontmatter: boolean field' + }; + } + + return { + name: checkName, + status: 'ok', + message: `${fieldName} = ${unquoted}` + }; +} + +/** + * Copilot CLI Check: Validate allowed-tools format (comma-separated list) when present. + */ +export function checkAllowedToolsFormat(value: string | undefined): AdvisoryCheck { + if (value === undefined) { + return { + name: 'copilot-allowed-tools', + status: 'ok', + message: 'allowed-tools not present (optional)' + }; + } + + const trimmed = value.trim(); + if (!trimmed) { + return { + name: 'copilot-allowed-tools', + status: 'warning', + message: 'allowed-tools is empty — omit the field or specify tool names', + evidence: 'Copilot CLI skill frontmatter: comma-separated list of tool names' + }; + } + + return { + name: 'copilot-allowed-tools', + status: 'ok', + message: `allowed-tools: ${trimmed}` + }; +} + +/** + * Copilot CLI Check: Detect "reference-only" skill pattern. + * When user-invocable=false AND disable-model-invocation=true, the skill + * is effectively a reference file loaded from disk but never invoked. + */ +export function checkReferenceOnlyPattern( + userInvocable: string | undefined, + disableModelInvocation: string | undefined +): AdvisoryCheck { + if (userInvocable === undefined && disableModelInvocation === undefined) { + return { + name: 'copilot-reference-only-pattern', + status: 'ok', + message: 'Standard invocable skill (defaults apply)' + }; + } + + const uiNorm = (userInvocable ?? 'true').trim().toLowerCase().replace(/^['"]|['"]$/g, ''); + const dmNorm = (disableModelInvocation ?? 'false').trim().toLowerCase().replace(/^['"]|['"]$/g, ''); + + const isNotUserInvocable = uiNorm === 'false' || uiNorm === 'no' || uiNorm === 'off'; + const isModelDisabled = dmNorm === 'true' || dmNorm === 'yes' || dmNorm === 'on'; + + if (isNotUserInvocable && isModelDisabled) { + return { + name: 'copilot-reference-only-pattern', + status: 'optimal', + message: 'Reference-only skill detected (user-invocable=false + disable-model-invocation=true) — loaded from disk but never invoked directly', + evidence: 'Copilot CLI: this combination creates a shared context/config file' + }; + } + + if (isNotUserInvocable) { + return { + name: 'copilot-reference-only-pattern', + status: 'ok', + message: 'Skill is not user-invocable but can be model-invoked' + }; + } + + if (isModelDisabled) { + return { + name: 'copilot-reference-only-pattern', + status: 'ok', + message: 'Skill is user-invocable only (model cannot invoke)' + }; + } + + return { + name: 'copilot-reference-only-pattern', + status: 'ok', + message: 'Standard invocable skill' + }; +} + // --------------------------------------------------------------------------- // Advisory Checks (Sensei-original, SkillsBench-informed) // --------------------------------------------------------------------------- @@ -821,6 +952,11 @@ export function scoreSkill(skillDir: string): ScoringResult { specChecks.push(checkCompatibilityCompliance(fm.compatibility)); specChecks.push(checkLicenseRecommendation(fm.fields)); specChecks.push(checkVersionRecommendation(fm.fields)); + // Copilot CLI extension fields + specChecks.push(checkBooleanField('user-invocable', fm.userInvocable)); + specChecks.push(checkBooleanField('disable-model-invocation', fm.disableModelInvocation)); + specChecks.push(checkAllowedToolsFormat(fm.allowedTools)); + specChecks.push(checkReferenceOnlyPattern(fm.userInvocable, fm.disableModelInvocation)); } return {