Skip to content

fix: detect SKILL body routing markers #236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
drvoss wants to merge 1 commit into
base: main
Choose a base branch
from
+33 −5
Open

fix: detect SKILL body routing markers #236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions internal/scoring/scoring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ func (h HeuristicScorer) Score(sk *skill.Skill) *ScoreResult {
desc := sk.Frontmatter.Description
name := sk.Frontmatter.Name
trimmedDesc := strings.TrimSpace(desc)
trimmedBody := strings.TrimSpace(sk.Body)
searchText := strings.TrimSpace(trimmedDesc + "\n" + trimmedBody)
result.DescriptionLen = utf8.RuneCountInString(trimmedDesc)

// Short-circuit: description >1024 chars is Invalid
Expand All @@ -147,12 +149,15 @@ func (h HeuristicScorer) Score(sk *skill.Skill) *ScoreResult {
return result
}

result.HasTriggers = containsAny(trimmedDesc, triggerPatterns)
result.HasTriggers = containsAny(searchText, triggerPatterns)
result.TriggerCount = countPhrasesAfterPattern(trimmedDesc, "USE FOR:") +
countPhrasesAfterPattern(trimmedDesc, "WHEN:")
countPhrasesAfterPattern(trimmedDesc, "WHEN:") +
countPhrasesAfterPattern(trimmedBody, "USE FOR:") +
countPhrasesAfterPattern(trimmedBody, "WHEN:")

result.HasAntiTriggers = containsAny(trimmedDesc, antiTriggerPatterns)
result.AntiTriggerCount = countPhrasesAfterPattern(trimmedDesc, "DO NOT USE FOR:")
result.HasAntiTriggers = containsAny(searchText, antiTriggerPatterns)
result.AntiTriggerCount = countPhrasesAfterPattern(trimmedDesc, "DO NOT USE FOR:") +
countPhrasesAfterPattern(trimmedBody, "DO NOT USE FOR:")

// Context-dependent anti-trigger risk (Issue #78)
// Severity scales with catalog size: High (≥15) → error, Moderate (6-14) → warning.
Expand All @@ -178,7 +183,7 @@ func (h HeuristicScorer) Score(sk *skill.Skill) *ScoreResult {
}
}

result.HasRoutingClarity = containsAny(trimmedDesc, routingClarityPatterns)
result.HasRoutingClarity = containsAny(searchText, routingClarityPatterns)

validateName(name, result)
validateDescriptionLength(result.DescriptionLen, result)
Expand Down
23 changes: 23 additions & 0 deletions internal/scoring/scoring_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,29 @@ DO NOT USE FOR: deployments.`
require.Equal(t, 4, result.TriggerCount)
}

func TestHeuristicScorer_DetectsSectionsInBody(t *testing.T) {
desc := strings.Repeat("This skill documents compliance behavior for authoring checks. ", 4)
sk := mkSkill("test-skill", desc)
sk.Body = `
**UTILITY SKILL** - Test skill for compliance scoring.

USE FOR: testing waza compliance scoring, reproducing trigger detection bugs.

DO NOT USE FOR: production use, unrelated tasks.

INVOKES: internal validators.
`

result := (&HeuristicScorer{}).Score(sk)

require.Equal(t, AdherenceHigh, result.Level)
require.True(t, result.HasTriggers)
require.True(t, result.HasAntiTriggers)
require.True(t, result.HasRoutingClarity)
require.Equal(t, 2, result.TriggerCount)
require.Equal(t, 2, result.AntiTriggerCount)
}

// Test #74: Invalid adherence level for >1024 char descriptions
func TestInvalidAdherenceLevel(t *testing.T) {
// Test that Invalid level exists and ranks below Low
Expand Down