diff --git a/checker/raw_result.go b/checker/raw_result.go index bf5132b7e45..a8f8f074278 100644 --- a/checker/raw_result.go +++ b/checker/raw_result.go @@ -29,6 +29,7 @@ import ( type RawResults struct { BinaryArtifactResults BinaryArtifactData BranchProtectionResults BranchProtectionsData + ChangelogResults ChangelogData CIIBestPracticesResults CIIBestPracticesData CITestResults CITestData CodeReviewResults CodeReviewData @@ -323,6 +324,14 @@ type DependencyUpdateToolData struct { Tools []Tool } +// ChangelogData contains the raw results for the Changelog check. +type ChangelogData struct { + ChangelogFiles []File + ChangelogVersions []string + ReleasesWithChangelog int + TotalReleases int +} + // WebhooksData contains the raw results // for the Webhook check. type WebhooksData struct { diff --git a/checks/all_checks.go b/checks/all_checks.go index a67298be27b..0a8570cca0f 100644 --- a/checks/all_checks.go +++ b/checks/all_checks.go @@ -39,6 +39,7 @@ func getAll(overrideExperimental bool) checker.CheckNameToFnMap { // TODO: remove this check when v6 is released delete(possibleChecks, CheckWebHooks) delete(possibleChecks, CheckSBOM) + delete(possibleChecks, CheckChangelog) } return possibleChecks diff --git a/checks/changelog.go b/checks/changelog.go new file mode 100644 index 00000000000..001e5475384 --- /dev/null +++ b/checks/changelog.go @@ -0,0 +1,75 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checks + +import ( + "os" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/checks/evaluation" + "github.com/ossf/scorecard/v5/checks/raw" + sce "github.com/ossf/scorecard/v5/errors" + "github.com/ossf/scorecard/v5/probes" + "github.com/ossf/scorecard/v5/probes/zrunner" +) + +// CheckChangelog is the registered name for Changelog. +const CheckChangelog = "Changelog" + +//nolint:gochecknoinits +func init() { + supportedRequestTypes := []checker.RequestType{ + checker.CommitBased, + checker.FileBased, + } + if err := registerCheck(CheckChangelog, Changelog, supportedRequestTypes); err != nil { + // this should never happen + panic(err) + } +} + +// Changelog runs the Changelog check. +func Changelog(c *checker.CheckRequest) checker.CheckResult { + _, enabled := os.LookupEnv("SCORECARD_EXPERIMENTAL") + if !enabled { + c.Dlogger.Warn(&checker.LogMessage{ + Text: "SCORECARD_EXPERIMENTAL is not set, not running the Changelog check", + }) + + e := sce.WithMessage(sce.ErrUnsupportedCheck, "SCORECARD_EXPERIMENTAL is not set, not running the Changelog check") + return checker.CreateRuntimeErrorResult(CheckChangelog, e) + } + + rawData, err := raw.Changelog(c) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckChangelog, e) + } + + // Set the raw results. + pRawResults := getRawResults(c) + pRawResults.ChangelogResults = rawData + + // Evaluate the probes. + findings, err := zrunner.Run(pRawResults, probes.Changelog) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckChangelog, e) + } + + ret := evaluation.Changelog(CheckChangelog, findings, c.Dlogger) + ret.Findings = findings + return ret +} diff --git a/checks/changelog_test.go b/checks/changelog_test.go new file mode 100644 index 00000000000..e5f84fe04f3 --- /dev/null +++ b/checks/changelog_test.go @@ -0,0 +1,150 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checks + +import ( + "io" + "strings" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/clients" + mockrepo "github.com/ossf/scorecard/v5/clients/mockclients" + scut "github.com/ossf/scorecard/v5/utests" +) + +const testChangelogContent = `# Changelog +## [2.0.0] - 2024-01-15 +### Added +- Feature +## [1.0.0] - 2023-06-01 +### Fixed +- Bug +` + +func TestChangelog(t *testing.T) { + tests := []struct { + name string + releases []clients.Release + files []string + expected scut.TestReturn + }{ + { + name: "changelog covers release", + releases: []clients.Release{ + {TagName: "v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: scut.TestReturn{ + Score: checker.MaxResultScore, + NumberOfInfo: 2, + }, + }, + { + name: "changelog file only, no releases", + releases: []clients.Release{}, + files: []string{"CHANGELOG.md"}, + expected: scut.TestReturn{ + Score: 3, // 3 for file, 0 for releases (not applicable) + NumberOfInfo: 1, + NumberOfDebug: 1, + }, + }, + { + name: "no changelog file, no releases", + releases: []clients.Release{}, + files: []string{}, + expected: scut.TestReturn{ + Score: checker.InconclusiveResultScore, + NumberOfWarn: 1, + NumberOfDebug: 1, + }, + }, + { + name: "changelog file, release not in changelog", + releases: []clients.Release{ + {TagName: "v3.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: scut.TestReturn{ + Score: 3, // 3 for file + 0 for releases + NumberOfInfo: 1, + NumberOfWarn: 1, + }, + }, + { + name: "no changelog file, releases have body text. 10/10", + releases: []clients.Release{ + {TagName: "v2.0.0", Body: "## Changes\n- Feature A\n"}, + {TagName: "v1.0.0", Body: "Initial release\n"}, + }, + files: []string{}, + expected: scut.TestReturn{ + Score: checker.MaxResultScore, + NumberOfWarn: 1, + NumberOfInfo: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("SCORECARD_EXPERIMENTAL", "true") + ctrl := gomock.NewController(t) + mockRepo := mockrepo.NewMockRepoClient(ctrl) + + mockRepo.EXPECT().ListReleases().DoAndReturn( + func() ([]clients.Release, error) { + return tt.releases, nil + }, + ).MaxTimes(1) + + mockRepo.EXPECT().ListFiles(gomock.Any()).DoAndReturn( + func(predicate func(string) (bool, error)) ([]string, error) { + var matched []string + for _, f := range tt.files { + ok, err := predicate(f) + if err != nil { + return nil, err + } + if ok { + matched = append(matched, f) + } + } + return matched, nil + }, + ).AnyTimes() + + if len(tt.files) > 0 { + mockRepo.EXPECT().GetFileReader(gomock.Any()).DoAndReturn( + func(path string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(testChangelogContent)), nil + }, + ).MaxTimes(1) + } + + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ + RepoClient: mockRepo, + Ctx: t.Context(), + Dlogger: &dl, + } + res := Changelog(&req) + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) + }) + } +} diff --git a/checks/evaluation/changelog.go b/checks/evaluation/changelog.go new file mode 100644 index 00000000000..a45863826ea --- /dev/null +++ b/checks/evaluation/changelog.go @@ -0,0 +1,146 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package evaluation + +import ( + "fmt" + "math" + "strconv" + + "github.com/ossf/scorecard/v5/checker" + sce "github.com/ossf/scorecard/v5/errors" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/probes/hasChangelogFile" + "github.com/ossf/scorecard/v5/probes/releasesHaveChangelog" +) + +const ( + changelogFileScore = 3 + changelogReleaseScore = 7 +) + +// Changelog applies the score policy for the Changelog check. +// +// Scoring: +// - Path A (has changelog file): 3 pts for file + up to 7 pts proportional +// based on how many of the last 5 releases have entries in the changelog. +// - Path B (no file, has releases with notes): up to 10 pts proportional +// based on how many of the last 5 releases have substantive body text. +// - No file and no releases: inconclusive. +func Changelog(name string, + findings []finding.Finding, + dl checker.DetailLogger, +) checker.CheckResult { + expectedProbes := []string{ + hasChangelogFile.Probe, + releasesHaveChangelog.Probe, + } + + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") + return checker.CreateRuntimeErrorResult(name, e) + } + + score := 0 + hasFile := false + hasReleases := false + var logLevel checker.DetailType + for i := range findings { + f := &findings[i] + switch f.Outcome { + case finding.OutcomeTrue: + logLevel = checker.DetailInfo + switch f.Probe { + case hasChangelogFile.Probe: + if !hasFile { + hasFile = true + score += changelogFileScore + } + case releasesHaveChangelog.Probe: + hasReleases = true + score += releaseScore(f, hasFile) + } + case finding.OutcomeFalse: + logLevel = checker.DetailWarn + if f.Probe == releasesHaveChangelog.Probe { + hasReleases = true + score += releaseScore(f, hasFile) + } + case finding.OutcomeNotApplicable: + logLevel = checker.DetailDebug + default: + continue + } + checker.LogFinding(dl, f, logLevel) + } + + if !hasFile && !hasReleases { + return checker.CreateInconclusiveResult(name, "no changelog file or releases found") + } + + if hasFile { + return checker.CreateResultWithScore(name, "changelog file detected", score) + } + + return checker.CreateResultWithScore(name, + fmt.Sprintf("%d out of %d release(s) have descriptive logs", + releaseCount(findings), releaseTotal(findings)), + score) +} + +// releaseScore returns the proportional score for the release probe. +// When a changelog file exists, releases are worth 7 points. +// When no file exists, releases are worth the full 10 points. +func releaseScore(f *finding.Finding, hasFile bool) int { + maxScore := changelogReleaseScore + if !hasFile { + maxScore = checker.MaxResultScore + } + withChangelog, err := strconv.Atoi(f.Values[releasesHaveChangelog.ReleasesWithChangelogKey]) + if err != nil { + return 0 + } + total, err := strconv.Atoi(f.Values[releasesHaveChangelog.ReleasesTotalKey]) + if err != nil || total == 0 { + return 0 + } + return int(math.Floor(float64(maxScore) * float64(withChangelog) / float64(total))) +} + +func releaseCount(findings []finding.Finding) int { + for i := range findings { + if findings[i].Probe == releasesHaveChangelog.Probe { + n, err := strconv.Atoi(findings[i].Values[releasesHaveChangelog.ReleasesWithChangelogKey]) + if err != nil { + return 0 + } + return n + } + } + return 0 +} + +func releaseTotal(findings []finding.Finding) int { + for i := range findings { + if findings[i].Probe == releasesHaveChangelog.Probe { + n, err := strconv.Atoi(findings[i].Values[releasesHaveChangelog.ReleasesTotalKey]) + if err != nil { + return 0 + } + return n + } + } + return 0 +} diff --git a/checks/evaluation/changelog_test.go b/checks/evaluation/changelog_test.go new file mode 100644 index 00000000000..69ea6116d10 --- /dev/null +++ b/checks/evaluation/changelog_test.go @@ -0,0 +1,185 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package evaluation + +import ( + "testing" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + scut "github.com/ossf/scorecard/v5/utests" +) + +func TestChangelog(t *testing.T) { + t.Parallel() + tests := []struct { + name string + findings []finding.Finding + result scut.TestReturn + }{ + { + name: "Path A: changelog file + all releases covered. 10/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeTrue, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeTrue, + Values: map[string]string{ + "releasesWithChangelog": "5", + "releasesTotal": "5", + }, + }, + }, + result: scut.TestReturn{ + Score: checker.MaxResultScore, // 3 + 7 = 10 + NumberOfInfo: 2, + }, + }, + { + name: "Path A: changelog file + 3/5 releases covered. 7/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeTrue, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeFalse, + Values: map[string]string{ + "releasesWithChangelog": "3", + "releasesTotal": "5", + }, + }, + }, + result: scut.TestReturn{ + Score: changelogFileScore + 4, + NumberOfInfo: 1, + NumberOfWarn: 1, + }, + }, + { + name: "Path A: changelog file + no releases covered. 3/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeTrue, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeFalse, + Values: map[string]string{ + "releasesWithChangelog": "0", + "releasesTotal": "5", + }, + }, + }, + result: scut.TestReturn{ + Score: changelogFileScore, // 3 + 0 = 3 + NumberOfInfo: 1, + NumberOfWarn: 1, + }, + }, + { + name: "Path A: changelog file + no releases. 3/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeTrue, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeNotApplicable, + }, + }, + result: scut.TestReturn{ + Score: changelogFileScore, // 3 + NumberOfInfo: 1, + NumberOfDebug: 1, + }, + }, + { + name: "Path B: no file, all releases have notes. 10/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeFalse, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeTrue, + Values: map[string]string{ + "releasesWithChangelog": "5", + "releasesTotal": "5", + }, + }, + }, + result: scut.TestReturn{ + Score: checker.MaxResultScore, // 10 * 5/5 = 10 + NumberOfWarn: 1, + NumberOfInfo: 1, + }, + }, + { + name: "Path B: no file, 3/5 releases have notes. 6/10", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeFalse, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeFalse, + Values: map[string]string{ + "releasesWithChangelog": "3", + "releasesTotal": "5", + }, + }, + }, + result: scut.TestReturn{ + Score: 6, + NumberOfWarn: 2, + }, + }, + { + name: "Inconclusive: no file, no releases", + findings: []finding.Finding{ + { + Probe: "hasChangelogFile", + Outcome: finding.OutcomeFalse, + }, + { + Probe: "releasesHaveChangelog", + Outcome: finding.OutcomeNotApplicable, + }, + }, + result: scut.TestReturn{ + Score: checker.InconclusiveResultScore, + NumberOfWarn: 1, + NumberOfDebug: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + dl := scut.TestDetailLogger{} + got := Changelog(tt.name, tt.findings, &dl) + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) + }) + } +} diff --git a/checks/raw/changelog.go b/checks/raw/changelog.go new file mode 100644 index 00000000000..454c8c93193 --- /dev/null +++ b/checks/raw/changelog.go @@ -0,0 +1,259 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raw + +import ( + "bufio" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/clients" + "github.com/ossf/scorecard/v5/finding" +) + +// reChangelogFile matches common changelog filenames at the repo root. +// - CHANGELOG, CHANGES: standard changelog names +// - NEWS: common in GNU projects +// - HISTORY: used by some projects for version history +// - RELEASE-NOTES, RELEASE_NOTES: common in Apache/Java projects +// +// Optional extensions: .md, .txt, .rst, .adoc. +var reChangelogFile = regexp.MustCompile( + `(?i)^(changelog|changes|news|history|release[-_]notes)(\.(md|txt|rst|adoc))?$`, +) + +// reVersion extracts semver-like version strings from changelog lines. +// Matches patterns like: +// - ## [1.0.0] - 2024-01-15 (Keep a Changelog) +// - ## 1.0.0 (simple markdown) +// - # 1.0.0 / 2024-01-15 (Ruby style) +// - Version 3.0 (2024-01-15) (GNU NEWS) +// - 3.0.0 (2024-01-15) (Python CHANGES.rst) +// - Release 3.0.0 (Apache style) +// +// The core pattern captures a semver-like string: major.minor with optional .patch +// and optional pre-release suffix (e.g., -rc.1, -beta.2). +var reVersion = regexp.MustCompile( + `(?:^|\s|\[|v)(\d+\.\d+(?:\.\d+)?(?:-[A-Za-z0-9.]+)?)(?:\]|\s|$)`, +) + +const changelogReleaseLookBack = 5 + +// Changelog retrieves the raw data for the Changelog check. +func Changelog(c *checker.CheckRequest) (checker.ChangelogData, error) { + var results checker.ChangelogData + + // Look for changelog files at the top level of the repo. + repoFiles, err := c.RepoClient.ListFiles(func(file string) (bool, error) { + return reChangelogFile.MatchString(file), nil + }) + if err != nil { + return results, fmt.Errorf("RepoClient.ListFiles: %w", err) + } + for _, f := range repoFiles { + results.ChangelogFiles = append(results.ChangelogFiles, checker.File{ + Path: f, + Type: finding.FileTypeSource, + }) + } + + // Read the first changelog file found and extract version strings + // that have substantive content. + if len(results.ChangelogFiles) > 0 { + versions, err := extractVersionsFromFile(c, results.ChangelogFiles[0].Path) + if err != nil { + // Non-fatal: we found the file but couldn't parse it. + if c.Dlogger != nil { + c.Dlogger.Debug(&checker.LogMessage{ + Text: fmt.Sprintf("could not parse changelog: %v", err), + }) + } + } else { + results.ChangelogVersions = versions + } + } + + // Get releases and check which ones are covered by the changelog. + releases, err := c.RepoClient.ListReleases() + if err != nil && !errors.Is(err, clients.ErrUnsupportedFeature) { + return results, fmt.Errorf("RepoClient.ListReleases: %w", err) + } + + versionSet := makeVersionSet(results.ChangelogVersions) + + for i, r := range releases { + if i >= changelogReleaseLookBack { + break + } + results.TotalReleases++ + tag := normalizeVersion(r.TagName) + if versionSet[tag] || hasSubstantiveBody(r.Body) { + results.ReleasesWithChangelog++ + } + } + + return results, nil +} + +// hasSubstantiveBody checks whether a release body has meaningful content +// beyond just whitespace or boilerplate like "Full Changelog: ...". +func hasSubstantiveBody(body string) bool { + for _, line := range strings.Split(body, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + // Skip GitHub auto-generated "Full Changelog" links. + if strings.HasPrefix(trimmed, "**Full Changelog**") { + continue + } + return true + } + return false +} + +// extractVersionsFromFile reads a changelog file and returns all version +// strings that have substantive content below them. +func extractVersionsFromFile(c *checker.CheckRequest, path string) ([]string, error) { + reader, err := c.RepoClient.GetFileReader(path) + if err != nil { + return nil, fmt.Errorf("GetFileReader: %w", err) + } + defer reader.Close() + + return extractVersions(reader), nil +} + +// extractVersions scans a changelog and returns versions that have +// substantive content. A version entry is considered substantive if +// there is at least one non-empty, non-header line between it and +// the next version header (or end of file). +func extractVersions(r io.Reader) []string { + var versions []string + seen := make(map[string]bool) + + var currentVersion string + hasContent := false + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + if v := extractVersionFromLine(line); v != "" { + // We hit a new version header. Finalize the previous one. + if currentVersion != "" && hasContent && !seen[currentVersion] { + seen[currentVersion] = true + versions = append(versions, currentVersion) + } + currentVersion = v + hasContent = false + continue + } + + // Check if this line is substantive content (not empty, not a + // sub-header like "### Added", not an rst underline like "===="). + if currentVersion != "" && !hasContent && isContentLine(trimmed) { + hasContent = true + } + } + + // Finalize the last version entry. + if currentVersion != "" && hasContent && !seen[currentVersion] { + versions = append(versions, currentVersion) + } + + return versions +} + +// extractVersionFromLine returns a normalized version string if the line +// contains a version header, or empty string if not. +func extractVersionFromLine(line string) string { + matches := reVersion.FindStringSubmatch(line) + if len(matches) < 2 { + return "" + } + // Only treat lines that look like headers — they should start with + // a markdown header (#), contain "version"/"release", start with + // the version number, or have brackets around the version. + trimmed := strings.TrimSpace(line) + v := normalizeVersion(matches[1]) + + switch { + case strings.HasPrefix(trimmed, "#"): + return v + case strings.HasPrefix(trimmed, v), strings.HasPrefix(trimmed, "v"+v): + return v + case strings.HasPrefix(strings.ToLower(trimmed), "version"): + return v + case strings.HasPrefix(strings.ToLower(trimmed), "release"): + return v + case strings.Contains(trimmed, "["+matches[1]+"]"): + return v + } + return "" +} + +// isContentLine returns true if the line is substantive content — +// not empty, not a markdown sub-header, not an rst underline. +func isContentLine(trimmed string) bool { + if trimmed == "" { + return false + } + // Markdown sub-headers (### Added, ### Fixed, etc.) + if strings.HasPrefix(trimmed, "#") { + return false + } + // RST underlines (====, ----, ~~~~) + if isRSTUnderline(trimmed) { + return false + } + return true +} + +// isRSTUnderline checks for reStructuredText section underlines. +func isRSTUnderline(s string) bool { + if len(s) < 3 { + return false + } + c := s[0] + if c != '=' && c != '-' && c != '~' && c != '^' && c != '+' { + return false + } + for i := 1; i < len(s); i++ { + if s[i] != c { + return false + } + } + return true +} + +// normalizeVersion strips a leading "v" prefix for comparison. +func normalizeVersion(tag string) string { + return strings.TrimPrefix(strings.TrimSpace(tag), "v") +} + +// makeVersionSet creates a lookup set from a slice of version strings. +func makeVersionSet(versions []string) map[string]bool { + m := make(map[string]bool, len(versions)) + for _, v := range versions { + m[v] = true + } + return m +} diff --git a/checks/raw/changelog_test.go b/checks/raw/changelog_test.go new file mode 100644 index 00000000000..11bcc98cef1 --- /dev/null +++ b/checks/raw/changelog_test.go @@ -0,0 +1,424 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raw + +import ( + "io" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/mock/gomock" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/clients" + mockrepo "github.com/ossf/scorecard/v5/clients/mockclients" + "github.com/ossf/scorecard/v5/finding" + scut "github.com/ossf/scorecard/v5/utests" +) + +func TestHasSubstantiveBody(t *testing.T) { + t.Parallel() + tests := []struct { + name string + body string + expected bool + }{ + {"empty body", "", false}, + {"whitespace only", " \n \n ", false}, + {"auto-generated full changelog link", "**Full Changelog**: https://github.com/org/repo/compare/v1...v2", false}, + {"real content", "## What's Changed\n- Added feature X", true}, + {"single line content", "Initial release", true}, + {"content after boilerplate", "**Full Changelog**: https://link\n\n- But also real notes", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := hasSubstantiveBody(tt.body) + if got != tt.expected { + t.Errorf("hasSubstantiveBody(%q) = %v, want %v", tt.body, got, tt.expected) + } + }) + } +} + +func TestExtractVersions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + content string + expected []string + }{ + { + name: "Keep a Changelog format with content", + content: `# Changelog +## [2.0.0] - 2024-01-15 +### Added +- Feature X +## [1.0.0] - 2023-06-01 +### Fixed +- Bug Y`, + expected: []string{"2.0.0", "1.0.0"}, + }, + { + name: "simple markdown headers with content", + content: `# Changelog +## 3.1.0 +- Feature +## 3.0.0 +- Initial`, + expected: []string{"3.1.0", "3.0.0"}, + }, + { + name: "GNU NEWS format with content", + content: `Version 2.0 (2024-01-15) +* New feature + +Version 1.0 (2023-06-01) +* Initial release`, + expected: []string{"2.0", "1.0"}, + }, + { + name: "Python CHANGES.rst format with content", + content: `3.0.0 (2024-01-15) +=================== +- New feature + +2.0.0 (2023-06-01) +=================== +- Previous`, + expected: []string{"3.0.0", "2.0.0"}, + }, + { + name: "pre-release versions with content", + content: `## [2.0.0-rc.1] - 2024-01-10 +- Release candidate changes +## [1.0.0] - 2023-06-01 +- Initial release`, + expected: []string{"2.0.0-rc.1", "1.0.0"}, + }, + { + name: "no versions found", + content: "This is just a readme with no version info.", + expected: nil, + }, + { + name: "v-prefixed versions with content", + content: `## v1.2.3 +- Something`, + expected: []string{"1.2.3"}, + }, + { + name: "version headers without content are excluded", + content: `## [2.0.0] - 2024-01-15 +## [1.0.0] - 2023-06-01 +- Has content`, + expected: []string{"1.0.0"}, + }, + { + name: "version with only sub-headers but no content is excluded", + content: `## [2.0.0] - 2024-01-15 +### Added +### Fixed +## [1.0.0] - 2023-06-01 +### Added +- Real content here`, + expected: []string{"1.0.0"}, + }, + { + name: "version with only empty lines is excluded", + content: `## [2.0.0] - 2024-01-15 + + +## [1.0.0] - 2023-06-01 +- Content`, + expected: []string{"1.0.0"}, + }, + { + name: "all versions have content", + content: `## [2.0.0] - 2024-01-15 +- Feature A +## [1.0.0] - 2023-06-01 +- Feature B`, + expected: []string{"2.0.0", "1.0.0"}, + }, + { + name: "RST underlines are not content", + content: `2.0.0 (2024-01-15) +=================== + +1.0.0 (2023-06-01) +=================== +- Actual content`, + expected: []string{"1.0.0"}, + }, + { + name: "duplicate versions only counted once", + content: `## [1.0.0] - 2024-01-15 +- First entry +## [1.0.0] - 2024-01-15 +- Duplicate entry`, + expected: []string{"1.0.0"}, + }, + { + name: "version in prose is not a header", + content: `# Changelog +## [2.0.0] - 2024-01-15 +This requires version 3.0 of Python +- Actual change`, + expected: []string{"2.0.0"}, + }, + { + name: "gaming attempt: empty version entries", + content: `## 2.0.0 +## 1.0.0`, + expected: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := extractVersions(strings.NewReader(tt.content)) + if diff := cmp.Diff(tt.expected, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestChangelog(t *testing.T) { + t.Parallel() + + changelogContent := `# Changelog +## [2.0.0] - 2024-01-15 +### Added +- Feature +## [1.0.0] - 2023-06-01 +### Fixed +- Bug +` + + tests := []struct { + name string + releases []clients.Release + files []string + changelogContent string + expected checker.ChangelogData + }{ + { + name: "changelog covers all releases", + changelogContent: changelogContent, + releases: []clients.Release{ + {TagName: "v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"2.0.0", "1.0.0"}, + TotalReleases: 2, + ReleasesWithChangelog: 2, + }, + }, + { + name: "changelog missing a release", + changelogContent: changelogContent, + releases: []clients.Release{ + {TagName: "v3.0.0"}, + {TagName: "v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"2.0.0", "1.0.0"}, + TotalReleases: 3, + ReleasesWithChangelog: 2, + }, + }, + { + name: "changelog file but no releases", + changelogContent: changelogContent, + releases: []clients.Release{}, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"2.0.0", "1.0.0"}, + }, + }, + { + name: "no changelog file, no releases", + releases: []clients.Release{}, + files: []string{}, + expected: checker.ChangelogData{}, + }, + { + name: "RELEASE-NOTES file detected", + changelogContent: "Release 1.0.0\n- Initial\n", + releases: []clients.Release{}, + files: []string{"RELEASE-NOTES.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "RELEASE-NOTES.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"1.0.0"}, + }, + }, + { + name: "empty changelog entries don't count", + changelogContent: `## [2.0.0] +## [1.0.0] +`, + releases: []clients.Release{ + {TagName: "v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + TotalReleases: 2, + ReleasesWithChangelog: 0, + }, + }, + { + name: "release body counts as changelog even without file entry", + changelogContent: changelogContent, + releases: []clients.Release{ + {TagName: "v3.0.0", Body: "## What's Changed\n- New feature\n"}, + {TagName: "v2.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"2.0.0", "1.0.0"}, + TotalReleases: 2, + ReleasesWithChangelog: 2, // v3.0.0 via body, v2.0.0 via changelog file + }, + }, + { + name: "release body with only auto-generated link does not count", + changelogContent: `## [1.0.0] +- Content +`, + releases: []clients.Release{ + {TagName: "v2.0.0", Body: "**Full Changelog**: https://github.com/org/repo/compare/v1.0.0...v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"1.0.0"}, + TotalReleases: 2, + ReleasesWithChangelog: 1, // only v1.0.0 via changelog file + }, + }, + { + name: "no changelog file but releases have bodies", + releases: []clients.Release{ + {TagName: "v2.0.0", Body: "## Changes\n- Feature A\n"}, + {TagName: "v1.0.0", Body: "Initial release\n"}, + }, + files: []string{}, + expected: checker.ChangelogData{ + TotalReleases: 2, + ReleasesWithChangelog: 2, + }, + }, + { + name: "lookback limited to 5 releases", + changelogContent: changelogContent, + releases: []clients.Release{ + {TagName: "v6.0.0"}, + {TagName: "v5.0.0"}, + {TagName: "v4.0.0"}, + {TagName: "v3.0.0"}, + {TagName: "v2.0.0"}, + {TagName: "v1.0.0"}, + }, + files: []string{"CHANGELOG.md"}, + expected: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + ChangelogVersions: []string{"2.0.0", "1.0.0"}, + TotalReleases: 5, + ReleasesWithChangelog: 1, // only v2.0.0 is in the changelog within lookback + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + mockRepo := mockrepo.NewMockRepoClient(ctrl) + + mockRepo.EXPECT().ListReleases().DoAndReturn( + func() ([]clients.Release, error) { + return tt.releases, nil + }, + ).MaxTimes(1) + + mockRepo.EXPECT().ListFiles(gomock.Any()).DoAndReturn( + func(predicate func(string) (bool, error)) ([]string, error) { + var matched []string + for _, f := range tt.files { + ok, err := predicate(f) + if err != nil { + return nil, err + } + if ok { + matched = append(matched, f) + } + } + return matched, nil + }, + ).AnyTimes() + + if tt.changelogContent != "" { + mockRepo.EXPECT().GetFileReader(gomock.Any()).DoAndReturn( + func(path string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(tt.changelogContent)), nil + }, + ).MaxTimes(1) + } + + dl := scut.TestDetailLogger{} + req := checker.CheckRequest{ + RepoClient: mockRepo, + Ctx: t.Context(), + Dlogger: &dl, + } + res, err := Changelog(&req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(res, tt.expected) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.expected, res)) + } + }) + } +} diff --git a/clients/githubrepo/releases.go b/clients/githubrepo/releases.go index 96ee6151031..4beb9173800 100644 --- a/clients/githubrepo/releases.go +++ b/clients/githubrepo/releases.go @@ -73,6 +73,7 @@ func releasesFrom(data []*github.RepositoryRelease) []clients.Release { TagName: r.GetTagName(), URL: r.GetURL(), TargetCommitish: r.GetTargetCommitish(), + Body: r.GetBody(), } for _, a := range r.Assets { release.Assets = append(release.Assets, clients.ReleaseAsset{ diff --git a/clients/gitlabrepo/releases.go b/clients/gitlabrepo/releases.go index f53e93075b7..7c1ac83ad01 100644 --- a/clients/gitlabrepo/releases.go +++ b/clients/gitlabrepo/releases.go @@ -71,6 +71,7 @@ func releasesFrom(data []*gitlab.Release) []clients.Release { release := clients.Release{ TagName: r.TagName, TargetCommitish: r.CommitPath, + Body: r.Description, } if len(r.Assets.Links) > 0 { release.URL = r.Assets.Links[0].DirectAssetURL diff --git a/clients/release.go b/clients/release.go index 32d47a0f2ec..a9a25406891 100644 --- a/clients/release.go +++ b/clients/release.go @@ -19,6 +19,7 @@ type Release struct { TagName string URL string TargetCommitish string + Body string Assets []ReleaseAsset } diff --git a/docs/checks.md b/docs/checks.md index 4b1173f44d8..67f1d416e94 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -189,6 +189,45 @@ However, note that in those overlapping cases, Scorecard can only report what it **Remediation steps** - Sign up for the [OpenSSF Best Practices program](https://www.bestpractices.dev/). +## Changelog + +Risk: `Low` (possibly missing context on release changes) + +This check tries to determine if the project maintains a changelog file and +whether releases include changelog entries. A changelog provides a human-readable +log of notable changes for each release, showing project maturity and helping +users understand what has changed between versions. + +This aligns with [OSPS-BR-04](https://github.com/ossf/security-baseline) which +requires that all releases provide a descriptive log of functional and security +modifications. + +The check currently looks for changelog files at the top level of the repository +(e.g. `CHANGELOG.md`, `CHANGES`, `NEWS`, `HISTORY`, `RELEASE-NOTES`) and checks +whether the last 5 releases include changelog-named assets. + +Changelog File Exists (3/10 points): + - A changelog file is found in the repository root. + +Releases Have Changelog Entries (up to 7/10 points): + - Points are awarded proportionally based on how many of the last 5 releases + have corresponding version entries in the changelog file. + - Releases with substantive release notes (GitHub/GitLab release body) also + count, even without a changelog file entry. + - Auto-generated "Full Changelog" links alone do not count. + - If the project has no releases, no points are awarded for this portion. + +Version matching works by extracting version strings from the changelog +(e.g. `## [1.0.0]`, `Version 1.0`, `1.0.0 (date)`) and comparing them +against release tags (with `v` prefix stripped). + + +**Remediation steps** +- Add a CHANGELOG.md file to the root of your repository. +- Follow the format recommended by [Keep a Changelog](https://keepachangelog.com/). +- Ensure the changelog is updated with each release to document notable changes. +- Include a changelog file as a release asset for each release. + ## Code-Review Risk: `High` (unintentional vulnerabilities or possible injection of malicious diff --git a/docs/checks/internal/checks.yaml b/docs/checks/internal/checks.yaml index b7ff5c8c37e..afa5d91724a 100644 --- a/docs/checks/internal/checks.yaml +++ b/docs/checks/internal/checks.yaml @@ -859,6 +859,51 @@ checks: that matches your [SPDX license identifier](https://spdx.org/licenses/). such as `LICENSES/Apache-2.0.txt`. + Changelog: + risk: Low + tags: supply-chain, security, releases + repos: GitHub, GitLab, local + short: Determines if the project maintains a changelog. + description: | + Risk: `Low` (possibly missing context on release changes) + + This check tries to determine if the project maintains a changelog file and + whether releases include changelog entries. A changelog provides a human-readable + log of notable changes for each release, showing project maturity and helping + users understand what has changed between versions. + + This aligns with [OSPS-BR-04](https://github.com/ossf/security-baseline) which + requires that all releases provide a descriptive log of functional and security + modifications. + + The check currently looks for changelog files at the top level of the repository + (e.g. `CHANGELOG.md`, `CHANGES`, `NEWS`, `HISTORY`, `RELEASE-NOTES`) and checks + whether the last 5 releases include changelog-named assets. + + Changelog File Exists (3/10 points): + - A changelog file is found in the repository root. + + Releases Have Changelog Entries (up to 7/10 points): + - Points are awarded proportionally based on how many of the last 5 releases + have corresponding version entries in the changelog file. + - Releases with substantive release notes (GitHub/GitLab release body) also + count, even without a changelog file entry. + - Auto-generated "Full Changelog" links alone do not count. + - If the project has no releases, no points are awarded for this portion. + + Version matching works by extracting version strings from the changelog + (e.g. `## [1.0.0]`, `Version 1.0`, `1.0.0 (date)`) and comparing them + against release tags (with `v` prefix stripped). + remediation: + - >- + Add a CHANGELOG.md file to the root of your repository. + - >- + Follow the format recommended by [Keep a Changelog](https://keepachangelog.com/). + - >- + Ensure the changelog is updated with each release to document notable changes. + - >- + Include a changelog file as a release asset for each release. + Webhooks: risk: Critical tags: security, infrastructure diff --git a/docs/probes.md b/docs/probes.md index 85878020809..febeea7b600 100644 --- a/docs/probes.md +++ b/docs/probes.md @@ -184,6 +184,20 @@ If no fuzzing tool is found, or the project uses a tool we don't detect, one fin If the probe finds no binary files, it returns a single OutcomeFalse. +## hasChangelogFile + +**Lifecycle**: experimental + +**Description**: Check that the project has a changelog file + +**Motivation**: A changelog provides a human-readable log of notable changes for each release. It shows project maturity and helps users understand what has changed between versions. This aligns with OSPS-BR-04 which requires releases to provide a descriptive log of modifications. + +**Implementation**: The implementation checks whether a changelog file (e.g. CHANGELOG.md, CHANGES, NEWS, HISTORY, RELEASE-NOTES) is present at the top level of the repository. + +**Outcomes**: If a changelog file is found, the probe returns OutcomeTrue for each file. +If no changelog file is found, the probe returns a single OutcomeFalse. + + ## hasDangerousWorkflowScriptInjection **Lifecycle**: stable @@ -426,6 +440,21 @@ For each of the last 5 releases, the probe returns OutcomeFalse, if the release If the project has no releases, the probe returns OutcomeNotApplicable. +## releasesHaveChangelog + +**Lifecycle**: experimental + +**Description**: Check that the project's releases are documented in the changelog + +**Motivation**: Releases that are documented in the changelog demonstrate that the project maintains descriptive logs of functional and security modifications per OSPS-BR-04. + +**Implementation**: The implementation reads the changelog file, extracts version strings, and checks whether the last 5 release tags have corresponding entries in the changelog. Releases with substantive release notes (GitHub/GitLab release body) also count. + +**Outcomes**: If all recent releases have changelog entries, the probe returns OutcomeTrue. +If no releases are found, the probe returns OutcomeNotApplicable. +If some or no releases have changelog entries, the probe returns OutcomeFalse. + + ## releasesHaveProvenance **Lifecycle**: stable diff --git a/internal/checknames/checknames.go b/internal/checknames/checknames.go index aaf6bd48327..562af052ccc 100644 --- a/internal/checknames/checknames.go +++ b/internal/checknames/checknames.go @@ -20,6 +20,7 @@ type CheckName = string const ( BinaryArtifacts CheckName = "Binary-Artifacts" BranchProtection CheckName = "Branch-Protection" + Changelog CheckName = "Changelog" CIIBestPractices CheckName = "CII-Best-Practices" CITests CheckName = "CI-Tests" CodeReview CheckName = "Code-Review" @@ -43,6 +44,7 @@ const ( var AllValidChecks []string = []string{ BinaryArtifacts, BranchProtection, + Changelog, CIIBestPractices, CITests, CodeReview, diff --git a/probes/entries.go b/probes/entries.go index 2a8e5c07b68..de1bc5c4bf0 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -30,6 +30,7 @@ import ( "github.com/ossf/scorecard/v5/probes/dismissesStaleReviews" "github.com/ossf/scorecard/v5/probes/fuzzed" "github.com/ossf/scorecard/v5/probes/hasBinaryArtifacts" + "github.com/ossf/scorecard/v5/probes/hasChangelogFile" "github.com/ossf/scorecard/v5/probes/hasDangerousWorkflowScriptInjection" "github.com/ossf/scorecard/v5/probes/hasDangerousWorkflowUntrustedCheckout" "github.com/ossf/scorecard/v5/probes/hasFSFOrOSIApprovedLicense" @@ -47,6 +48,7 @@ import ( "github.com/ossf/scorecard/v5/probes/packagedWithAutomatedWorkflow" "github.com/ossf/scorecard/v5/probes/pinsDependencies" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" + "github.com/ossf/scorecard/v5/probes/releasesHaveChangelog" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" "github.com/ossf/scorecard/v5/probes/requiresApproversForPullRequests" @@ -89,6 +91,10 @@ var ( DependencyToolUpdates = []ProbeImpl{ dependencyUpdateToolConfigured.Run, } + Changelog = []ProbeImpl{ + hasChangelogFile.Run, + releasesHaveChangelog.Run, + } Fuzzing = []ProbeImpl{ fuzzed.Run, } @@ -183,6 +189,7 @@ var ( func init() { All = concatMultipleProbes([][]ProbeImpl{ BinaryArtifacts, + Changelog, CIIBestPractices, CITests, CodeReview, diff --git a/probes/hasChangelogFile/def.yml b/probes/hasChangelogFile/def.yml new file mode 100644 index 00000000000..5b4fb085241 --- /dev/null +++ b/probes/hasChangelogFile/def.yml @@ -0,0 +1,41 @@ +# Copyright 2024 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: hasChangelogFile +lifecycle: experimental +short: Check that the project has a changelog file +motivation: > + A changelog provides a human-readable log of notable changes for each release. + It shows project maturity and helps users understand what has changed between versions. + This aligns with OSPS-BR-04 which requires releases to provide a descriptive log of modifications. +implementation: > + The implementation checks whether a changelog file (e.g. CHANGELOG.md, CHANGES, NEWS, HISTORY, RELEASE-NOTES) + is present at the top level of the repository. +outcome: + - If a changelog file is found, the probe returns OutcomeTrue for each file. + - If no changelog file is found, the probe returns a single OutcomeFalse. +remediation: + onOutcome: False + effort: Low + text: + - Add a CHANGELOG.md file to the root of your repository. + - "Follow the format recommended by [Keep a Changelog](https://keepachangelog.com/)." + - Ensure the changelog is updated with each release to document notable changes. +ecosystem: + languages: + - all + clients: + - github + - gitlab + - localdir diff --git a/probes/hasChangelogFile/impl.go b/probes/hasChangelogFile/impl.go new file mode 100644 index 00000000000..9fc4243116f --- /dev/null +++ b/probes/hasChangelogFile/impl.go @@ -0,0 +1,63 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hasChangelogFile + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/internal/checknames" + "github.com/ossf/scorecard/v5/internal/probes" + "github.com/ossf/scorecard/v5/probes/internal/utils/uerror" +) + +func init() { + probes.MustRegister(Probe, Run, []checknames.CheckName{checknames.Changelog}) +} + +//go:embed *.yml +var fs embed.FS + +const Probe = "hasChangelogFile" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + var findings []finding.Finding + + changelogFiles := raw.ChangelogResults.ChangelogFiles + if len(changelogFiles) == 0 { + f, err := finding.NewFalse(fs, Probe, "project does not have a changelog file", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + return findings, Probe, nil + } + + for i := range changelogFiles { + loc := changelogFiles[i].Location() + f, err := finding.NewTrue(fs, Probe, "project has a changelog file", loc) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/hasChangelogFile/impl_test.go b/probes/hasChangelogFile/impl_test.go new file mode 100644 index 00000000000..9b767fad03d --- /dev/null +++ b/probes/hasChangelogFile/impl_test.go @@ -0,0 +1,110 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hasChangelogFile + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/probes/internal/utils/test" + "github.com/ossf/scorecard/v5/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + tests := []struct { + err error + raw *checker.RawResults + name string + outcomes []finding.Outcome + }{ + { + name: "changelog file found", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeTrue, + }, + }, + { + name: "multiple changelog files found", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + ChangelogFiles: []checker.File{ + {Path: "CHANGELOG.md", Type: finding.FileTypeSource}, + {Path: "NEWS", Type: finding.FileTypeSource}, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeTrue, + finding.OutcomeTrue, + }, + }, + { + name: "no changelog file", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + ChangelogFiles: nil, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeFalse, + }, + }, + { + name: "empty changelog files", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + ChangelogFiles: []checker.File{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeFalse, + }, + }, + { + name: "nil raw data", + raw: nil, + err: uerror.ErrNil, + outcomes: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + test.AssertOutcomes(t, findings, tt.outcomes) + }) + } +} diff --git a/probes/releasesHaveChangelog/def.yml b/probes/releasesHaveChangelog/def.yml new file mode 100644 index 00000000000..677886845b7 --- /dev/null +++ b/probes/releasesHaveChangelog/def.yml @@ -0,0 +1,40 @@ +# Copyright 2024 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesHaveChangelog +lifecycle: experimental +short: Check that the project's releases are documented in the changelog +motivation: > + Releases that are documented in the changelog demonstrate that the project maintains + descriptive logs of functional and security modifications per OSPS-BR-04. +implementation: > + The implementation reads the changelog file, extracts version strings, and checks + whether the last 5 release tags have corresponding entries in the changelog. + Releases with substantive release notes (GitHub/GitLab release body) also count. +outcome: + - If all recent releases have changelog entries, the probe returns OutcomeTrue. + - If no releases are found, the probe returns OutcomeNotApplicable. + - If some or no releases have changelog entries, the probe returns OutcomeFalse. +remediation: + onOutcome: False + effort: Medium + text: + - Ensure each release version has a corresponding entry in your changelog file. + - "Follow the format recommended by [Keep a Changelog](https://keepachangelog.com/)." +ecosystem: + languages: + - all + clients: + - github + - gitlab diff --git a/probes/releasesHaveChangelog/impl.go b/probes/releasesHaveChangelog/impl.go new file mode 100644 index 00000000000..48f252b6e4a --- /dev/null +++ b/probes/releasesHaveChangelog/impl.go @@ -0,0 +1,79 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package releasesHaveChangelog + +import ( + "embed" + "fmt" + "strconv" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/internal/checknames" + "github.com/ossf/scorecard/v5/internal/probes" + "github.com/ossf/scorecard/v5/probes/internal/utils/uerror" +) + +func init() { + probes.MustRegister(Probe, Run, []checknames.CheckName{checknames.Changelog}) +} + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesHaveChangelog" + ReleasesWithChangelogKey = "releasesWithChangelog" + ReleasesTotalKey = "releasesTotal" +) + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + var findings []finding.Finding + + total := raw.ChangelogResults.TotalReleases + withChangelog := raw.ChangelogResults.ReleasesWithChangelog + + if total == 0 { + f, err := finding.NewNotApplicable(fs, Probe, "no releases found", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + return findings, Probe, nil + } + + // Emit a single finding with the counts for the evaluation to use. + var f *finding.Finding + var err error + if withChangelog == total { + f, err = finding.NewTrue(fs, Probe, + fmt.Sprintf("%d out of %d release(s) have a changelog entry", withChangelog, total), nil) + } else { + f, err = finding.NewFalse(fs, Probe, + fmt.Sprintf("%d out of %d release(s) have a changelog entry", withChangelog, total), nil) + } + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f.WithValue(ReleasesWithChangelogKey, strconv.Itoa(withChangelog)) + f.WithValue(ReleasesTotalKey, strconv.Itoa(total)) + findings = append(findings, *f) + + return findings, Probe, nil +} diff --git a/probes/releasesHaveChangelog/impl_test.go b/probes/releasesHaveChangelog/impl_test.go new file mode 100644 index 00000000000..26fdf6a6a0c --- /dev/null +++ b/probes/releasesHaveChangelog/impl_test.go @@ -0,0 +1,108 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package releasesHaveChangelog + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/probes/internal/utils/test" + "github.com/ossf/scorecard/v5/probes/internal/utils/uerror" +) + +func Test_Run(t *testing.T) { + t.Parallel() + tests := []struct { + err error + raw *checker.RawResults + name string + outcomes []finding.Outcome + }{ + { + name: "all releases have changelog", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + TotalReleases: 3, + ReleasesWithChangelog: 3, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeTrue, + }, + }, + { + name: "some releases have changelog", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + TotalReleases: 5, + ReleasesWithChangelog: 2, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeFalse, + }, + }, + { + name: "no releases have changelog", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + TotalReleases: 3, + ReleasesWithChangelog: 0, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeFalse, + }, + }, + { + name: "no releases at all", + raw: &checker.RawResults{ + ChangelogResults: checker.ChangelogData{ + TotalReleases: 0, + ReleasesWithChangelog: 0, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNotApplicable, + }, + }, + { + name: "nil raw data", + raw: nil, + err: uerror.ErrNil, + outcomes: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + test.AssertOutcomes(t, findings, tt.outcomes) + }) + } +}