diff --git a/pkg/config/scheduledfeed.go b/pkg/config/scheduledfeed.go index 899ba4ed..5e708651 100644 --- a/pkg/config/scheduledfeed.go +++ b/pkg/config/scheduledfeed.go @@ -17,6 +17,7 @@ import ( "github.com/ossf/package-feeds/pkg/events" "github.com/ossf/package-feeds/pkg/feeds" "github.com/ossf/package-feeds/pkg/feeds/crates" + "github.com/ossf/package-feeds/pkg/feeds/github" "github.com/ossf/package-feeds/pkg/feeds/goproxy" "github.com/ossf/package-feeds/pkg/feeds/npm" "github.com/ossf/package-feeds/pkg/feeds/nuget" @@ -161,6 +162,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed, switch fc.Type { case crates.FeedName: return crates.New(fc.Options, eventHandler) + case github.FeedName: + return github.New(fc.Options) case goproxy.FeedName: return goproxy.New(fc.Options) case npm.FeedName: diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go index 3719b421..a49e92a8 100644 --- a/pkg/feeds/feed.go +++ b/pkg/feeds/feed.go @@ -3,6 +3,7 @@ package feeds import ( "errors" "fmt" + "sort" "time" ) @@ -72,3 +73,10 @@ func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package { func (err UnsupportedOptionError) Error() string { return fmt.Sprintf("unsupported option `%v` supplied to %v feed", err.Option, err.Feed) } + +func SortPackages(pkgs []*Package) { + // Ensure packages are sorted by CreatedDate in order of most recent + sort.SliceStable(pkgs, func(i, j int) bool { + return pkgs[j].CreatedDate.Before(pkgs[i].CreatedDate) + }) +} diff --git a/pkg/feeds/github/README.md b/pkg/feeds/github/README.md new file mode 100644 index 00000000..8fd3efd2 --- /dev/null +++ b/pkg/feeds/github/README.md @@ -0,0 +1,15 @@ +# Github Feed + +This feed allows polling of releases from specific github repositories. This feed **requires** a list of repositories to be specified. + +## Configuration options + +The `packages` field is required by the github feed. + +``` +feeds: +- type: github + options: + packages: + - "ossf/package-feeds" +``` \ No newline at end of file diff --git a/pkg/feeds/github/github.go b/pkg/feeds/github/github.go new file mode 100644 index 00000000..264d2633 --- /dev/null +++ b/pkg/feeds/github/github.go @@ -0,0 +1,117 @@ +package github + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" +) + +const ( + FeedName = "github" +) + +var ( + urlFormatString = "https://api.github.com/repos/%v/releases?per_page=%v" + httpClient = &http.Client{ + Timeout: 10 * time.Second, + } + releasesPerQuery = 20 + errPackageOptionsUnset = errors.New("github feed requires packages to be configured as a feed option") + errMinimumPackagesRequired = errors.New("github feed requires a minimum of 1 package supplied under options") +) + +type Feed struct { + repositories []string + options feeds.FeedOptions +} + +func New(feedOptions feeds.FeedOptions) (*Feed, error) { + if feedOptions.Packages == nil { + return nil, errPackageOptionsUnset + } + if len(*feedOptions.Packages) == 0 { + return nil, errMinimumPackagesRequired + } + return &Feed{ + repositories: *feedOptions.Packages, + options: feedOptions, + }, nil +} + +func (feed Feed) GetFeedOptions() feeds.FeedOptions { + return feed.options +} + +func (feed Feed) GetName() string { + return "github" +} + +type releaseList []*release + +type release struct { + TagName string `json:"tag_name"` + + // TODO: Add optional filter of Draft/Prerelease + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + + PublishedAt time.Time `json:"published_at"` +} + +func fetchReleases(repository string) (releaseList, error) { + releases := releaseList{} + resp, err := httpClient.Get(fmt.Sprintf(urlFormatString, repository, releasesPerQuery)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + err = json.NewDecoder(resp.Body).Decode(&releases) + if err != nil { + return nil, err + } + return releases, nil +} + +func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, []error) { + pkgs := []*feeds.Package{} + pkgChannel := make(chan []*feeds.Package) + errs := []error{} + errChannel := make(chan error) + + for _, repo := range feed.repositories { + go func(repo string) { + repoReleases, err := fetchReleases(repo) + if err != nil { + errChannel <- err + return + } + pkgChannel <- releasesToPackages(repo, repoReleases) + }(repo) + } + + for i := 0; i < len(feed.repositories); i++ { + select { + case pkgSlice := <-pkgChannel: + pkgs = append(pkgs, pkgSlice...) + case err := <-errChannel: + errs = append(errs, err) + } + } + // Sort packages to ensure deterministic ordering. + feeds.SortPackages(pkgs) + + pkgs = feeds.ApplyCutoff(pkgs, cutoff) + return pkgs, errs +} + +func releasesToPackages(repo string, releases releaseList) []*feeds.Package { + pkgs := []*feeds.Package{} + for _, release := range releases { + pkgs = append(pkgs, feeds.NewPackage(release.PublishedAt, repo, release.TagName, FeedName)) + } + return pkgs +} diff --git a/pkg/feeds/github/github_test.go b/pkg/feeds/github/github_test.go new file mode 100644 index 00000000..cd55af42 --- /dev/null +++ b/pkg/feeds/github/github_test.go @@ -0,0 +1,127 @@ +package github + +import ( + "errors" + "net/http" + "testing" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" + testutils "github.com/ossf/package-feeds/pkg/utils/test" +) + +func TestGithubLatest(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + "/repos/fooOrg/bar/releases": barResponse, + "/repos/fooOrg/baz/releases": bazResponse, + } + srv := testutils.HTTPServerMock(handlers) + + urlFormatString = srv.URL + "/repos/%v/releases?per_page=%v" + + packages := []string{ + "fooOrg/bar", + "fooOrg/baz", + } + + cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + feed, err := New(feeds.FeedOptions{Packages: &packages}) + if err != nil { + t.Fatalf("Unexpected error during feed creation: %v", err) + } + + pkgs, errs := feed.Latest(cutoff) + if len(errs) > 0 { + t.Fatalf("Failed to poll latest packages from feed: %v", errs[0]) + } + if len(pkgs) != 6 { + t.Fatalf("Polling feed did not return the expected number of packages") + } + expectedVersions := []string{"2.1.0", "1.1.0", "2.0.5", "1.0.5", "1.0.0", "2.0.0"} + expectedRepos := []string{"fooOrg/baz", "fooOrg/bar", "fooOrg/baz", "fooOrg/bar", "fooOrg/bar", "fooOrg/baz"} + + for i := range pkgs { + if pkgs[i].Version != expectedVersions[i] { + t.Errorf("Unexpected version %v found when expecting %v", pkgs[i].Version, expectedVersions[i]) + } + if pkgs[i].Type != FeedName { + t.Errorf("Type set incorrectly for feed type") + } + if pkgs[i].Name != expectedRepos[i] { + t.Errorf("Unexpected name %v found when expecting %v", pkgs[i].Name, expectedRepos[i]) + } + } +} + +func TestConfigurationErrors(t *testing.T) { + t.Parallel() + + _, err := New(feeds.FeedOptions{}) + if !errors.Is(err, errPackageOptionsUnset) { + t.Fatalf("Expected to fail due to missing packages option") + } + + packages := &[]string{} + _, err = New(feeds.FeedOptions{Packages: packages}) + if !errors.Is(err, errMinimumPackagesRequired) { + t.Fatalf("Expected to fail due to missing packages option") + } +} + +func barResponse(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(` +[ + { + "tag_name": "1.1.0", + "draft": false, + "prerelease": false, + "published_at": "2021-02-04T10:22:41Z" + }, + { + "tag_name": "1.0.5", + "draft": false, + "prerelease": false, + "published_at": "2021-01-04T10:22:41Z" + }, + { + "tag_name": "1.0.0", + "draft": false, + "prerelease": false, + "published_at": "2021-01-02T10:22:41Z" + } +] +`)) + if err != nil { + http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError) + } +} + +func bazResponse(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(` +[ + { + "tag_name": "2.1.0", + "draft": false, + "prerelease": false, + "published_at": "2021-04-05T10:22:41Z" + }, + { + "tag_name": "2.0.5", + "draft": false, + "prerelease": false, + "published_at": "2021-02-01T10:22:41Z" + }, + { + "tag_name": "2.0.0", + "draft": false, + "prerelease": false, + "published_at": "2021-01-01T10:22:41Z" + } +] +`)) + if err != nil { + http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError) + } +} diff --git a/pkg/feeds/lossy_logging.go b/pkg/feeds/lossy_logging.go index 744f99fa..9f1f125f 100644 --- a/pkg/feeds/lossy_logging.go +++ b/pkg/feeds/lossy_logging.go @@ -1,8 +1,6 @@ package feeds import ( - "sort" - log "github.com/sirupsen/logrus" "github.com/ossf/package-feeds/pkg/events" @@ -32,10 +30,7 @@ func (lfa *LossyFeedAlerter) ProcessPackages(feed string, packages []*Package) { pkgs := make([]*Package, len(packages)) copy(pkgs, packages) - // Ensure packages are sorted by CreatedDate in order of most recent - sort.SliceStable(pkgs, func(i, j int) bool { - return pkgs[j].CreatedDate.Before(pkgs[i].CreatedDate) - }) + SortPackages(pkgs) previousPackages, ok := lfa.previousPackages[feed] nonZeroResults := len(pkgs) > 0 && len(previousPackages) > 0