diff --git a/Documentation/reference/config.md b/Documentation/reference/config.md index bbb7eaca14..47661fec11 100644 --- a/Documentation/reference/config.md +++ b/Documentation/reference/config.md @@ -330,6 +330,7 @@ If the value is nil the default list of Matchers will run: * alpine-matcher * aws-matcher * debian-matcher +* echo-matcher * gobin * java-maven * oracle @@ -366,6 +367,7 @@ If the value is nil (or `null` in yaml) the default set of Updaters will run: * alpine * aws * debian +* echo * oracle * osv * photon diff --git a/cmd/clair/main.go b/cmd/clair/main.go index b7e024a7c9..3f7f637873 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -19,6 +19,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/quay/clair/v4/cmd" + _ "github.com/quay/clair/v4/echo" "github.com/quay/clair/v4/health" "github.com/quay/clair/v4/httptransport" "github.com/quay/clair/v4/initialize" diff --git a/cmd/clairctl/export.go b/cmd/clairctl/export.go index 228aa6ec11..31fe26d826 100644 --- a/cmd/clairctl/export.go +++ b/cmd/clairctl/export.go @@ -16,6 +16,7 @@ import ( _ "github.com/quay/claircore/updater/defaults" "github.com/urfave/cli/v2" + _ "github.com/quay/clair/v4/echo" "github.com/quay/clair/v4/internal/httputil" ) diff --git a/cmd/clairctl/main.go b/cmd/clairctl/main.go index 97720e19af..527d17afa2 100644 --- a/cmd/clairctl/main.go +++ b/cmd/clairctl/main.go @@ -13,6 +13,7 @@ import ( "github.com/urfave/cli/v2" "github.com/quay/clair/v4/cmd" + _ "github.com/quay/clair/v4/echo" "github.com/quay/clair/v4/internal/logging" ) diff --git a/config.yaml.sample b/config.yaml.sample index 7a267e5c29..10c3540acf 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -29,6 +29,7 @@ matcher: - "alpine" - "aws" - "debian" + - "echo" - "oracle" - "osv" - "photon" @@ -41,6 +42,7 @@ matchers: - "alpine-matcher" - "aws-matcher" - "debian-matcher" + - "echo-matcher" - "gobin" - "java-maven" - "oracle" diff --git a/config/matchers.go b/config/matchers.go index 433fa69b44..7dd7f6cced 100644 --- a/config/matchers.go +++ b/config/matchers.go @@ -13,6 +13,7 @@ type Matchers struct { // "alpine" // "aws" // "debian" + // "echo-matcher" // "oracle" // "photon" // "python" diff --git a/config/updaters.go b/config/updaters.go index e6b90e9cf7..7801a90e59 100644 --- a/config/updaters.go +++ b/config/updaters.go @@ -23,6 +23,7 @@ type Updaters struct { // "aws" // "clair.cvss" // "debian" + // "echo" // "oracle" // "osv" // "photon" diff --git a/echo/defaults.go b/echo/defaults.go new file mode 100644 index 0000000000..3889d067be --- /dev/null +++ b/echo/defaults.go @@ -0,0 +1,40 @@ +package echo + +import ( + "context" + "sync" + "time" + + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/matchers/registry" + "github.com/quay/claircore/updater" +) + +var ( + once sync.Once + regerr error +) + +func init() { + ctx, done := context.WithTimeout(context.Background(), 1*time.Minute) + defer done() + once.Do(func() { regerr = register(ctx) }) +} + +// Error reports if an error was encountered when initializing the Echo +// updater and matcher. +func Error() error { + return regerr +} + +func register(ctx context.Context) error { + f, err := NewFactory(ctx) + if err != nil { + return err + } + updater.Register("echo", f) + + registry.Register("echo-matcher", driver.MatcherStatic(&Matcher{})) + + return nil +} diff --git a/echo/distributionscanner.go b/echo/distributionscanner.go new file mode 100644 index 0000000000..74131a9250 --- /dev/null +++ b/echo/distributionscanner.go @@ -0,0 +1,80 @@ +package echo + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "runtime/trace" + + "github.com/quay/claircore" + "github.com/quay/claircore/indexer" + "github.com/quay/claircore/osrelease" +) + +var ( + _ indexer.DistributionScanner = (*DistributionScanner)(nil) + _ indexer.VersionedScanner = (*DistributionScanner)(nil) +) + +// DistributionScanner attempts to discover if a layer +// displays characteristics of an Echo distribution. +type DistributionScanner struct{} + +// Name implements [indexer.VersionedScanner]. +func (*DistributionScanner) Name() string { return "echo" } + +// Version implements [indexer.VersionedScanner]. +func (*DistributionScanner) Version() string { return "1" } + +// Kind implements [indexer.VersionedScanner]. +func (*DistributionScanner) Kind() string { return "distribution" } + +// Scan implements [indexer.DistributionScanner]. +func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) { + defer trace.StartRegion(ctx, "Scanner.Scan").End() + log := slog.With("version", ds.Version(), "layer", l.Hash.String()) + log.DebugContext(ctx, "start") + defer log.DebugContext(ctx, "done") + + sys, err := l.FS() + if err != nil { + return nil, fmt.Errorf("echo: unable to open layer: %w", err) + } + d, err := findDist(ctx, log, sys) + if err != nil { + return nil, err + } + if d == nil { + return nil, nil + } + return []*claircore.Distribution{d}, nil +} + +func findDist(ctx context.Context, log *slog.Logger, sys fs.FS) (*claircore.Distribution, error) { + f, err := sys.Open(osrelease.Path) + switch { + case errors.Is(err, nil): + case errors.Is(err, fs.ErrNotExist): + log.DebugContext(ctx, "no os-release file") + return nil, nil + default: + return nil, fmt.Errorf("echo: unexpected error: %w", err) + } + kv, err := osrelease.Parse(ctx, f) + if err != nil { + log.InfoContext(ctx, "malformed os-release file", "reason", err) + return nil, nil + } + if kv[`ID`] != `echo` { + return nil, nil + } + + ver := kv[`VERSION_ID`] + if ver == "" { + log.InfoContext(ctx, "echo os-release missing VERSION_ID") + return nil, nil + } + return mkDist(ver), nil +} diff --git a/echo/doc.go b/echo/doc.go new file mode 100644 index 0000000000..fb3c417204 --- /dev/null +++ b/echo/doc.go @@ -0,0 +1,2 @@ +// Package echo contains an Indexer, Matcher, and Updater for Echo Linux. +package echo diff --git a/echo/ecosystem.go b/echo/ecosystem.go new file mode 100644 index 0000000000..519f4cc4f5 --- /dev/null +++ b/echo/ecosystem.go @@ -0,0 +1,38 @@ +package echo + +import ( + "context" + + "github.com/quay/claircore/debian" + "github.com/quay/claircore/dpkg" + "github.com/quay/claircore/indexer" + "github.com/quay/claircore/linux" + "github.com/quay/claircore/ubuntu" +) + +// NewDpkgEcosystem provides the set of scanners and coalescers for the dpkg +// ecosystem, extended to include the Echo distribution scanner alongside the +// Debian and Ubuntu scanners. +func NewDpkgEcosystem(ctx context.Context) *indexer.Ecosystem { + return &indexer.Ecosystem{ + PackageScanners: func(ctx context.Context) ([]indexer.PackageScanner, error) { + return []indexer.PackageScanner{ + &dpkg.Scanner{}, + &dpkg.DistrolessScanner{}, + }, nil + }, + DistributionScanners: func(ctx context.Context) ([]indexer.DistributionScanner, error) { + return []indexer.DistributionScanner{ + &debian.DistributionScanner{}, + &ubuntu.DistributionScanner{}, + &DistributionScanner{}, + }, nil + }, + RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) { + return []indexer.RepositoryScanner{}, nil + }, + Coalescer: func(ctx context.Context) (indexer.Coalescer, error) { + return linux.NewCoalescer(), nil + }, + } +} diff --git a/echo/matcher.go b/echo/matcher.go new file mode 100644 index 0000000000..b161789598 --- /dev/null +++ b/echo/matcher.go @@ -0,0 +1,61 @@ +package echo + +import ( + "context" + + version "github.com/knqyf263/go-deb-version" + + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" +) + +// Matcher is a [driver.Matcher] for Echo distributions. +type Matcher struct{} + +var _ driver.Matcher = (*Matcher)(nil) + +// Name implements [driver.Matcher]. +func (*Matcher) Name() string { + return "echo-matcher" +} + +// Filter implements [driver.Matcher]. +func (*Matcher) Filter(record *claircore.IndexRecord) bool { + if record.Distribution == nil { + return false + } + return record.Distribution.DID == "echo" +} + +// Query implements [driver.Matcher]. +func (*Matcher) Query() []driver.MatchConstraint { + return []driver.MatchConstraint{ + driver.DistributionDID, + } +} + +// Vulnerable implements [driver.Matcher]. +func (*Matcher) Vulnerable(ctx context.Context, record *claircore.IndexRecord, vuln *claircore.Vulnerability) (bool, error) { + if vuln.FixedInVersion == "" { + return true, nil + } + // If fixed_version is 0, the package is unaffected. + if vuln.FixedInVersion == "0" { + return false, nil + } + + v1, err := version.NewVersion(record.Package.Version) + if err != nil { + return false, err + } + v2, err := version.NewVersion(vuln.FixedInVersion) + if err != nil { + return false, err + } + + if v1.LessThan(v2) { + return true, nil + } + + return false, nil +} diff --git a/echo/parser.go b/echo/parser.go new file mode 100644 index 0000000000..010e5450a6 --- /dev/null +++ b/echo/parser.go @@ -0,0 +1,53 @@ +package echo + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + + "github.com/quay/claircore" +) + +// advisoryData maps source package name to its vulnerabilities. +type advisoryData map[string]map[string]cveEntry + +// cveEntry holds vulnerability data for a single CVE. +type cveEntry struct { + FixedVersion string `json:"fixed_version"` +} + +// Parse implements [driver.Parser]. +func (u *echoUpdater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vulnerability, error) { + slog.InfoContext(ctx, "starting parse") + defer r.Close() + + var data advisoryData + if err := json.NewDecoder(r).Decode(&data); err != nil { + return nil, fmt.Errorf("echo: unable to parse advisory JSON: %w", err) + } + + dist := getDist() + + var vs []*claircore.Vulnerability + for pkg, cves := range data { + for cveID, entry := range cves { + v := &claircore.Vulnerability{ + Updater: u.Name(), + Name: cveID, + Links: linkPrefix + cveID, + Dist: dist, + FixedInVersion: entry.FixedVersion, + Package: &claircore.Package{ + Name: pkg, + Kind: claircore.SOURCE, + }, + } + vs = append(vs, v) + } + } + + slog.InfoContext(ctx, "parsed advisory database", "vulnerabilities", len(vs)) + return vs, nil +} diff --git a/echo/releases.go b/echo/releases.go new file mode 100644 index 0000000000..c48c7f3090 --- /dev/null +++ b/echo/releases.go @@ -0,0 +1,37 @@ +package echo + +import ( + "sync" + + "github.com/quay/claircore" +) + +var releases sync.Map + +func mkDist(ver string) *claircore.Distribution { + v, _ := releases.LoadOrStore(ver, &claircore.Distribution{ + PrettyName: "Echo Linux", + Name: "Echo Linux", + VersionID: ver, + DID: "echo", + }) + return v.(*claircore.Distribution) +} + +func getDist() *claircore.Distribution { + v, ok := releases.Load("generic") + if !ok { + return mkDist("generic") + } + return v.(*claircore.Distribution) +} + +const ( + // linkPrefix is the URL prefix for Echo advisory links. + linkPrefix = `https://advisory.echohq.com/cve/` + + // DefaultAdvisoryURL is the URL for the Echo advisory data. + // + //doc:url updater + DefaultAdvisoryURL = `https://advisory.echohq.com/data.json` +) diff --git a/echo/updater.go b/echo/updater.go new file mode 100644 index 0000000000..3f3484e297 --- /dev/null +++ b/echo/updater.go @@ -0,0 +1,166 @@ +package echo + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/tmp" +) + +var ( + _ driver.UpdaterSetFactory = (*Factory)(nil) + _ driver.Configurable = (*Factory)(nil) + _ driver.Updater = (*echoUpdater)(nil) + _ driver.Configurable = (*echoUpdater)(nil) +) + +// Factory creates an Updater for Echo Linux. +// +// [Configure] must be called before [UpdaterSet]. +type Factory struct { + c *http.Client + jsonURL *url.URL +} + +// NewFactory constructs a Factory. +// +// [Configure] must be called before [UpdaterSet]. +func NewFactory(_ context.Context) (*Factory, error) { + return &Factory{}, nil +} + +// FactoryConfig is the configuration honored by the Factory. +type FactoryConfig struct { + // URL is a URL to the Echo advisory JSON feed. + URL string `json:"url" yaml:"url"` +} + +// Configure implements [driver.Configurable]. +func (f *Factory) Configure(_ context.Context, cf driver.ConfigUnmarshaler, c *http.Client) error { + f.c = c + var cfg FactoryConfig + if err := cf(&cfg); err != nil { + return fmt.Errorf("echo: factory configuration error: %w", err) + } + + u, err := url.Parse(DefaultAdvisoryURL) + if cfg.URL != "" { + u, err = url.Parse(cfg.URL) + } + if err != nil { + return fmt.Errorf("echo: bad advisory URL: %w", err) + } + f.jsonURL = u + + return nil +} + +// UpdaterSet implements [driver.UpdaterSetFactory]. +func (f *Factory) UpdaterSet(_ context.Context) (driver.UpdaterSet, error) { + s := driver.NewUpdaterSet() + + u := &echoUpdater{ + jsonURL: f.jsonURL.String(), + } + + if err := s.Add(u); err != nil { + return s, fmt.Errorf("echo: unable to add updater: %w", err) + } + + return s, nil +} + +type echoUpdater struct { + jsonURL string + c *http.Client +} + +// Name implements [driver.Updater]. +func (u *echoUpdater) Name() string { + return "echo/updater" +} + +// UpdaterConfig is the configuration for the updater. +type UpdaterConfig struct { + // URL is a URL to the Echo advisory JSON feed. + URL string `json:"url" yaml:"url"` +} + +// Configure implements [driver.Configurable]. +func (u *echoUpdater) Configure(_ context.Context, f driver.ConfigUnmarshaler, c *http.Client) error { + u.c = c + var cfg UpdaterConfig + if err := f(&cfg); err != nil { + return err + } + if cfg.URL != "" { + u.jsonURL = cfg.URL + slog.Info("echo: configured advisory URL") + } + return nil +} + +// Fetch implements [driver.Fetcher]. +func (u *echoUpdater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + log := slog.With("database", u.jsonURL) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.jsonURL, nil) + if err != nil { + return nil, "", fmt.Errorf("echo: failed to create request: %w", err) + } + if fingerprint != "" { + req.Header.Set("If-Modified-Since", string(fingerprint)) + } + + resp, err := u.c.Do(req) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + return nil, "", fmt.Errorf("echo: failed to retrieve advisory database: %w", err) + } + + fp := resp.Header.Get("Last-Modified") + + switch resp.StatusCode { + case http.StatusOK: + if fingerprint == "" || fp != string(fingerprint) { + log.InfoContext(ctx, "fetching latest advisory database") + break + } + fallthrough + case http.StatusNotModified: + return nil, fingerprint, driver.Unchanged + default: + return nil, "", fmt.Errorf("echo: unexpected response: %v", resp.Status) + } + + f, err := tmp.NewFile("", "echo.") + if err != nil { + return nil, "", err + } + + var success bool + defer func() { + if !success { + if err := f.Close(); err != nil { + log.WarnContext(ctx, "unable to close spool", "reason", err) + } + } + }() + if _, err := io.Copy(f, resp.Body); err != nil { + return nil, "", fmt.Errorf("echo: failed to read http body: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, "", fmt.Errorf("echo: failed to seek body: %w", err) + } + log.InfoContext(ctx, "fetched latest advisory database successfully") + + success = true + return f, driver.Fingerprint(fp), nil +} diff --git a/go.mod b/go.mod index 407642e97a..8dd7a73fe8 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/grafana/pyroscope-go/godeltaprof v0.1.9 github.com/jackc/pgx/v5 v5.9.1 github.com/klauspost/compress v1.18.4 + github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d github.com/prometheus/client_golang v1.23.2 github.com/quay/clair/config v1.4.3 github.com/quay/claircore v1.5.51 @@ -69,7 +70,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect - github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d // indirect github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/initialize/services.go b/initialize/services.go index 66c21cffa4..c81a7a4b97 100644 --- a/initialize/services.go +++ b/initialize/services.go @@ -13,15 +13,25 @@ import ( "github.com/go-jose/go-jose/v3/jwt" "github.com/jackc/pgx/v5/pgxpool" "github.com/quay/clair/config" + "github.com/quay/claircore/alpine" "github.com/quay/claircore/datastore/postgres" "github.com/quay/claircore/enricher/cvss" + "github.com/quay/claircore/gobin" + ccindexer "github.com/quay/claircore/indexer" + "github.com/quay/claircore/java" "github.com/quay/claircore/libindex" "github.com/quay/claircore/libvuln" "github.com/quay/claircore/libvuln/driver" "github.com/quay/claircore/pkg/ctxlock/v2" + "github.com/quay/claircore/python" + "github.com/quay/claircore/rhel" + "github.com/quay/claircore/rhel/rhcc" + "github.com/quay/claircore/rpm" + "github.com/quay/claircore/ruby" "golang.org/x/net/publicsuffix" clairerror "github.com/quay/clair/v4/clair-error" + "github.com/quay/clair/v4/echo" "github.com/quay/clair/v4/httptransport" "github.com/quay/clair/v4/httptransport/client" "github.com/quay/clair/v4/indexer" @@ -131,9 +141,25 @@ func localIndexer(ctx context.Context, cfg *config.Config) (indexer.Service, err return nil, mkErr(err) } + // Explicitly set ecosystems to include the Echo distribution scanner + // in the dpkg ecosystem. This overrides claircore's defaults + // (libindex.New sets these when Ecosystems is nil). + ecosystems := []*ccindexer.Ecosystem{ + echo.NewDpkgEcosystem(ctx), + alpine.NewEcosystem(ctx), + rhel.NewEcosystem(ctx), + rpm.NewEcosystem(ctx), + python.NewEcosystem(ctx), + java.NewEcosystem(ctx), + rhcc.NewEcosystem(ctx), + gobin.NewEcosystem(ctx), + ruby.NewEcosystem(ctx), + } + opts := libindex.Options{ Store: store, Locker: locker, + Ecosystems: ecosystems, ScanLockRetry: time.Duration(cfg.Indexer.ScanLockRetry) * time.Second, LayerScanConcurrency: cfg.Indexer.LayerScanConcurrency, }