diff --git a/query/config_analysis.go b/query/config_analysis.go new file mode 100644 index 00000000..1dc34652 --- /dev/null +++ b/query/config_analysis.go @@ -0,0 +1,34 @@ +package query + +import ( + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" + "github.com/google/uuid" +) + +func FindConfigAnalysisByResourceSelector(ctx context.Context, limit int, resourceSelectors ...types.ResourceSelector) ([]models.ConfigAnalysis, error) { + ids, err := FindConfigAnalysisIDsByResourceSelector(ctx, limit, resourceSelectors...) + if err != nil { + return nil, err + } + + return GetConfigAnalysisByIDs(ctx, ids) +} + +func FindConfigAnalysisIDsByResourceSelector(ctx context.Context, limit int, resourceSelectors ...types.ResourceSelector) ([]uuid.UUID, error) { + return queryTableWithResourceSelectors(ctx, configAnalysisItemsView, limit, resourceSelectors...) +} + +func GetConfigAnalysisByIDs(ctx context.Context, ids []uuid.UUID) ([]models.ConfigAnalysis, error) { + if len(ids) == 0 { + return nil, nil + } + + var analyses []models.ConfigAnalysis + if err := ctx.DB().Where("id IN ?", ids).Find(&analyses).Error; err != nil { + return nil, err + } + + return analyses, nil +} diff --git a/query/models.go b/query/models.go index 669845ae..e03f1ac5 100644 --- a/query/models.go +++ b/query/models.go @@ -192,6 +192,10 @@ type QueryModel struct { // True when the table has properties column HasProperties bool + // True when the table has a "deleted_at" column. + // When false, the default `deleted_at IS NULL` filter is not applied. + HasDeletedAt bool + // FieldMapper maps the value of these fields FieldMapper map[string]func(ctx context.Context, id string) (any, error) } @@ -211,6 +215,7 @@ var ConfigItemQueryModel = QueryModel{ HasTags: true, HasAgents: true, HasLabels: true, + HasDeletedAt: true, Aliases: map[string]string{ "created": "created_at", "updated": "updated_at", @@ -243,6 +248,7 @@ var ConfigItemSummaryQueryModel = QueryModel{ HasAgents: true, HasLabels: true, HasProperties: true, + HasDeletedAt: true, Aliases: map[string]string{ "created": "created_at", "updated": "updated_at", @@ -293,6 +299,7 @@ var ComponentQueryModel = QueryModel{ HasProperties: true, HasAgents: true, HasLabels: true, + HasDeletedAt: true, FieldMapper: map[string]func(ctx context.Context, id string) (any, error){ "agent_id": AgentMapper, "created_at": DateMapper, @@ -316,8 +323,9 @@ var CheckQueryModel = QueryModel{ "health": "status", "check_type": "type", }, - HasAgents: true, - HasLabels: true, + HasAgents: true, + HasLabels: true, + HasDeletedAt: true, FieldMapper: map[string]func(ctx context.Context, id string) (any, error){ "agent_id": AgentMapper, "created_at": DateMapper, @@ -327,9 +335,10 @@ var CheckQueryModel = QueryModel{ } var PlaybookQueryModel = QueryModel{ - Table: models.Playbook{}.TableName(), - HasTags: true, - Columns: []string{"id", "name", "namespace", "created_at", "updated_at", "deleted_at"}, + Table: models.Playbook{}.TableName(), + HasTags: true, + HasDeletedAt: true, + Columns: []string{"id", "name", "namespace", "created_at", "updated_at", "deleted_at"}, Aliases: map[string]string{ "created": "created_at", "updated": "updated_at", @@ -343,8 +352,9 @@ var PlaybookQueryModel = QueryModel{ } var ConnectionQueryModel = QueryModel{ - Table: models.Connection{}.TableName(), - Columns: []string{"id", "name", "namespace", "type"}, + Table: models.Connection{}.TableName(), + Columns: []string{"id", "name", "namespace", "type"}, + HasDeletedAt: true, } var ConfigChangeQueryModel = QueryModel{ @@ -356,6 +366,7 @@ var ConfigChangeQueryModel = QueryModel{ JSONMapColumns: []string{"tags", "details"}, HasAgents: true, HasTags: true, + HasDeletedAt: true, Aliases: map[string]string{ "created": "created_at", "first_observed": "first_observed", @@ -374,6 +385,7 @@ var ViewQueryModel = QueryModel{ Columns: []string{"name", "namespace"}, JSONMapColumns: []string{"labels"}, HasLabels: true, + HasDeletedAt: true, } var CanaryQueryModel = QueryModel{ @@ -385,6 +397,7 @@ var CanaryQueryModel = QueryModel{ JSONMapColumns: []string{"labels", "spec"}, HasLabels: true, HasAgents: true, + HasDeletedAt: true, Aliases: map[string]string{ "created": "created_at", "updated": "updated_at", @@ -399,6 +412,37 @@ var CanaryQueryModel = QueryModel{ }, } +const configAnalysisItemsView = "config_analysis_items" + +// ConfigAnalysisQueryModel powers resource selector search for config insights. +// It queries config_analysis_items so config parent fields (agent, deleted_at, +// type, tags, labels) are available like they are for catalog_changes. +var ConfigAnalysisQueryModel = QueryModel{ + Table: configAnalysisItemsView, + Columns: []string{ + "id", "config_id", "scraper_id", "source", "analyzer", "analysis_type", + "severity", "status", "summary", "message", "first_observed", "last_observed", + "name", "type", "config_type", "config_class", "agent_id", "deleted_at", "path", + }, + JSONMapColumns: []string{"tags", "labels", "config"}, + HasProperties: true, + HasTags: true, + HasLabels: true, + HasAgents: true, + HasDeletedAt: true, + Aliases: map[string]string{ + "analyzer_type": "analysis_type", + "config_type": "type", + "namespace": "tags.namespace", + }, + FieldMapper: map[string]func(ctx context.Context, id string) (any, error){ + "agent_id": AgentMapper, + "first_observed": DateMapper, + "last_observed": DateMapper, + "deleted_at": DateMapper, + }, +} + func GetModelFromTable(table string) (QueryModel, error) { switch table { case models.ConfigItem{}.TableName(): @@ -419,6 +463,8 @@ func GetModelFromTable(table string) (QueryModel, error) { return ConfigItemSummaryQueryModel, nil case models.View{}.TableName(): return ViewQueryModel, nil + case models.ConfigAnalysis{}.TableName(), configAnalysisItemsView: + return ConfigAnalysisQueryModel, nil default: return QueryModel{}, fmt.Errorf("invalid table") } diff --git a/query/resource_selector.go b/query/resource_selector.go index 9bca1fec..5fce0f95 100644 --- a/query/resource_selector.go +++ b/query/resource_selector.go @@ -30,23 +30,25 @@ type SearchResourcesRequest struct { // Limit the number of results returned per resource type Limit int `json:"limit"` - Canaries []types.ResourceSelector `json:"canaries"` - Checks []types.ResourceSelector `json:"checks"` - Components []types.ResourceSelector `json:"components"` - Configs []types.ResourceSelector `json:"configs"` - ConfigChanges []types.ResourceSelector `json:"config_changes"` - Playbooks []types.ResourceSelector `json:"playbooks"` - Connections []types.ResourceSelector `json:"connections"` + Canaries []types.ResourceSelector `json:"canaries"` + Checks []types.ResourceSelector `json:"checks"` + Components []types.ResourceSelector `json:"components"` + Configs []types.ResourceSelector `json:"configs"` + ConfigChanges []types.ResourceSelector `json:"config_changes"` + ConfigAnalysis []types.ResourceSelector `json:"config_analysis"` + Playbooks []types.ResourceSelector `json:"playbooks"` + Connections []types.ResourceSelector `json:"connections"` } type SearchResourcesResponse struct { - Canaries []SelectedResource `json:"canaries,omitempty"` - Checks []SelectedResource `json:"checks,omitempty"` - Components []SelectedResource `json:"components,omitempty"` - Configs []SelectedResource `json:"configs,omitempty"` - ConfigChanges []SelectedResource `json:"config_changes,omitempty"` - Playbooks []SelectedResource `json:"playbooks,omitempty"` - Connections []SelectedResource `json:"connections,omitempty"` + Canaries []SelectedResource `json:"canaries,omitempty"` + Checks []SelectedResource `json:"checks,omitempty"` + Components []SelectedResource `json:"components,omitempty"` + Configs []SelectedResource `json:"configs,omitempty"` + ConfigChanges []SelectedResource `json:"config_changes,omitempty"` + ConfigAnalysis []SelectedResource `json:"config_analysis,omitempty"` + Playbooks []SelectedResource `json:"playbooks,omitempty"` + Connections []SelectedResource `json:"connections,omitempty"` } func (r *SearchResourcesResponse) GetIDs() []string { @@ -56,6 +58,7 @@ func (r *SearchResourcesResponse) GetIDs() []string { ids = append(ids, lo.Map(r.Configs, func(c SelectedResource, _ int) string { return c.ID })...) ids = append(ids, lo.Map(r.Components, func(c SelectedResource, _ int) string { return c.ID })...) ids = append(ids, lo.Map(r.ConfigChanges, func(c SelectedResource, _ int) string { return c.ID })...) + ids = append(ids, lo.Map(r.ConfigAnalysis, func(c SelectedResource, _ int) string { return c.ID })...) ids = append(ids, lo.Map(r.Playbooks, func(c SelectedResource, _ int) string { return c.ID })...) ids = append(ids, lo.Map(r.Connections, func(c SelectedResource, _ int) string { return c.ID })...) return ids @@ -75,6 +78,9 @@ type SelectedResource struct { // Status is the resource's free-form operational status (e.g. "Running", // "Pending"). Populated for configs and components. Status string `json:"status,omitempty"` + // Severity is populated for resource kinds that carry a severity + // (e.g. config insights). nil for other kinds. + Severity *string `json:"severity,omitempty"` } func SearchResources(ctx context.Context, req SearchResourcesRequest) (*SearchResourcesResponse, error) { @@ -189,6 +195,28 @@ func SearchResources(ctx context.Context, req SearchResourcesRequest) (*SearchRe return nil }) + eg.Go(func() error { + if items, err := FindConfigAnalysisByResourceSelector(ctx, req.Limit, req.ConfigAnalysis...); err != nil { + return err + } else { + for i := range items { + var severity *string + if s := string(items[i].Severity); s != "" { + severity = &s + } + output.ConfigAnalysis = append(output.ConfigAnalysis, SelectedResource{ + ID: items[i].ID.String(), + Name: items[i].Analyzer, + Type: string(items[i].AnalysisType), + Status: items[i].Status, + Severity: severity, + }) + } + } + + return nil + }) + eg.Go(func() error { if items, err := FindPlaybooksByResourceSelector(ctx, req.Limit, req.Playbooks...); err != nil { return err @@ -288,7 +316,7 @@ func SetResourceSelectorClause( query = query.Clauses(clauses...) } - if !resourceSelector.IncludeDeleted && !searchSetDeleted { + if !resourceSelector.IncludeDeleted && !searchSetDeleted && qm.HasDeletedAt { query = query.Where("deleted_at IS NULL") } diff --git a/schema/apply.go b/schema/apply.go index 6ad285f9..fe238146 100644 --- a/schema/apply.go +++ b/schema/apply.go @@ -50,6 +50,7 @@ func Apply(ctx context.Context, connection string) error { exclude := []string{ "config_items.properties_values", "components.properties_values", + "config_analysis.properties_values", "config_locations.config_locations_location_pattern_idx", // These indexes are managed in the views/037_notification_group_resources.sql file diff --git a/tests/fixtures/dummy/config_analysis.go b/tests/fixtures/dummy/config_analysis.go index 1e441277..c365314f 100644 --- a/tests/fixtures/dummy/config_analysis.go +++ b/tests/fixtures/dummy/config_analysis.go @@ -5,25 +5,43 @@ import ( "github.com/google/uuid" ) +var LogisticsDBRDSAnalysis = models.ConfigAnalysis{ + ID: uuid.New(), + ConfigID: LogisticsDBRDS.ID, + Analyzer: "rds-port-exposed", + AnalysisType: models.AnalysisTypeSecurity, + Severity: models.SeverityCritical, + Message: "Port exposed to public", + FirstObserved: &CurrentTime, + Status: models.AnalysisStatusOpen, +} + +var EC2InstanceBAnalysis = models.ConfigAnalysis{ + ID: uuid.New(), + ConfigID: EC2InstanceB.ID, + Analyzer: "ec2-ssh-key-not-rotated", + AnalysisType: models.AnalysisTypeSecurity, + Severity: models.SeverityCritical, + Message: "SSH key not rotated", + FirstObserved: &CurrentTime, + Status: models.AnalysisStatusOpen, +} + +var LogisticsAPIPodAnalysis = models.ConfigAnalysis{ + ID: uuid.New(), + ConfigID: LogisticsAPIPodConfig.ID, + Analyzer: "pod-security-context", + AnalysisType: models.AnalysisTypeSecurity, + Severity: models.SeverityHigh, + Message: "Pod security context is missing", + FirstObserved: &CurrentTime, + Status: models.AnalysisStatusOpen, +} + func AllDummyConfigAnalysis() []models.ConfigAnalysis { return []models.ConfigAnalysis{ - { - ID: uuid.New(), - ConfigID: LogisticsDBRDS.ID, - AnalysisType: models.AnalysisTypeSecurity, - Severity: "critical", - Message: "Port exposed to public", - FirstObserved: &CurrentTime, - Status: models.AnalysisStatusOpen, - }, - { - ID: uuid.New(), - ConfigID: EC2InstanceB.ID, - AnalysisType: models.AnalysisTypeSecurity, - Severity: "critical", - Message: "SSH key not rotated", - FirstObserved: &CurrentTime, - Status: models.AnalysisStatusOpen, - }, + LogisticsDBRDSAnalysis, + EC2InstanceBAnalysis, + LogisticsAPIPodAnalysis, } } diff --git a/tests/query_resource_selector_test.go b/tests/query_resource_selector_test.go index beff2e7a..b71a3526 100644 --- a/tests/query_resource_selector_test.go +++ b/tests/query_resource_selector_test.go @@ -724,6 +724,100 @@ var _ = ginkgo.Describe("View Resource Selector", func() { }) }) +var _ = ginkgo.Describe("Config Analysis Resource Selector", func() { + // Other suites insert additional config_analysis rows that are never cleaned + // up, so the severity/status/type cases are scoped by config_id to stay + // deterministic. + logisticsConfigID := dummy.LogisticsDBRDSAnalysis.ConfigID.String() + ec2ConfigID := dummy.EC2InstanceBAnalysis.ConfigID.String() + + testData := []struct { + description string + resourceSelector types.ResourceSelector + expectedIDs []uuid.UUID + }{ + { + description: "by config_id", + resourceSelector: types.ResourceSelector{Search: "config_id=" + logisticsConfigID}, + expectedIDs: []uuid.UUID{dummy.LogisticsDBRDSAnalysis.ID}, + }, + { + description: "config is not a config_id alias", + resourceSelector: types.ResourceSelector{Search: "config=" + logisticsConfigID}, + expectedIDs: nil, + }, + { + description: "by analyzer", + resourceSelector: types.ResourceSelector{Search: "analyzer=ec2-ssh-key-not-rotated"}, + expectedIDs: []uuid.UUID{dummy.EC2InstanceBAnalysis.ID}, + }, + { + description: "by analyzer prefix", + resourceSelector: types.ResourceSelector{Search: "analyzer=rds*"}, + expectedIDs: []uuid.UUID{dummy.LogisticsDBRDSAnalysis.ID}, + }, + { + description: "by analysis_type", + resourceSelector: types.ResourceSelector{Search: "analysis_type=security config_id=" + ec2ConfigID}, + expectedIDs: []uuid.UUID{dummy.EC2InstanceBAnalysis.ID}, + }, + { + description: "by config type", + resourceSelector: types.ResourceSelector{Search: "type=" + *dummy.LogisticsDBRDS.Type + " config_id=" + logisticsConfigID}, + expectedIDs: []uuid.UUID{dummy.LogisticsDBRDSAnalysis.ID}, + }, + { + description: "by severity", + resourceSelector: types.ResourceSelector{Search: "severity=critical config_id=" + logisticsConfigID}, + expectedIDs: []uuid.UUID{dummy.LogisticsDBRDSAnalysis.ID}, + }, + { + description: "by status", + resourceSelector: types.ResourceSelector{Search: "status=open config_id=" + ec2ConfigID}, + expectedIDs: []uuid.UUID{dummy.EC2InstanceBAnalysis.ID}, + }, + { + description: "no match for unknown analyzer", + resourceSelector: types.ResourceSelector{Search: "analyzer=does-not-exist"}, + expectedIDs: nil, + }, + } + + for _, test := range testData { + ginkgo.It(test.description, func() { + analyses, err := query.FindConfigAnalysisByResourceSelector(DefaultContext, -1, test.resourceSelector) + Expect(err).To(BeNil()) + gotIDs := lo.Map(analyses, func(a models.ConfigAnalysis, _ int) uuid.UUID { return a.ID }) + Expect(gotIDs).To(ConsistOf(test.expectedIDs)) + }) + } + + ginkgo.It("finds high or critical pod analysis", func() { + analyses, err := query.FindConfigAnalysisByResourceSelector(DefaultContext, -1, types.ResourceSelector{Search: "severity=high,critical type=Kubernetes::Pod"}) + Expect(err).To(BeNil()) + gotIDs := lo.Map(analyses, func(a models.ConfigAnalysis, _ int) uuid.UUID { return a.ID }) + Expect(gotIDs).To(ContainElement(dummy.LogisticsAPIPodAnalysis.ID)) + }) + + ginkgo.It("does not expose analysis json search", func() { + _, err := query.FindConfigAnalysisByResourceSelector(DefaultContext, -1, types.ResourceSelector{Search: "analysis.foo=bar"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("query for column:analysis.foo")) + }) + + ginkgo.It("flows through SearchResources", func() { + response, err := query.SearchResources(DefaultContext, query.SearchResourcesRequest{ + ConfigAnalysis: []types.ResourceSelector{{Search: "analyzer=rds-port-exposed"}}, + }) + Expect(err).To(BeNil()) + Expect(response.ConfigAnalysis).To(HaveLen(1)) + Expect(response.ConfigAnalysis[0].ID).To(Equal(dummy.LogisticsDBRDSAnalysis.ID.String())) + Expect(response.ConfigAnalysis[0].Name).To(Equal("rds-port-exposed")) + Expect(response.ConfigAnalysis[0].Type).To(Equal(string(models.AnalysisTypeSecurity))) + Expect(response.ConfigAnalysis[0].Severity).To(Equal(lo.ToPtr(string(models.SeverityCritical)))) + }) +}) + var _ = ginkgo.Describe("Resoure Selector with PEG", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { _ = query.SyncConfigCache(DefaultContext) diff --git a/views/002_seed.sql b/views/002_seed.sql index 98b366aa..b488c7f6 100644 --- a/views/002_seed.sql +++ b/views/002_seed.sql @@ -39,7 +39,7 @@ END $$; DO $$ DECLARE tbl TEXT; - tables TEXT[] := ARRAY['components', 'config_items']; + tables TEXT[] := ARRAY['components', 'config_items', 'config_analysis']; BEGIN FOREACH tbl IN ARRAY tables LOOP IF NOT EXISTS ( diff --git a/views/006_config_views.sql b/views/006_config_views.sql index f3211f68..7e677538 100644 --- a/views/006_config_views.sql +++ b/views/006_config_views.sql @@ -595,9 +595,18 @@ FOR EACH ROW DROP VIEW IF EXISTS config_analysis_items; +-- Used by resource selector search for config analysis / insights. CREATE OR REPLACE VIEW config_analysis_items AS SELECT ca.*, + ci.name, + ci.deleted_at, + ci.type, + ci.tags, + ci.labels, + ci.config, + ci.agent_id, + ci.path, ci.name as config_name, ci.type as config_type, ci.config_class