diff --git a/cmd/grype-db/cli/commands/build.go b/cmd/grype-db/cli/commands/build.go index 43b3ba77..79dc6f73 100644 --- a/cmd/grype-db/cli/commands/build.go +++ b/cmd/grype-db/cli/commands/build.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/scylladb/go-set/strset" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -46,6 +47,9 @@ func Build(app *application.Application) *cobra.Command { Args: cobra.NoArgs, PreRunE: app.Setup(&cfg), RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateCPEParts(cfg.Build.IncludeCPEParts); err != nil { + return err + } return app.Run(cmd.Context(), async(func() error { return runBuild(cfg) })) @@ -57,6 +61,19 @@ func Build(app *application.Application) *cobra.Command { return cmd } +func validateCPEParts(parts []string) error { + validParts := strset.New("a", "o", "h") + for _, part := range parts { + if !validParts.Has(part) { + return fmt.Errorf("invalid CPE part: %s", part) + } + } + if len(parts) == 0 { + return errors.New("no CPE parts provided") + } + return nil +} + func runBuild(cfg buildConfig) error { // make the db dir if it does not already exist if _, err := os.Stat(cfg.Build.Directory); os.IsNotExist(err) { @@ -92,10 +109,11 @@ func runBuild(cfg buildConfig) error { } return process.Build(process.BuildConfig{ - SchemaVersion: cfg.SchemaVersion, - Directory: cfg.Directory, - States: states, - Timestamp: earliestTimestamp(states), + SchemaVersion: cfg.SchemaVersion, + Directory: cfg.Directory, + States: states, + Timestamp: earliestTimestamp(states), + IncludeCPEParts: cfg.IncludeCPEParts, }) } diff --git a/cmd/grype-db/cli/options/build.go b/cmd/grype-db/cli/options/build.go index 1db55b71..1014dc5b 100644 --- a/cmd/grype-db/cli/options/build.go +++ b/cmd/grype-db/cli/options/build.go @@ -17,14 +17,15 @@ type Build struct { SchemaVersion int `yaml:"schema-version" json:"schema-version" mapstructure:"schema-version"` // unbound options - // (none) + IncludeCPEParts []string `yaml:"include-cpe-parts" json:"include-cpe-parts" mapstructure:"include-cpe-parts"` } func DefaultBuild() Build { return Build{ - DBLocation: DefaultDBLocation(), - SkipValidation: false, - SchemaVersion: process.DefaultSchemaVersion, + DBLocation: DefaultDBLocation(), + SkipValidation: false, + SchemaVersion: process.DefaultSchemaVersion, + IncludeCPEParts: []string{"a"}, } } @@ -54,7 +55,7 @@ func (o *Build) BindFlags(flags *pflag.FlagSet, v *viper.Viper) error { } // set default values for non-bound struct items - // (none) + v.SetDefault("build.include-cpe-parts", o.IncludeCPEParts) return o.DBLocation.BindFlags(flags, v) } diff --git a/pkg/process/build.go b/pkg/process/build.go index 16254505..12efe4ad 100644 --- a/pkg/process/build.go +++ b/pkg/process/build.go @@ -25,10 +25,11 @@ import ( ) type BuildConfig struct { - SchemaVersion int - Directory string - States provider.States - Timestamp time.Time + SchemaVersion int + Directory string + States provider.States + Timestamp time.Time + IncludeCPEParts []string } func Build(cfg BuildConfig) error { @@ -38,7 +39,7 @@ func Build(cfg BuildConfig) error { "providers", cfg.States.Names()). Info("building database") - processors, err := getProcessors(cfg.SchemaVersion) + processors, err := getProcessors(cfg) if err != nil { return err } @@ -89,8 +90,8 @@ func mergeOpeners(entries []openerEntry) <-chan entry.Opener { return out } -func getProcessors(schemaVersion int) ([]data.Processor, error) { - switch schemaVersion { +func getProcessors(cfg BuildConfig) ([]data.Processor, error) { + switch cfg.SchemaVersion { case grypeDBv1.SchemaVersion: return v1.Processors(), nil case grypeDBv2.SchemaVersion: @@ -100,9 +101,9 @@ func getProcessors(schemaVersion int) ([]data.Processor, error) { case grypeDBv4.SchemaVersion: return v4.Processors(), nil case grypeDBv5.SchemaVersion: - return v5.Processors(), nil + return v5.Processors(v5.NewConfig(v5.WithCPEParts(cfg.IncludeCPEParts))), nil default: - return nil, fmt.Errorf("unable to create processor: unsupported schema version: %+v", schemaVersion) + return nil, fmt.Errorf("unable to create processor: unsupported schema version: %+v", cfg.SchemaVersion) } } diff --git a/pkg/process/v5/processors.go b/pkg/process/v5/processors.go index cdc5b982..238c812c 100644 --- a/pkg/process/v5/processors.go +++ b/pkg/process/v5/processors.go @@ -1,6 +1,8 @@ package v5 import ( + "github.com/scylladb/go-set/strset" + "github.com/anchore/grype-db/pkg/data" "github.com/anchore/grype-db/pkg/process/processors" "github.com/anchore/grype-db/pkg/process/v5/transformers/github" @@ -10,11 +12,32 @@ import ( "github.com/anchore/grype-db/pkg/process/v5/transformers/os" ) -func Processors() []data.Processor { +type Config struct { + NVD nvd.Config +} + +type Option func(cfg *Config) + +func WithCPEParts(included []string) Option { + return func(cfg *Config) { + cfg.NVD.CPEParts = strset.New(included...) + } +} + +func NewConfig(options ...Option) Config { + var cfg Config + for _, option := range options { + option(&cfg) + } + + return cfg +} + +func Processors(cfg Config) []data.Processor { return []data.Processor{ processors.NewGitHubProcessor(github.Transform), processors.NewMSRCProcessor(msrc.Transform), - processors.NewNVDProcessor(nvd.Transform), + processors.NewNVDProcessor(nvd.Transformer(cfg.NVD)), processors.NewOSProcessor(os.Transform), processors.NewMatchExclusionProcessor(matchexclusions.Transform), } diff --git a/pkg/process/v5/transformers/nvd/transform.go b/pkg/process/v5/transformers/nvd/transform.go index c3eb06f9..6cff6964 100644 --- a/pkg/process/v5/transformers/nvd/transform.go +++ b/pkg/process/v5/transformers/nvd/transform.go @@ -21,7 +21,26 @@ import ( "github.com/anchore/syft/syft/cpe" ) -func Transform(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) { +type Config struct { + CPEParts *strset.Set +} + +func defaultConfig() Config { + return Config{ + CPEParts: strset.New("a"), + } +} + +func Transformer(cfg Config) data.NVDTransformer { + if cfg == (Config{}) { + cfg = defaultConfig() + } + return func(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) { + return transform(cfg, vulnerability) + } +} + +func transform(cfg Config, vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) { // TODO: stop capturing record source in the vulnerability metadata record (now that feed groups are not real) recordSource := "nvdv2:nvdv2:cves" @@ -32,7 +51,7 @@ func Transform(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) { entryNamespace := grypeNamespace.String() - uniquePkgs := findUniquePkgs(vulnerability.Configurations...) + uniquePkgs := findUniquePkgs(cfg, vulnerability.Configurations...) // extract all links var links []string diff --git a/pkg/process/v5/transformers/nvd/transform_test.go b/pkg/process/v5/transformers/nvd/transform_test.go index 41312e48..59e0a013 100644 --- a/pkg/process/v5/transformers/nvd/transform_test.go +++ b/pkg/process/v5/transformers/nvd/transform_test.go @@ -32,6 +32,7 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) { tests := []struct { name string + config Config numEntries int fixture string vulns []grypeDB.Vulnerability @@ -708,6 +709,9 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + if test.config == (Config{}) { + test.config = defaultConfig() + } f, err := os.Open(test.fixture) require.NoError(t, err) t.Cleanup(func() { @@ -719,7 +723,7 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) { var vulns []grypeDB.Vulnerability for _, entry := range entries { - dataEntries, err := Transform(entry.Cve) + dataEntries, err := transform(test.config, entry.Cve) require.NoError(t, err) for _, entry := range dataEntries { diff --git a/pkg/process/v5/transformers/nvd/unique_pkg.go b/pkg/process/v5/transformers/nvd/unique_pkg.go index 8718359a..b1284ebf 100644 --- a/pkg/process/v5/transformers/nvd/unique_pkg.go +++ b/pkg/process/v5/transformers/nvd/unique_pkg.go @@ -31,7 +31,7 @@ func (p pkgCandidate) String() string { return fmt.Sprintf("%s|%s|%s|%s", p.Vendor, p.Product, p.TargetSoftware, p.PlatformCPE) } -func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, error) { +func newPkgCandidate(tCfg Config, match nvd.CpeMatch, platformCPE string) (*pkgCandidate, error) { // we are only interested in packages that are vulnerable (not related to secondary match conditioning) if !match.Vulnerable { return nil, nil @@ -42,8 +42,9 @@ func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, err return nil, fmt.Errorf("unable to create uniquePkgEntry from '%s': %w", match.Criteria, err) } - // we are only interested in applications, not hardware or operating systems - if c.Part() != cpe.Application { + // we are interested in applications, conditionally operating systems, but never hardware + part := c.Part() + if !tCfg.CPEParts.Has(string(part)) { return nil, nil } @@ -55,15 +56,15 @@ func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, err }, nil } -func findUniquePkgs(cfgs ...nvd.Configuration) uniquePkgTracker { +func findUniquePkgs(tCfg Config, cfgs ...nvd.Configuration) uniquePkgTracker { set := newUniquePkgTracker() for _, c := range cfgs { - _findUniquePkgs(set, c) + _findUniquePkgs(tCfg, set, c) } return set } -func platformPackageCandidates(set uniquePkgTracker, c nvd.Configuration) bool { +func platformPackageCandidates(tCfg Config, set uniquePkgTracker, c nvd.Configuration) bool { nodes := c.Nodes /* Turn a configuration like this: @@ -108,7 +109,7 @@ func platformPackageCandidates(set uniquePkgTracker, c nvd.Configuration) bool { for _, application := range applicationNode.CpeMatch { for _, maybePlatform := range matches { platform := maybePlatform.Criteria - candidate, err := newPkgCandidate(application, platform) + candidate, err := newPkgCandidate(tCfg, application, platform) if err != nil { log.Debugf("unable processing uniquePkg with multiple platforms: %v", err) continue @@ -151,18 +152,18 @@ func noCPEsVulnerable(node nvd.Node) bool { return true } -func _findUniquePkgs(set uniquePkgTracker, c nvd.Configuration) { +func _findUniquePkgs(tCfg Config, set uniquePkgTracker, c nvd.Configuration) { if len(c.Nodes) == 0 { return } - if platformPackageCandidates(set, c) { + if platformPackageCandidates(tCfg, set, c) { return } for _, node := range c.Nodes { for _, match := range node.CpeMatch { - candidate, err := newPkgCandidate(match, "") + candidate, err := newPkgCandidate(tCfg, match, "") if err != nil { // Do not halt all execution because of being unable to create // a PkgCandidate. This can happen when a CPE is invalid which diff --git a/pkg/process/v5/transformers/nvd/unique_pkg_test.go b/pkg/process/v5/transformers/nvd/unique_pkg_test.go index de3f7783..d178578b 100644 --- a/pkg/process/v5/transformers/nvd/unique_pkg_test.go +++ b/pkg/process/v5/transformers/nvd/unique_pkg_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/scylladb/go-set/strset" "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +23,7 @@ func newUniquePkgTrackerFromSlice(candidates []pkgCandidate) uniquePkgTracker { func TestFindUniquePkgs(t *testing.T) { tests := []struct { name string + config Config nodes []nvd.Node operator *nvd.Operator expected uniquePkgTracker @@ -62,7 +64,7 @@ func TestFindUniquePkgs(t *testing.T) { expected: newUniquePkgTrackerFromSlice([]pkgCandidate{}), }, { - name: "skip-os", + name: "skip-os-by-default", nodes: []nvd.Node{ { CpeMatch: []nvd.CpeMatch{ @@ -75,6 +77,29 @@ func TestFindUniquePkgs(t *testing.T) { }, expected: newUniquePkgTrackerFromSlice([]pkgCandidate{}), }, + { + name: "include-os-explicitly", + config: Config{ + CPEParts: strset.New("a", "o"), + }, + nodes: []nvd.Node{ + { + CpeMatch: []nvd.CpeMatch{ + { + Criteria: "cpe:2.3:o:vendor:product:2.2.0:*:*:*:*:target:*:*", + Vulnerable: true, + }, + }, + }, + }, + expected: newUniquePkgTrackerFromSlice([]pkgCandidate{ + { + Product: "product", + Vendor: "vendor", + TargetSoftware: "target", + }, + }), + }, { name: "duplicate-by-product", nodes: []nvd.Node{ @@ -402,7 +427,10 @@ func TestFindUniquePkgs(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - actual := findUniquePkgs(nvd.Configuration{Nodes: test.nodes, Operator: test.operator}) + if test.config == (Config{}) { + test.config = defaultConfig() + } + actual := findUniquePkgs(test.config, nvd.Configuration{Nodes: test.nodes, Operator: test.operator}) missing, extra := test.expected.Diff(actual) if len(missing) != 0 { for _, c := range missing { @@ -566,7 +594,7 @@ func Test_UniquePackageTrackerHandlesOnlyPlatformDiff(t *testing.T) { } tracker := newUniquePkgTracker() for _, c := range candidates { - candidate, err := newPkgCandidate(applicationNode, c.PlatformCPE) + candidate, err := newPkgCandidate(defaultConfig(), applicationNode, c.PlatformCPE) require.NoError(t, err) tracker.Add(*candidate, cpeMatch) } @@ -576,14 +604,15 @@ func Test_UniquePackageTrackerHandlesOnlyPlatformDiff(t *testing.T) { func TestPlatformPackageCandidates(t *testing.T) { type testCase struct { name string - config nvd.Configuration + config Config + state nvd.Configuration wantChanged bool wantSet uniquePkgTracker } tests := []testCase{ { name: "application X platform", - config: nvd.Configuration{ + state: nvd.Configuration{ Negate: nil, Nodes: []nvd.Node{ { @@ -641,7 +670,7 @@ func TestPlatformPackageCandidates(t *testing.T) { }, { name: "top-level OR is excluded", - config: nvd.Configuration{ + state: nvd.Configuration{ Operator: opRef(nvd.Or), }, wantChanged: false, @@ -649,14 +678,14 @@ func TestPlatformPackageCandidates(t *testing.T) { }, { name: "top-level nil op is excluded", - config: nvd.Configuration{ + state: nvd.Configuration{ Operator: nil, }, wantChanged: false, }, { name: "single hardware node results in exclusion", - config: nvd.Configuration{ + state: nvd.Configuration{ Negate: nil, Nodes: []nvd.Node{ { @@ -695,8 +724,11 @@ func TestPlatformPackageCandidates(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + if tc.config == (Config{}) { + tc.config = defaultConfig() + } set := newUniquePkgTracker() - result := platformPackageCandidates(set, tc.config) + result := platformPackageCandidates(tc.config, set, tc.state) assert.Equal(t, result, tc.wantChanged) if tc.wantSet == nil { tc.wantSet = newUniquePkgTracker() @@ -717,8 +749,18 @@ func boolRef(b bool) *bool { return &b } -func mustNewPackage(t *testing.T, match nvd.CpeMatch, platformCPE string) pkgCandidate { - p, err := newPkgCandidate(match, platformCPE) +func mustNewPackage(t *testing.T, match nvd.CpeMatch, platformCPE string, cfg ...Config) pkgCandidate { + var tCfg *Config + switch len(cfg) { + case 0: + c := defaultConfig() + tCfg = &c + case 1: + tCfg = &cfg[0] + default: + t.Fatalf("too many configs provided") + } + p, err := newPkgCandidate(*tCfg, match, platformCPE) require.NoError(t, err) return *p }