add config option for dynamic cpes part filtering (#410)

Signed-off-by: Alex Goodman <[email protected]>

cmd/grype-db/cli/commands/build.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/scylladb/go-set/strset"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
@@ -46,6 +47,9 @@ func Build(app *application.Application) *cobra.Command {
 		Args:    cobra.NoArgs,
 		PreRunE: app.Setup(&cfg),
 		RunE: func(cmd *cobra.Command, _ []string) error {
+			if err := validateCPEParts(cfg.Build.IncludeCPEParts); err != nil {
+				return err
+			}
 			return app.Run(cmd.Context(), async(func() error {
 				return runBuild(cfg)
 			}))
@@ -57,6 +61,19 @@ func Build(app *application.Application) *cobra.Command {
 	return cmd
 }
 
+func validateCPEParts(parts []string) error {
+	validParts := strset.New("a", "o", "h")
+	for _, part := range parts {
+		if !validParts.Has(part) {
+			return fmt.Errorf("invalid CPE part: %s", part)
+		}
+	}
+	if len(parts) == 0 {
+		return errors.New("no CPE parts provided")
+	}
+	return nil
+}
+
 func runBuild(cfg buildConfig) error {
 	// make the db dir if it does not already exist
 	if _, err := os.Stat(cfg.Build.Directory); os.IsNotExist(err) {
@@ -92,10 +109,11 @@ func runBuild(cfg buildConfig) error {
 	}
 
 	return process.Build(process.BuildConfig{
-		SchemaVersion: cfg.SchemaVersion,
-		Directory:     cfg.Directory,
-		States:        states,
-		Timestamp:     earliestTimestamp(states),
+		SchemaVersion:   cfg.SchemaVersion,
+		Directory:       cfg.Directory,
+		States:          states,
+		Timestamp:       earliestTimestamp(states),
+		IncludeCPEParts: cfg.IncludeCPEParts,
 	})
 }
 

cmd/grype-db/cli/options/build.go

@@ -17,14 +17,15 @@ type Build struct {
 	SchemaVersion  int  `yaml:"schema-version" json:"schema-version" mapstructure:"schema-version"`
 
 	// unbound options
-	// (none)
+	IncludeCPEParts []string `yaml:"include-cpe-parts" json:"include-cpe-parts" mapstructure:"include-cpe-parts"`
 }
 
 func DefaultBuild() Build {
 	return Build{
-		DBLocation:     DefaultDBLocation(),
-		SkipValidation: false,
-		SchemaVersion:  process.DefaultSchemaVersion,
+		DBLocation:      DefaultDBLocation(),
+		SkipValidation:  false,
+		SchemaVersion:   process.DefaultSchemaVersion,
+		IncludeCPEParts: []string{"a"},
 	}
 }
 
@@ -54,7 +55,7 @@ func (o *Build) BindFlags(flags *pflag.FlagSet, v *viper.Viper) error {
 	}
 
 	// set default values for non-bound struct items
-	// (none)
+	v.SetDefault("build.include-cpe-parts", o.IncludeCPEParts)
 
 	return o.DBLocation.BindFlags(flags, v)
 }

pkg/process/build.go

@@ -25,10 +25,11 @@ import (
 )
 
 type BuildConfig struct {
-	SchemaVersion int
-	Directory     string
-	States        provider.States
-	Timestamp     time.Time
+	SchemaVersion   int
+	Directory       string
+	States          provider.States
+	Timestamp       time.Time
+	IncludeCPEParts []string
 }
 
 func Build(cfg BuildConfig) error {
@@ -38,7 +39,7 @@ func Build(cfg BuildConfig) error {
 		"providers", cfg.States.Names()).
 		Info("building database")
 
-	processors, err := getProcessors(cfg.SchemaVersion)
+	processors, err := getProcessors(cfg)
 	if err != nil {
 		return err
 	}
@@ -89,8 +90,8 @@ func mergeOpeners(entries []openerEntry) <-chan entry.Opener {
 	return out
 }
 
-func getProcessors(schemaVersion int) ([]data.Processor, error) {
-	switch schemaVersion {
+func getProcessors(cfg BuildConfig) ([]data.Processor, error) {
+	switch cfg.SchemaVersion {
 	case grypeDBv1.SchemaVersion:
 		return v1.Processors(), nil
 	case grypeDBv2.SchemaVersion:
@@ -100,9 +101,9 @@ func getProcessors(schemaVersion int) ([]data.Processor, error) {
 	case grypeDBv4.SchemaVersion:
 		return v4.Processors(), nil
 	case grypeDBv5.SchemaVersion:
-		return v5.Processors(), nil
+		return v5.Processors(v5.NewConfig(v5.WithCPEParts(cfg.IncludeCPEParts))), nil
 	default:
-		return nil, fmt.Errorf("unable to create processor: unsupported schema version: %+v", schemaVersion)
+		return nil, fmt.Errorf("unable to create processor: unsupported schema version: %+v", cfg.SchemaVersion)
 	}
 }
 

pkg/process/v5/processors.go

@@ -1,6 +1,8 @@
 package v5
 
 import (
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/grype-db/pkg/data"
 	"github.com/anchore/grype-db/pkg/process/processors"
 	"github.com/anchore/grype-db/pkg/process/v5/transformers/github"
@@ -10,11 +12,32 @@ import (
 	"github.com/anchore/grype-db/pkg/process/v5/transformers/os"
 )
 
-func Processors() []data.Processor {
+type Config struct {
+	NVD nvd.Config
+}
+
+type Option func(cfg *Config)
+
+func WithCPEParts(included []string) Option {
+	return func(cfg *Config) {
+		cfg.NVD.CPEParts = strset.New(included...)
+	}
+}
+
+func NewConfig(options ...Option) Config {
+	var cfg Config
+	for _, option := range options {
+		option(&cfg)
+	}
+
+	return cfg
+}
+
+func Processors(cfg Config) []data.Processor {
 	return []data.Processor{
 		processors.NewGitHubProcessor(github.Transform),
 		processors.NewMSRCProcessor(msrc.Transform),
-		processors.NewNVDProcessor(nvd.Transform),
+		processors.NewNVDProcessor(nvd.Transformer(cfg.NVD)),
 		processors.NewOSProcessor(os.Transform),
 		processors.NewMatchExclusionProcessor(matchexclusions.Transform),
 	}

pkg/process/v5/transformers/nvd/transform.go

@@ -21,7 +21,26 @@ import (
 	"github.com/anchore/syft/syft/cpe"
 )
 
-func Transform(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) {
+type Config struct {
+	CPEParts *strset.Set
+}
+
+func defaultConfig() Config {
+	return Config{
+		CPEParts: strset.New("a"),
+	}
+}
+
+func Transformer(cfg Config) data.NVDTransformer {
+	if cfg == (Config{}) {
+		cfg = defaultConfig()
+	}
+	return func(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) {
+		return transform(cfg, vulnerability)
+	}
+}
+
+func transform(cfg Config, vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) {
 	// TODO: stop capturing record source in the vulnerability metadata record (now that feed groups are not real)
 	recordSource := "nvdv2:nvdv2:cves"
 
@@ -32,7 +51,7 @@ func Transform(vulnerability unmarshal.NVDVulnerability) ([]data.Entry, error) {
 
 	entryNamespace := grypeNamespace.String()
 
-	uniquePkgs := findUniquePkgs(vulnerability.Configurations...)
+	uniquePkgs := findUniquePkgs(cfg, vulnerability.Configurations...)
 
 	// extract all links
 	var links []string

pkg/process/v5/transformers/nvd/transform_test.go

@@ -32,6 +32,7 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) {
 
 	tests := []struct {
 		name       string
+		config     Config
 		numEntries int
 		fixture    string
 		vulns      []grypeDB.Vulnerability
@@ -708,6 +709,9 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			if test.config == (Config{}) {
+				test.config = defaultConfig()
+			}
 			f, err := os.Open(test.fixture)
 			require.NoError(t, err)
 			t.Cleanup(func() {
@@ -719,7 +723,7 @@ func TestParseAllNVDVulnerabilityEntries(t *testing.T) {
 
 			var vulns []grypeDB.Vulnerability
 			for _, entry := range entries {
-				dataEntries, err := Transform(entry.Cve)
+				dataEntries, err := transform(test.config, entry.Cve)
 				require.NoError(t, err)
 
 				for _, entry := range dataEntries {

pkg/process/v5/transformers/nvd/unique_pkg.go

@@ -31,7 +31,7 @@ func (p pkgCandidate) String() string {
 	return fmt.Sprintf("%s|%s|%s|%s", p.Vendor, p.Product, p.TargetSoftware, p.PlatformCPE)
 }
 
-func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, error) {
+func newPkgCandidate(tCfg Config, match nvd.CpeMatch, platformCPE string) (*pkgCandidate, error) {
 	// we are only interested in packages that are vulnerable (not related to secondary match conditioning)
 	if !match.Vulnerable {
 		return nil, nil
@@ -42,8 +42,9 @@ func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, err
 		return nil, fmt.Errorf("unable to create uniquePkgEntry from '%s': %w", match.Criteria, err)
 	}
 
-	// we are only interested in applications, not hardware or operating systems
-	if c.Part() != cpe.Application {
+	// we are interested in applications, conditionally operating systems, but never hardware
+	part := c.Part()
+	if !tCfg.CPEParts.Has(string(part)) {
 		return nil, nil
 	}
 
@@ -55,15 +56,15 @@ func newPkgCandidate(match nvd.CpeMatch, platformCPE string) (*pkgCandidate, err
 	}, nil
 }
 
-func findUniquePkgs(cfgs ...nvd.Configuration) uniquePkgTracker {
+func findUniquePkgs(tCfg Config, cfgs ...nvd.Configuration) uniquePkgTracker {
 	set := newUniquePkgTracker()
 	for _, c := range cfgs {
-		_findUniquePkgs(set, c)
+		_findUniquePkgs(tCfg, set, c)
 	}
 	return set
 }
 
-func platformPackageCandidates(set uniquePkgTracker, c nvd.Configuration) bool {
+func platformPackageCandidates(tCfg Config, set uniquePkgTracker, c nvd.Configuration) bool {
 	nodes := c.Nodes
 	/*
 		Turn a configuration like this:
@@ -108,7 +109,7 @@ func platformPackageCandidates(set uniquePkgTracker, c nvd.Configuration) bool {
 	for _, application := range applicationNode.CpeMatch {
 		for _, maybePlatform := range matches {
 			platform := maybePlatform.Criteria
-			candidate, err := newPkgCandidate(application, platform)
+			candidate, err := newPkgCandidate(tCfg, application, platform)
 			if err != nil {
 				log.Debugf("unable processing uniquePkg with multiple platforms: %v", err)
 				continue
@@ -151,18 +152,18 @@ func noCPEsVulnerable(node nvd.Node) bool {
 	return true
 }
 
-func _findUniquePkgs(set uniquePkgTracker, c nvd.Configuration) {
+func _findUniquePkgs(tCfg Config, set uniquePkgTracker, c nvd.Configuration) {
 	if len(c.Nodes) == 0 {
 		return
 	}
 
-	if platformPackageCandidates(set, c) {
+	if platformPackageCandidates(tCfg, set, c) {
 		return
 	}
 
 	for _, node := range c.Nodes {
 		for _, match := range node.CpeMatch {
-			candidate, err := newPkgCandidate(match, "")
+			candidate, err := newPkgCandidate(tCfg, match, "")
 			if err != nil {
 				// Do not halt all execution because of being unable to create
 				// a PkgCandidate. This can happen when a CPE is invalid which