From 32873c2029229799a008318ff9cf5ac46e29315c Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Thu, 27 Jul 2023 11:24:24 -0400 Subject: [PATCH] :sparkles: Deduplicate dependencies from a single source (#270) closes #259 --------- Signed-off-by: Fabian von Feilitzsch --- demo-dep-output.yaml | 175 -------------------------------------- provider/provider.go | 37 ++++++++ provider/provider_test.go | 94 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 175 deletions(-) diff --git a/demo-dep-output.yaml b/demo-dep-output.yaml index 5281fc1d..64b83b00 100644 --- a/demo-dep-output.yaml +++ b/demo-dep-output.yaml @@ -66,11 +66,6 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/modern-go/reflect2 - version: v1.0.2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: golang.org/x/net version: v0.0.0-20220127200216-cd36cc0744dd labels: @@ -91,11 +86,6 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: golang.org/x/text - version: v0.3.7 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: golang.org/x/tools version: v0.0.0-20180917221912-90fa682c2a6e labels: @@ -126,11 +116,6 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/gogo/protobuf - version: v1.3.2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/google/cel-go version: v0.10.1 labels: @@ -146,11 +131,6 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/google/gofuzz - version: v1.1.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/google/uuid version: v1.1.2 labels: @@ -201,11 +181,6 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: gopkg.in/yaml.v2 - version: v2.4.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: k8s.io/api version: v0.24.4 labels: @@ -266,21 +241,11 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: k8s.io/apimachinery - version: v0.24.4 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/armon/go-socks5 version: v0.0.0-20160902184237-e75332964ef5 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/davecgh/go-spew - version: v1.1.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/elazarl/goproxy version: v0.0.0-20180725130230-947c36da3153 labels: @@ -291,41 +256,11 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/gogo/protobuf - version: v1.3.2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/golang/protobuf version: v1.5.2 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/google/gnostic - version: v0.5.7-v3refs - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/google/go-cmp - version: v0.5.5 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/google/gofuzz - version: v1.1.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/google/uuid - version: v1.1.2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/json-iterator/go - version: v1.1.12 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/kr/text version: v0.2.0 labels: @@ -356,151 +291,41 @@ labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/spf13/pflag - version: v1.0.5 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/stretchr/testify - version: v1.7.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: golang.org/x/net - version: v0.0.0-20220127200216-cd36cc0744dd - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: golang.org/x/sys version: v0.0.0-20220209214540-3681064d5158 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: google.golang.org/protobuf - version: v1.27.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: gopkg.in/check.v1 version: v1.0.0-20200227125254-8fa46927fb4f labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: gopkg.in/inf.v0 - version: v0.9.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: gopkg.in/yaml.v2 - version: v2.4.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: k8s.io/klog/v2 - version: v2.60.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: k8s.io/kube-openapi - version: v0.0.0-20220328201542-3ee0da9b0b42 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: k8s.io/utils - version: v0.0.0-20220210201930-3a6ce19ff2f9 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: sigs.k8s.io/json - version: v0.0.0-20211208200746-9f7c6b3444d2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: sigs.k8s.io/structured-merge-diff/v4 - version: v4.2.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: sigs.k8s.io/yaml - version: v1.2.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: k8s.io/klog/v2 - version: v2.60.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/go-logr/logr - version: v1.2.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: k8s.io/utils - version: v0.0.0-20220210201930-3a6ce19ff2f9 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: github.com/davecgh/go-spew - version: v1.1.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/spf13/afero version: v1.2.2 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/stretchr/testify - version: v1.3.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: k8s.io/klog/v2 version: v2.0.0 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: sigs.k8s.io/json - version: v0.0.0-20211208200746-9f7c6b3444d2 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - - name: sigs.k8s.io/structured-merge-diff/v4 - version: v4.2.1 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: gopkg.in/yaml.v2 version: v2.2.8 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/google/gofuzz - version: v1.0.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/json-iterator/go version: v1.1.6 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/modern-go/concurrent - version: v0.0.0-20180306012644-bacd9c7ef1dd - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - name: github.com/modern-go/reflect2 version: v1.0.1 labels: - konveyor.io/dep-source=downloadable - konveyor.io/language=go - - name: github.com/stretchr/testify - version: v1.3.0 - labels: - - konveyor.io/dep-source=downloadable - - konveyor.io/language=go - fileURI: file:///analyzer-lsp/examples/customers-tomcat-legacy/pom.xml provider: java dependencies: diff --git a/provider/provider.go b/provider/provider.go index f4f263f1..346a731d 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -244,6 +244,7 @@ func FullDepsResponse(clients []ServiceClient) (map[uri.URI][]*Dep, error) { for k, v := range r { deps[k] = v } + deps = deduplicateDependencies(deps) } return deps, nil } @@ -373,6 +374,7 @@ func (p *ProviderCondition) Evaluate(ctx context.Context, log logr.Logger, condC if err != nil { return engine.ConditionResponse{}, err } + deps = deduplicateDependencies(deps) } incidents := []engine.IncidentContext{} @@ -620,3 +622,38 @@ func ConvertDagItemsToList(items []DepDAGItem) []*Dep { } return deps } + +func deduplicateDependencies(dependencies map[uri.URI][]*Dep) map[uri.URI][]*Dep { + // Just need this so I can differentiate between dependencies that aren't found + // and dependencies that are at index 0 + intPtr := func(i int) *int { + return &i + } + deduped := map[uri.URI][]*Dep{} + for uri, deps := range dependencies { + deduped[uri] = []*Dep{} + depSeen := map[string]*int{} + for _, dep := range deps { + id := dep.Name + dep.Version + dep.ResolvedIdentifier + if depSeen[id+"direct"] != nil { + // We've already seen it and it's direct, nothing to do + continue + } else if depSeen[id+"indirect"] != nil && !dep.Indirect { + // We've seen it as an indirect, need to update the dep in + // the list to reflect that it's actually a direct dependency + deduped[uri][*depSeen[id+"indirect"]].Indirect = false + depSeen[id+"direct"] = depSeen[id+"indirect"] + } else { + // We haven't seen this before and need to update the dedup + // list and mark that we've seen it + deduped[uri] = append(deduped[uri], dep) + if dep.Indirect { + depSeen[id+"indirect"] = intPtr(len(deduped) - 1) + } else { + depSeen[id+"direct"] = intPtr(len(deduped) - 1) + } + } + } + } + return deduped +} diff --git a/provider/provider_test.go b/provider/provider_test.go index e943e45f..c02ff56c 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -2,6 +2,7 @@ package provider import ( "context" + "reflect" "testing" "github.com/go-logr/logr" @@ -241,3 +242,96 @@ func Test_matchDepLabelSelector(t *testing.T) { }) } } + +func Test_deduplication(t *testing.T) { + tests := []struct { + title string + dependencies map[uri.URI][]*Dep + expected map[uri.URI][]*Dep + }{ + { + title: "no duplicates within a file should result in an unchanged list", + dependencies: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + uri.URI("file2"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + expected: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + uri.URI("file2"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + }, + { + title: "different versions or shas of the same dependency should not be deduped", + dependencies: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep1", Version: "v2.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcde"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcdf"}, + }, + }, + expected: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep1", Version: "v2.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcde"}, + {Name: "dep2", Version: "v1.0.0", ResolvedIdentifier: "abcdf"}, + }, + }, + }, + { + title: "duplicates within a file should be removed", + dependencies: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + expected: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + }, + { + title: "direct dependencies should be preferred over indirect", + dependencies: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd", Indirect: true}, + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + expected: map[uri.URI][]*Dep{ + uri.URI("file1"): []*Dep{ + {Name: "dep1", Version: "v1.0.0", ResolvedIdentifier: "abcd"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.title, func(t *testing.T) { + deduped := deduplicateDependencies(tt.dependencies) + for uri, deps := range tt.expected { + for i, dep := range deps { + if !reflect.DeepEqual(deduped[uri][i], dep) { + t.Errorf("Expected '%+v', got '%+v'", tt.expected, deduped) + } + } + } + }) + } + +}