refactor(npm): refactor lockfile v1 analyzing

module/npm/lockfile_v1.go

@@ -1,78 +1,24 @@
 package npm
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/murphysecurity/murphysec/model"
+	"github.com/murphysecurity/murphysec/module/npm/shared"
+	v1 "github.com/murphysecurity/murphysec/module/npm/v1"
 	"github.com/samber/lo"
 )
 
-type v1Dep struct {
-	Version      string            `json:"version"`
-	Dependencies map[string]v1Dep  `json:"dependencies"`
-	Dev          bool              `json:"dev"`
-	Optional     bool              `json:"optional"`
-	Requires     map[string]string `json:"requires"`
-}
-
-type v1Lockfile struct {
-	Name         string           `json:"name"`
-	Dependencies map[string]v1Dep `json:"dependencies"`
-}
-
-func processV1Lockfile(data []byte, requires []string) ([]model.DependencyItem, error) {
-	var e error
-	var lockfile v1Lockfile
-	e = json.Unmarshal(data, &lockfile)
+func processV1Lockfile(data []byte, pkg *pkgFile) ([]model.DependencyItem, error) {
+	lf, e := v1.ParseLockfile(data)
 	if e != nil {
-		return nil, fmt.Errorf("parsing v1 lockfile: bad format, %w", e)
-	}
-	requires = lo.Uniq(requires)
-	var r []model.DependencyItem
-	for _, depName := range requires {
-		if dep, ok := lockfile.Dependencies[depName]; ok {
-			if rr := v1ConvDepRecursive(depName, dep, []map[string]v1Dep{lockfile.Dependencies}, make(map[string]struct{})); rr != nil {
-				r = append(r, *rr)
-			}
-		}
-	}
-	return r, nil
-}
-
-func v1ConvDepRecursive(name string, dep v1Dep, pp []map[string]v1Dep, visited map[string]struct{}) *model.DependencyItem {
-	if _, ok := visited[name]; ok {
-		return nil
-	}
-	visited[name] = struct{}{}
-	defer func() { delete(visited, name) }()
-	r := model.DependencyItem{
-		Component: model.Component{
-			CompName:    name,
-			CompVersion: dep.Version,
-			EcoRepo:     EcoRepo,
-		},
+		return nil, e
 	}
-	if dep.Dev {
-		r.IsOnline.SetOnline(false)
-	}
-	var queryMaps = make([]map[string]v1Dep, len(pp), len(pp)+1)
-	copy(queryMaps, pp)
-	if dep.Dependencies != nil {
-		queryMaps = append(queryMaps, dep.Dependencies)
-	}
-o:
-	for depName := range dep.Requires {
-		queryMapOff := len(queryMaps) - 1
-		for queryMapOff >= 0 {
-			if child, ok := queryMaps[queryMapOff][depName]; ok {
-				rr := v1ConvDepRecursive(depName, child, queryMaps[:queryMapOff+1], visited)
-				if rr != nil {
-					r.Dependencies = append(r.Dependencies, *rr)
-				}
-				continue o
-			}
-			queryMapOff--
-		}
+	entries := pkg.DependenciesEntries()
+	entries = append(entries, pkg.DevDependenciesEntries()...)
+	lo.Uniq(entries)
+	nodes, e := lf.Build(pkg.DependenciesEntries(), false)
+	if e != nil {
+		return nil, fmt.Errorf("build dependencies tree: %w", e)
 	}
-	return &r
+	return shared.ConvNodes(nodes), nil
 }

module/npm/npm.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"github.com/murphysecurity/murphysec/model"
 	"github.com/murphysecurity/murphysec/utils"
-	"github.com/samber/lo"
 	"os"
 	"path/filepath"
 )
@@ -76,10 +75,7 @@ func ScanNpmProject(ctx context.Context) ([]model.Module, error) {
 
 	module.ModuleName = packageFile.Name
 	module.ModuleVersion = packageFile.Version
-	var requires = lo.Keys(packageFile.Dependencies)
-	requires = append(requires, lo.Keys(packageFile.DevDependencies)...)
-	requires = lo.Uniq(requires)
-	deps, e := processV1Lockfile(data, requires)
+	deps, e := processV1Lockfile(data, packageFile)
 	if e != nil {
 		return nil, e
 	}

module/npm/package_file.go

@@ -3,6 +3,7 @@ package npm
 import (
 	"encoding/json"
 	"fmt"
+	"sort"
 )
 
 type pkgFile struct {
@@ -12,6 +13,33 @@ type pkgFile struct {
 	DevDependencies map[string]string `json:"devDependencies"`
 }
 
+func (p pkgFile) DependenciesEntries() [][2]string {
+	var r [][2]string
+	for n, v := range p.Dependencies {
+		r = append(r, [2]string{n, v})
+	}
+	sortEntries(r)
+	return r
+}
+
+func (p pkgFile) DevDependenciesEntries() [][2]string {
+	var r [][2]string
+	for n, v := range p.DevDependencies {
+		r = append(r, [2]string{n, v})
+	}
+	sortEntries(r)
+	return r
+}
+
+func sortEntries(input [][2]string) {
+	sort.Slice(input, func(i, j int) bool {
+		if input[i][0] == input[j][0] {
+			return input[i][1] < input[j][1]
+		}
+		return input[i][0] < input[j][0]
+	})
+}
+
 func parsePkgFile(data []byte) (*pkgFile, error) {
 	var e error
 	var r pkgFile

module/npm/shared/component.go

@@ -0,0 +1,57 @@
+package shared
+
+import (
+	"fmt"
+	"github.com/murphysecurity/murphysec/model"
+)
+
+type Node struct {
+	Name     string
+	Version  string
+	Children []*Node
+	model.IsOnline
+	Dev bool
+}
+
+type dependencyNotFoundError struct {
+	name    string
+	version string
+}
+
+func (c dependencyNotFoundError) Error() string {
+	return fmt.Sprintf("dependency not found: %s@%s", c.name, c.version)
+}
+
+func CreateDependencyNotFoundError(name, version string) error {
+	return &dependencyNotFoundError{name: name, version: version}
+}
+
+func ConvNodes(input []*Node) []model.DependencyItem {
+	var r = _ConvNodes0(input)
+	for i := range r {
+		r[i].IsDirectDependency = true
+	}
+	return r
+}
+
+func _ConvNodes0(input []*Node) []model.DependencyItem {
+	var r []model.DependencyItem
+	for _, node := range input {
+		d := model.DependencyItem{
+			Component: model.Component{
+				CompName:    node.Name,
+				CompVersion: node.Version,
+				EcoRepo:     EcoRepo,
+			},
+			Dependencies: _ConvNodes0(node.Children),
+			IsOnline:     node.IsOnline,
+		}
+		if node.Dev {
+			d.IsOnline.SetOnline(false)
+		}
+		r = append(r, d)
+	}
+	return r
+}
+
+var EcoRepo = model.EcoRepo{Ecosystem: "npm"}

module/npm/shared/visited.go

@@ -0,0 +1,68 @@
+package shared
+
+import (
+	"github.com/repeale/fp-go"
+	"github.com/samber/lo"
+	"strings"
+)
+
+type Visited struct {
+	name    string
+	version string
+	parent  *Visited
+}
+
+func (v *Visited) Depth() int {
+	var depth = 0
+	var curr = v
+	for curr != nil {
+		curr = curr.parent
+		depth++
+	}
+	return depth
+}
+
+func (v *Visited) Contains(name, version string) bool {
+	var curr = v
+	for curr != nil {
+		if curr.name == name && curr.version == version {
+			return true
+		}
+		curr = curr.parent
+	}
+	return false
+}
+
+func (v *Visited) CreateSub(name, version string) *Visited {
+	if v.Contains(name, version) {
+		return nil
+	}
+	return &Visited{name: name, version: version, parent: v}
+}
+
+func CreateVisited(name, version string) *Visited {
+	return &Visited{
+		name:    name,
+		version: version,
+	}
+}
+
+type revisitError struct {
+	v *Visited
+}
+
+func (r revisitError) Error() string {
+	var arr [][2]string
+	var curr = r.v
+	for curr != nil {
+		arr = append(arr, [2]string{curr.name, curr.version})
+		curr = curr.parent
+	}
+	lo.Reverse(arr)
+	s := strings.Join(fp.Map(func(a [2]string) string { return a[0] + "@" + a[1] })(arr), " -> ")
+	return "revisit: " + s
+}
+
+func CreateRevisitError(v *Visited) error {
+	return revisitError{v: v}
+}

module/npm/v1/model.go

@@ -0,0 +1,102 @@
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/murphysecurity/murphysec/module/npm/shared"
+)
+
+type Lockfile struct {
+	root lockRoot
+}
+
+func (l *Lockfile) Build(requires [][2]string, strict bool) ([]*shared.Node, error) {
+	var r []*shared.Node
+	for _, require := range requires {
+		name := require[0]
+		versionCons := require[1]
+		n, e := buildTree(name, versionCons, &l.root.lockPkg, nil, strict)
+		if e != nil {
+			if !strict {
+				continue
+			}
+			return nil, fmt.Errorf("v1.ParseLockfile: %w", e)
+		}
+		r = append(r, n)
+	}
+	return r, nil
+}
+
+type lockPkg struct {
+	Version      string              `json:"version"`
+	Optional     *bool               `json:"optional"`
+	Requires     map[string]string   `json:"requires"`
+	Dependencies map[string]*lockPkg `json:"dependencies"`
+	Dev          *bool               `json:"dev"`
+	parent       *lockPkg
+}
+
+type lockRoot struct {
+	Name string `json:"name"`
+	lockPkg
+	Requires bool `json:"requires"`
+}
+
+func postprocessPkg(pkg *lockPkg, parent *lockPkg) {
+	pkg.parent = parent
+	for _, p := range pkg.Dependencies {
+		postprocessPkg(p, pkg)
+	}
+}
+
+func buildTree(name string, versionConstraint string, current *lockPkg, visited *shared.Visited, strict bool) (*shared.Node, error) {
+	childVisited := visited.CreateSub(name, versionConstraint)
+	if childVisited == nil {
+		return nil, shared.CreateRevisitError(visited)
+	}
+	for current != nil {
+		childPkg := current.Dependencies[name]
+		if childPkg == nil {
+			current = current.parent
+			continue
+		}
+		// found
+		var node shared.Node
+		node.Name = name
+		node.Version = childPkg.Version
+		if childPkg.Optional != nil {
+			node.IsOnline.SetOnline(!*childPkg.Optional)
+		}
+		if childPkg.Dev != nil {
+			node.Dev = *childPkg.Dev
+		} else {
+			node.Dev = false
+		}
+		for childName, versionCons := range childPkg.Requires {
+			childNode, e := buildTree(childName, versionCons, childPkg, childVisited, strict)
+			if e != nil {
+				if !strict {
+					continue
+				}
+				return nil, e
+			}
+			if childNode == nil {
+				panic("childNode == nil")
+			}
+			node.Children = append(node.Children, childNode)
+		}
+		return &node, nil
+	}
+	return nil, shared.CreateDependencyNotFoundError(name, versionConstraint)
+}
+
+func ParseLockfile(data []byte) (*Lockfile, error) {
+	var e error
+	var lockRoot lockRoot
+	e = json.Unmarshal(data, &lockRoot)
+	if e != nil {
+		return nil, fmt.Errorf("v1.ParseLockfile: %w", e)
+	}
+	postprocessPkg(&lockRoot.lockPkg, nil)
+	return &Lockfile{lockRoot}, e
+}