Skip to content

Commit

Permalink
fix(npm): lockfile
Browse files Browse the repository at this point in the history
  • Loading branch information
iseki0 committed Jul 10, 2023
1 parent b57fa90 commit 7c1ea83
Show file tree
Hide file tree
Showing 7 changed files with 148 additions and 107 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/samber/lo v1.38.1 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
Expand Down
78 changes: 78 additions & 0 deletions module/npm/lockfile_v1.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package npm

import (
"encoding/json"
"fmt"
"github.com/murphysecurity/murphysec/model"
"github.com/samber/lo"
)

type v1Dep struct {
Version string `json:"version"`
Dependencies map[string]v1Dep `json:"dependencies"`
Dev bool `json:"dev"`
Optional bool `json:"optional"`
Requires map[string]string `json:"requires"`
}

type v1Lockfile struct {
Name string `json:"name"`
Dependencies map[string]v1Dep `json:"dependencies"`
}

func processV1Lockfile(data []byte, requires []string) ([]model.DependencyItem, error) {
var e error
var lockfile v1Lockfile
e = json.Unmarshal(data, &lockfile)
if e != nil {
return nil, fmt.Errorf("parsing v1 lockfile: bad format, %w", e)
}
requires = lo.Uniq(requires)
var r []model.DependencyItem
for _, depName := range requires {
if dep, ok := lockfile.Dependencies[depName]; ok {
if rr := v1ConvDepRecursive(depName, dep, []map[string]v1Dep{lockfile.Dependencies}, make(map[string]struct{})); rr != nil {
r = append(r, *rr)
}
}
}
return r, nil
}

func v1ConvDepRecursive(name string, dep v1Dep, pp []map[string]v1Dep, visited map[string]struct{}) *model.DependencyItem {
if _, ok := visited[name]; ok {
return nil
}
visited[name] = struct{}{}
defer func() { delete(visited, name) }()
r := model.DependencyItem{
Component: model.Component{
CompName: name,
CompVersion: dep.Version,
EcoRepo: EcoRepo,
},
}
if dep.Dev {
r.IsOnline.SetOnline(false)
}
var queryMaps = make([]map[string]v1Dep, len(pp), len(pp)+1)
copy(queryMaps, pp)
if dep.Dependencies != nil {
queryMaps = append(queryMaps, dep.Dependencies)
}
o:
for depName := range dep.Requires {
queryMapOff := len(queryMaps) - 1
for queryMapOff >= 0 {
if child, ok := queryMaps[queryMapOff][depName]; ok {
rr := v1ConvDepRecursive(depName, child, queryMaps[:queryMapOff+1], visited)
if rr != nil {
r.Dependencies = append(r.Dependencies, *rr)
}
continue o
}
queryMapOff--
}
}
return &r
}
2 changes: 1 addition & 1 deletion module/npm/lockfile_v3.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type v3ParsedLockfile struct {
Deps []model.DependencyItem
}

func parseLockfileV3(data []byte) (r *v3ParsedLockfile, e error) {
func processLockfileV3(data []byte) (r *v3ParsedLockfile, e error) {
var lockfile v3Lockfile
if e := json.Unmarshal(data, &lockfile); e != nil {
return nil, fmt.Errorf("parse lockfile failed: %w", e)
Expand Down
17 changes: 17 additions & 0 deletions module/npm/lockfile_version.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package npm

import (
"encoding/json"
"fmt"
)

func parseLockfileVersion(data []byte) (int, error) {
type unknownVersionLockfile struct {
LockfileVersion int `json:"lockfileVersion"`
}
var u unknownVersionLockfile
if e := json.Unmarshal(data, &u); e != nil {
return 0, fmt.Errorf("parsing lockfile version: %w", e)
}
return u.LockfileVersion, nil
}
132 changes: 26 additions & 106 deletions module/npm/npm.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,12 @@ package npm

import (
"context"
"encoding/json"
"fmt"
"github.com/murphysecurity/murphysec/infra/logctx"
"github.com/murphysecurity/murphysec/model"
"github.com/murphysecurity/murphysec/utils"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/samber/lo"
"os"
"path/filepath"
"strings"
)

type Inspector struct{}
Expand Down Expand Up @@ -41,25 +37,34 @@ func (i *Inspector) InspectProject(ctx context.Context) error {

func ScanNpmProject(ctx context.Context) ([]model.Module, error) {
dir := model.UseInspectionTask(ctx).Dir()
pkgFile := filepath.Join(dir, "package-lock.json")
packagePath := filepath.Join(dir, "package.json")
module := model.Module{
PackageManager: "npm",
ModuleName: "",
ModuleVersion: "",
ModulePath: pkgFile,
ModulePath: packagePath,
}
logger := logctx.Use(ctx)
logger.Debug("Read package-lock.json", zap.String("path", pkgFile))
data, e := os.ReadFile(pkgFile)

data, e := os.ReadFile(packagePath)
if e != nil {
return nil, errors.WithMessage(e, "Errors when reading package-lock.json")
return nil, fmt.Errorf("reading package file: %w", e)
}
packageFile, e := parsePkgFile(data)
if e != nil {
return nil, e
}

lockfilePath := filepath.Join(dir, "package-lock.json")
data, e = os.ReadFile(lockfilePath)
if e != nil {
return nil, fmt.Errorf("reading package-lock file: %w", e)
}
lockfileVer, e := parseLockfileVersion(data)
if e != nil {
return nil, e
}
if lockfileVer == 3 {
parsed, e := parseLockfileV3(data)
parsed, e := processLockfileV3(data)
if e != nil {
return nil, fmt.Errorf("v3lockfile: %w", e)
}
Expand All @@ -68,106 +73,21 @@ func ScanNpmProject(ctx context.Context) ([]model.Module, error) {
module.Dependencies = parsed.Deps
return []model.Module{module}, nil
}
var lockfile NpmPkgLock
if e := json.Unmarshal(data, &lockfile); e != nil {
return nil, e
}
if lockfile.LockfileVersion > 2 || lockfile.LockfileVersion < 1 {
return nil, errors.New(fmt.Sprintf("unsupported lockfileVersion: %d", lockfile.LockfileVersion))
}
module.ModuleName = lockfile.Name
module.ModuleVersion = lockfile.Version
for s := range lockfile.Dependencies {
if strings.HasPrefix(s, "node_modules/") {
delete(lockfile.Dependencies, s)
}
}
var rootComp []string
{
// kahn
indegree := map[string]int{}
for s := range lockfile.Dependencies {
indegree[s] = 0
}
for _, it := range lockfile.Dependencies {
for d := range it.Requires {
indegree[d] = indegree[d] + 1
}
}

for k, i := range indegree {
if i == 0 {
rootComp = append(rootComp, k)
}
}
}
if len(rootComp) == 0 {
logger.Warn("Not found root component")
}

m := map[string]int{}
for _, it := range rootComp {
if d := _convDep(it, lockfile, m, 0); d != nil {
module.Dependencies = append(module.Dependencies, *d)
}
module.ModuleName = packageFile.Name
module.ModuleVersion = packageFile.Version
var requires = lo.Keys(packageFile.Dependencies)
requires = append(requires, lo.Keys(packageFile.DevDependencies)...)
requires = lo.Uniq(requires)
deps, e := processV1Lockfile(data, requires)
if e != nil {
return nil, e
}
module.Dependencies = utils.NoNilSlice(deps)
return []model.Module{module}, nil
}

func _convDep(root string, m NpmPkgLock, visited map[string]int, deep int) *model.DependencyItem {
if deep > 5 {
return nil
}
if _, ok := visited[root]; ok {
return nil
}
visited[root] = deep
defer delete(visited, root)
d, ok := m.Dependencies[root]
if !ok {
return nil
}
r := model.DependencyItem{
Component: model.Component{
CompName: root,
CompVersion: d.Version,
EcoRepo: EcoRepo,
},
IsDirectDependency: deep == 0,
}
for depName := range d.Requires {
cd := _convDep(depName, m, visited, deep+1)
if cd == nil {
continue
}
r.Dependencies = append(r.Dependencies, *cd)
}
return &r
}

//goland:noinspection GoNameStartsWithPackageName
type NpmPkgLock struct {
Name string `json:"name"`
Version string `json:"version"`
LockfileVersion int `json:"lockfileVersion"`
Dependencies map[string]struct {
Version string `json:"version"`
Requires map[string]interface{} `json:"requires"`
} `json:"dependencies"`
}

var EcoRepo = model.EcoRepo{
Ecosystem: "npm",
Repository: "",
}

func parseLockfileVersion(data []byte) (int, error) {
type unknownVersionLockfile struct {
LockfileVersion int `json:"lockfileVersion"`
}
var u unknownVersionLockfile
if e := json.Unmarshal(data, &u); e != nil {
return 0, fmt.Errorf("parse lockfile version failed: %w", e)
}
return u.LockfileVersion, nil
}
23 changes: 23 additions & 0 deletions module/npm/package_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package npm

import (
"encoding/json"
"fmt"
)

type pkgFile struct {
Name string `json:"name"`
Version string `json:"version"`
Dependencies map[string]string `json:"dependencies"`
DevDependencies map[string]string `json:"devDependencies"`
}

func parsePkgFile(data []byte) (*pkgFile, error) {
var e error
var r pkgFile
e = json.Unmarshal(data, &r)
if e != nil {
return nil, fmt.Errorf("parsing package file: bad format, %w", e)
}
return &r, nil
}

0 comments on commit 7c1ea83

Please sign in to comment.