diff --git a/providers/os/resources/npm.go b/providers/os/resources/npm.go new file mode 100644 index 0000000000..6d5bb63e12 --- /dev/null +++ b/providers/os/resources/npm.go @@ -0,0 +1,450 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package resources + +import ( + "cmp" + "errors" + "fmt" + "path/filepath" + "slices" + "strings" + "sync" + + "github.com/rs/zerolog/log" + "github.com/spf13/afero" + "go.mondoo.com/cnquery/v10/llx" + "go.mondoo.com/cnquery/v10/providers-sdk/v1/plugin" + "go.mondoo.com/cnquery/v10/providers/os/connection/shared" + "go.mondoo.com/cnquery/v10/providers/os/resources/npm" + "go.mondoo.com/cnquery/v10/types" +) + +func initNpmPackages(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) { + if x, ok := args["path"]; ok { + _, ok := x.Value.(string) + if !ok { + return nil, nil, errors.New("Wrong type for 'path' in npm initialization, it must be a string") + } + } else { + // empty path means search through default locations + args["path"] = llx.StringData("") + } + + return args, nil, nil +} + +func (r *mqlNpmPackages) id() (string, error) { + path := r.Path.Data + if path == "" { + return "npm.packages", nil + } + + return "npm.packages/" + path, nil +} + +func getFileContent(runtime *plugin.Runtime, path string) (*mqlFile, error) { + f, err := CreateResource(runtime, "file", map[string]*llx.RawData{ + "path": llx.StringData(path), + }) + if err != nil { + return nil, err + } + file := f.(*mqlFile) + return file, nil +} + +var ( + // we need to add extra testing for windows paths + //windowsDefaultNpmPaths = []string{ + // "C:\\Users\\%\\AppData\\Roaming\\npm", + //} + linuxDefaultNpmPaths = []string{ + "/usr/local/lib", + "/opt/homebrew/lib", + "/usr/lib", + "/home/%/.npm-global/lib", + "/Users/%/.npm-global/lib", + } +) + +func (r *mqlNpmPackages) gatherPackagesFromSystemDefaults(conn shared.Connection) ([]*npm.Package, []*npm.Package, []string, error) { + var directPackageList []*npm.Package + var transitivePackageList []*npm.Package + evidenceFiles := []string{} + log.Debug().Msg("searching for npm packages in default locations") + afs := &afero.Afero{Fs: conn.FileSystem()} + // we search through default system locations + for _, pattern := range linuxDefaultNpmPaths { + log.Debug().Str("path", pattern).Msg("searching for npm packages") + m, err := afero.Glob(conn.FileSystem(), pattern) + if err != nil { + log.Debug().Err(err).Str("path", pattern).Msg("could not search for npm packages") + // nothing to do, we just ignore it + } + for _, walkPath := range m { + // we walk through the directories and check if there is a node_modules directory + log.Debug().Str("path", walkPath).Msg("found npm package") + nodeModulesPath := filepath.Join(walkPath, "node_modules") + var files, err = afs.ReadDir(nodeModulesPath) + if err != nil { + continue + } + for i := range files { + f := files[i] + p := f.Name() + + if !f.IsDir() { + continue + } + + log.Debug().Str("path", p).Msg("checking for package-lock.json or package.json file") + + // check if there is a package-lock.json or package.json file + packageLockPath := filepath.Join(nodeModulesPath, p, "/package-lock.json") + packageJsonPath := filepath.Join(nodeModulesPath, p, "/package.json") + + packageLockExists, _ := afs.Exists(packageLockPath) + packageJsonExists, _ := afs.Exists(packageJsonPath) + + // add files to evidence + if packageLockExists { + evidenceFiles = append(evidenceFiles, packageLockPath) + } + if packageJsonExists { + evidenceFiles = append(evidenceFiles, packageJsonPath) + } + + // parse npm files + if packageLockExists { + log.Debug().Str("path", packageLockPath).Msg("found package-lock.json file") + f, err := getFileContent(r.MqlRuntime, packageLockPath) + if err != nil { + continue + } + content := f.GetContent() + if content.Error != nil { + continue + } + + p := &npm.PackageLockParser{} + info, err := p.Parse(strings.NewReader(content.Data), packageLockPath) + if err != nil { + log.Error().Err(err).Str("path", packageLockPath).Msg("could not parse package-lock.json file") + } + root := info.Root() + if root != nil { + directPackageList = append(directPackageList, root) + } + transitive := info.Transitive() + if transitive != nil { + transitivePackageList = append(transitivePackageList, transitive...) + } + + } else if packageJsonExists { + log.Debug().Str("path", packageJsonPath).Msg("found package.json file") + f, err := getFileContent(r.MqlRuntime, packageJsonPath) + if err != nil { + continue + } + content := f.GetContent() + if content.Error != nil { + continue + } + + p := &npm.PackageJsonParser{} + info, err := p.Parse(strings.NewReader(content.Data), packageJsonPath) + if err != nil { + log.Error().Err(err).Str("path", packageJsonPath).Msg("could not parse package.json file") + } + root := info.Root() + if root != nil { + directPackageList = append(directPackageList, root) + } + transitive := info.Transitive() + if transitive != nil { + transitivePackageList = append(transitivePackageList, transitive...) + } + } + } + } + } + return directPackageList, transitivePackageList, evidenceFiles, nil +} + +func (r *mqlNpmPackages) gatherPackagesFromLocation(conn shared.Connection, path string) (*npm.Package, []*npm.Package, []*npm.Package, []string, error) { + evidenceFiles := []string{} + + // specific path was provided + afs := &afero.Afero{Fs: conn.FileSystem()} + isDir, err := afs.IsDir(path) + if err != nil { + return nil, nil, nil, nil, err + } + + loadPackageLock := false + packageLockPath := "" + loadPackageJson := false + packageJsonPath := "" + + if isDir { + // check if there is a package-lock.json or package.json file + packageLockPath = filepath.Join(path, "/package-lock.json") + packageJsonPath = filepath.Join(path, "/package.json") + } else { + loadPackageJson = strings.HasSuffix(path, "package-lock.json") + if loadPackageJson { + packageLockPath = path + } + loadPackageLock = strings.HasSuffix(path, "package.json") + if loadPackageLock { + packageJsonPath = path + } + + if !loadPackageJson && !loadPackageLock { + return nil, nil, nil, nil, fmt.Errorf("path %s is not a package.json or package-lock.json file", path) + } + } + + loadPackageLock, _ = afs.Exists(packageLockPath) + loadPackageJson, _ = afs.Exists(packageJsonPath) + + if !loadPackageLock && !loadPackageJson { + return nil, nil, nil, nil, fmt.Errorf("path %s does not contain a package-lock.json or package.json file", path) + } + + // add source files as evidence to files list + if loadPackageLock { + evidenceFiles = append(evidenceFiles, packageLockPath) + } + if loadPackageJson { + evidenceFiles = append(evidenceFiles, packageJsonPath) + } + + // parse npm files + var info npm.NpmPackageInfo + if loadPackageLock { + // if there is a package-lock.json file, we use it + f, err := getFileContent(r.MqlRuntime, packageLockPath) + if err != nil { + return nil, nil, nil, nil, err + } + content := f.GetContent() + if content.Error != nil { + return nil, nil, nil, nil, content.Error + } + + p := &npm.PackageLockParser{} + info, err = p.Parse(strings.NewReader(content.Data), packageLockPath) + if err != nil { + return nil, nil, nil, nil, err + } + } else if loadPackageJson { + // if there is a package.json file, we use it + f, err := getFileContent(r.MqlRuntime, packageJsonPath) + if err != nil { + return nil, nil, nil, nil, err + } + content := f.GetContent() + if content.Error != nil { + return nil, nil, nil, nil, content.Error + } + + p := &npm.PackageJsonParser{} + info, err = p.Parse(strings.NewReader(content.Data), packageJsonPath) + if err != nil { + return nil, nil, nil, nil, err + } + } else { + return nil, nil, nil, nil, errors.New("could not parse package-lock.json or package.json file") + } + + return info.Root(), info.Direct(), info.Transitive(), evidenceFiles, nil +} + +type mqlNpmPackagesInternal struct { + mutex sync.Mutex +} + +func (r *mqlNpmPackages) gatherData() error { + // ensure we only gather data once, happens when multiple fields are called by MQL + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.Path.Error != nil { + return r.Path.Error + } + path := r.Path.Data + + // we check if the path is a directory or a file + // if it is a directory, we check if there is a package-lock.json or package.json file + conn := r.MqlRuntime.Connection.(shared.Connection) + + var root *npm.Package + var directDependencies []*npm.Package + var transitiveDependencies []*npm.Package + var filePaths []string + var err error + if path == "" { + // no specific path was provided, we search through default locations + // here we are not going to have a root package, only direct and transitive dependencies + directDependencies, transitiveDependencies, filePaths, err = r.gatherPackagesFromSystemDefaults(conn) + } else { + // specific path was provided and most likely it is a package-lock.json or package.json file or a directory + // that contains one of those files. We will have a root package direct and transitive dependencies + root, directDependencies, transitiveDependencies, filePaths, err = r.gatherPackagesFromLocation(conn, path) + } + + if err != nil { + return err + } + + // sort packages by name + sortFn := func(a, b *npm.Package) int { + if n := cmp.Compare(a.Name, b.Name); n != 0 { + return n + } + // if names are equal, order by version + return cmp.Compare(a.Version, b.Version) + } + slices.SortFunc(directDependencies, sortFn) + slices.SortFunc(transitiveDependencies, sortFn) + + if root != nil { + mqlPkg, err := newNpmPackages(r.MqlRuntime, root) + if err != nil { + return err + } + r.Root = plugin.TValue[*mqlNpmPackage]{Data: mqlPkg, State: plugin.StateIsSet} + } else { + r.Root = plugin.TValue[*mqlNpmPackage]{State: plugin.StateIsSet | plugin.StateIsNull} + } + + // create a resource for each package + transitiveResources := []interface{}{} + for i := range transitiveDependencies { + newNpmPackages, err := newNpmPackages(r.MqlRuntime, transitiveDependencies[i]) + if err != nil { + return err + } + transitiveResources = append(transitiveResources, newNpmPackages) + } + r.List = plugin.TValue[[]interface{}]{Data: transitiveResources, State: plugin.StateIsSet} + + directResources := []interface{}{} + for i := range directDependencies { + newNpmPackages, err := newNpmPackages(r.MqlRuntime, directDependencies[i]) + if err != nil { + return err + } + directResources = append(directResources, newNpmPackages) + } + r.DirectDependencies = plugin.TValue[[]interface{}]{Data: directResources, State: plugin.StateIsSet} + + // create files for each path + mqlFiles := []interface{}{} + for i := range filePaths { + path := filePaths[i] + lf, err := CreateResource(r.MqlRuntime, "pkgFileInfo", map[string]*llx.RawData{ + "path": llx.StringData(path), + }) + if err != nil { + return err + } + mqlFiles = append(mqlFiles, lf) + } + r.Files = plugin.TValue[[]interface{}]{Data: mqlFiles, State: plugin.StateIsSet} + + return nil +} + +func newNpmPackages(runtime *plugin.Runtime, pkg *npm.Package) (*mqlNpmPackage, error) { + cpes := []interface{}{} + for i := range pkg.Cpes { + cpe, err := runtime.CreateSharedResource("cpe", map[string]*llx.RawData{ + "uri": llx.StringData(pkg.Cpes[i]), + }) + if err != nil { + return nil, err + } + cpes = append(cpes, cpe) + } + + // create files for each path + mqlFiles := []interface{}{} + for i := range pkg.EvidenceLocations { + path := pkg.EvidenceLocations[i] + lf, err := CreateResource(runtime, "pkgFileInfo", map[string]*llx.RawData{ + "path": llx.StringData(path), + }) + if err != nil { + return nil, err + } + mqlFiles = append(mqlFiles, lf) + } + + mqlPkg, err := CreateResource(runtime, "npm.package", map[string]*llx.RawData{ + "id": llx.StringData(pkg.Name), + "name": llx.StringData(pkg.Name), + "version": llx.StringData(pkg.Version), + "purl": llx.StringData(pkg.Purl), + "cpes": llx.ArrayData(cpes, types.Resource("cpe")), + "files": llx.ArrayData(mqlFiles, types.Resource("pkgFileInfo")), + }) + if err != nil { + return nil, err + } + return mqlPkg.(*mqlNpmPackage), nil +} + +func (r *mqlNpmPackages) root() (*mqlNpmPackage, error) { + return nil, r.gatherData() +} + +func (r *mqlNpmPackages) directDependencies() ([]interface{}, error) { + return nil, r.gatherData() +} + +func (r *mqlNpmPackages) list() ([]interface{}, error) { + return nil, r.gatherData() +} + +func (r *mqlNpmPackages) files() ([]interface{}, error) { + return nil, r.gatherData() +} + +func (k *mqlNpmPackage) id() (string, error) { + return k.Id.Data, nil +} + +func (r *mqlNpmPackage) name() (string, error) { + return "", r.populateData() +} + +func (r *mqlNpmPackage) version() (string, error) { + return "", r.populateData() +} + +func (r *mqlNpmPackage) purl() (string, error) { + return "", r.populateData() +} + +func (r *mqlNpmPackage) cpes() ([]interface{}, error) { + return nil, r.populateData() +} + +func (r *mqlNpmPackage) files() ([]interface{}, error) { + return nil, errors.New("not implemented") +} + +func (r *mqlNpmPackage) populateData() error { + // future iterations will read an npm package.json file and populate the data + // all data is already available in the package object + r.Name = plugin.TValue[string]{State: plugin.StateIsSet | plugin.StateIsNull} + r.Version = plugin.TValue[string]{State: plugin.StateIsSet | plugin.StateIsNull} + r.Purl = plugin.TValue[string]{State: plugin.StateIsSet | plugin.StateIsNull} + r.Cpes = plugin.TValue[[]interface{}]{State: plugin.StateIsSet | plugin.StateIsNull} + r.Files = plugin.TValue[[]interface{}]{State: plugin.StateIsSet | plugin.StateIsNull} + return nil +} diff --git a/providers/os/resources/npm/npm.go b/providers/os/resources/npm/npm.go index 249d924baa..0974f5d749 100644 --- a/providers/os/resources/npm/npm.go +++ b/providers/os/resources/npm/npm.go @@ -13,17 +13,24 @@ import ( ) type Parser interface { - Parse(r io.Reader) (*Package, []*Package, error) + Parse(r io.Reader, filename string) (NpmPackageInfo, error) +} + +type NpmPackageInfo interface { + Root() *Package + Direct() []*Package + Transitive() []*Package } type Package struct { - Name string - File string - License string - Description string - Version string - Purl string - Cpes []string + Name string + File string + License string + Description string + Version string + Purl string + Cpes []string + EvidenceLocations []string } // NewPackageUrl creates a npm package url for a given package name and version @@ -43,14 +50,14 @@ func NewPackageUrl(name string, version string) string { packageurl.TypeNPM, namespace, name, - version, + cleanVersion(version), nil, "").String() } func NewCpes(name string, version string) []string { cpes := []string{} - cpeEntry, err := cpe.NewPackage2Cpe(name, name, version, "", "") + cpeEntry, err := cpe.NewPackage2Cpe(name, name, cleanVersion(version), "", "") // we only add the cpe if it could be created // if the cpe could not be created, we log the error and continue to ensure the package is still added to the list if err != nil { @@ -60,3 +67,13 @@ func NewCpes(name string, version string) []string { } return cpes } + +func cleanVersion(version string) string { + v := strings.ReplaceAll(version, "^", "") + v = strings.ReplaceAll(v, "~", "") + v = strings.ReplaceAll(v, ">", "") + v = strings.ReplaceAll(v, "<", "") + v = strings.ReplaceAll(v, "=", "") + v = strings.ReplaceAll(v, " ", "") + return v +} diff --git a/providers/os/resources/npm/package_json.go b/providers/os/resources/npm/package_json.go index 1edbe1d5f2..2b2a4d4994 100644 --- a/providers/os/resources/npm/package_json.go +++ b/providers/os/resources/npm/package_json.go @@ -32,6 +32,9 @@ type packageJson struct { Engines map[string]string `jsonn:"engines"` CPU []string `json:"cpu"` OS []string `json:"os"` + + // evidence is a list of file paths where the package.json was found + evidence []string `json:"-"` } // packageJsonPeople represents the author of the package @@ -160,36 +163,61 @@ func (a *packageJsonLicense) UnmarshalJSON(b []byte) error { type PackageJsonParser struct{} -func (p *PackageJsonParser) Parse(r io.Reader) (*Package, []*Package, error) { +func (p *PackageJsonParser) Parse(r io.Reader, filename string) (NpmPackageInfo, error) { data, err := io.ReadAll(r) if err != nil { - return nil, nil, err + return nil, err } var packageJson packageJson err = json.Unmarshal(data, &packageJson) if err != nil { - return nil, nil, err + return nil, err + } + + if filename != "" { + packageJson.evidence = append(packageJson.evidence, filename) } - // add own package + return &packageJson, nil +} + +func (p *packageJson) Root() *Package { + + // root package root := &Package{ - Name: packageJson.Name, - Version: packageJson.Version, - Purl: NewPackageUrl(packageJson.Name, packageJson.Version), - Cpes: NewCpes(packageJson.Name, packageJson.Version), + Name: p.Name, + Version: p.Version, + Purl: NewPackageUrl(p.Name, p.Version), + Cpes: NewCpes(p.Name, p.Version), + EvidenceLocations: p.evidence, + } + + return root +} + +func (p *packageJson) Direct() []*Package { + return nil +} + +func (p *packageJson) Transitive() []*Package { + // transitive dependencies, includes the root package + transitive := []*Package{} + + r := p.Root() + if r != nil { + transitive = append(transitive, r) } - // add all dependencies - entries := []*Package{} - for k, v := range packageJson.Dependencies { - entries = append(entries, &Package{ - Name: k, - Version: v, - Purl: NewPackageUrl(k, v), - Cpes: NewCpes(k, v), + for k, v := range p.Dependencies { + transitive = append(transitive, &Package{ + Name: k, + Version: v, + Purl: NewPackageUrl(k, v), + Cpes: NewCpes(k, v), + EvidenceLocations: p.evidence, }) } - return root, entries, nil + return transitive } diff --git a/providers/os/resources/npm/package_json_test.go b/providers/os/resources/npm/package_json_test.go index cb4ff1bbd7..0e6ee77113 100644 --- a/providers/os/resources/npm/package_json_test.go +++ b/providers/os/resources/npm/package_json_test.go @@ -274,31 +274,46 @@ func TestPackageJsonParser(t *testing.T) { require.NoError(t, err) defer f.Close() - root, pkgs, err := (&PackageJsonParser{}).Parse(f) + info, err := (&PackageJsonParser{}).Parse(f, "path/package.json") assert.Nil(t, err) - assert.Equal(t, 30, len(pkgs)) + root := info.Root() assert.Equal(t, &Package{ - Name: "express", - Version: "4.16.4", - Purl: "pkg:npm/express@4.16.4", - Cpes: []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"}, + Name: "express", + Version: "4.16.4", + Purl: "pkg:npm/express@4.16.4", + Cpes: []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/package.json"}, }, root) - p := findPkg(pkgs, "path-to-regexp") + transitive := info.Transitive() + assert.Equal(t, 31, len(transitive)) + + // ensure the package is in the transitive list + p := findPkg(transitive, "express") + assert.Equal(t, &Package{ + Name: "express", + Version: "4.16.4", + Purl: "pkg:npm/express@4.16.4", + Cpes: []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/package.json"}, + }, p) + + p = findPkg(transitive, "path-to-regexp") assert.Equal(t, &Package{ - Name: "path-to-regexp", - Version: "0.1.7", - Purl: "pkg:npm/path-to-regexp@0.1.7", - Cpes: []string{"cpe:2.3:a:path-to-regexp:path-to-regexp:0.1.7:*:*:*:*:*:*:*"}, + Name: "path-to-regexp", + Version: "0.1.7", + Purl: "pkg:npm/path-to-regexp@0.1.7", + Cpes: []string{"cpe:2.3:a:path-to-regexp:path-to-regexp:0.1.7:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/package.json"}, }, p) - p = findPkg(pkgs, "range-parser") + p = findPkg(transitive, "range-parser") assert.Equal(t, &Package{ - Name: "range-parser", - Version: "~1.2.0", - // TODO: we need to handle the range properly - Purl: "pkg:npm/range-parser@~1.2.0", - Cpes: []string{"cpe:2.3:a:range-parser:range-parser:\\~1.2.0:*:*:*:*:*:*:*"}, + Name: "range-parser", + Version: "~1.2.0", + Purl: "pkg:npm/range-parser@1.2.0", + Cpes: []string{"cpe:2.3:a:range-parser:range-parser:1.2.0:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/package.json"}, }, p) } diff --git a/providers/os/resources/npm/package_lock.go b/providers/os/resources/npm/package_lock.go index bf84541b40..38bf5b228f 100644 --- a/providers/os/resources/npm/package_lock.go +++ b/providers/os/resources/npm/package_lock.go @@ -5,6 +5,7 @@ package npm import ( "io" + "strings" "encoding/json" ) @@ -26,6 +27,9 @@ type packageLock struct { // Dependencies contains legacy data for supporting versions of npm that use lockfileVersion: 1 or lower. // We can ignore that for lockfileVersion: 2+ Dependencies map[string]packageLockDependency `jsonn:"dependencies"` + + // evidence is a list of file paths where the package-lock was found + evidence []string `json:"-"` } type packageLockDependency struct { @@ -36,11 +40,12 @@ type packageLockDependency struct { } type packageLockPackage struct { - Name string `json:"name"` - Version string `json:"version"` - Resolved string `json:"resolved"` - Integrity string `json:"integrity"` - License packageLockLicense `json:"license"` + Name string `json:"name"` + Version string `json:"version"` + Resolved string `json:"resolved"` + Integrity string `json:"integrity"` + License packageLockLicense `json:"license"` + Dependencies map[string]string `json:"dependencies"` } type packageLockLicense []string @@ -67,48 +72,98 @@ func (l *packageLockLicense) UnmarshalJSON(data []byte) (err error) { // PackageLockParser is the parser for the package.lock file npm format. // see https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json -type PackageLockParser struct{} +type PackageLockParser struct { +} -func (p *PackageLockParser) Parse(r io.Reader) (*Package, []*Package, error) { +func (p *PackageLockParser) Parse(r io.Reader, filename string) (NpmPackageInfo, error) { var packageJsonLock packageLock err := json.NewDecoder(r).Decode(&packageJsonLock) if err != nil { - return nil, nil, err + return nil, err + } + + if filename != "" { + packageJsonLock.evidence = append(packageJsonLock.evidence, filename) } - // add own package + return &packageJsonLock, nil +} + +func (p *packageLock) Root() *Package { root := &Package{ - Name: packageJsonLock.Name, - Version: packageJsonLock.Version, - Purl: NewPackageUrl(packageJsonLock.Name, packageJsonLock.Version), - Cpes: NewCpes(packageJsonLock.Name, packageJsonLock.Version), + Name: p.Name, + Version: p.Version, + Purl: NewPackageUrl(p.Name, p.Version), + Cpes: NewCpes(p.Name, p.Version), + EvidenceLocations: p.evidence, + } + return root +} + +func (p *packageLock) Direct() []*Package { + // search for root package, read the packages field + + // at this point we only support lockfileVersion: 2 with direct dependencies + if p.Packages == nil { + return nil } - // add all dependencies - entries := []*Package{} - if packageJsonLock.Packages != nil { - for k, v := range packageJsonLock.Packages { + rootPkg, ok := p.Packages[""] + if !ok { + return nil + } + + filteredList := []*Package{} + for k := range rootPkg.Dependencies { + pkg, ok := p.Packages[k] + if !ok { + continue + } + + filteredList = append(filteredList, &Package{ + Name: packageLockPackageName(k), + Version: pkg.Version, + Purl: NewPackageUrl(k, pkg.Version), + Cpes: NewCpes(k, pkg.Version), + EvidenceLocations: p.evidence, + }) + } + + return filteredList +} + +func (p *packageLock) Transitive() []*Package { + transitive := []*Package{} + if p.Packages != nil { + for k, v := range p.Packages { name := k + // skip root package since we have that already if name == "" { name = v.Name } - entries = append(entries, &Package{ - Name: name, - Version: v.Version, - Purl: NewPackageUrl(name, v.Version), - Cpes: NewCpes(name, v.Version), + + transitive = append(transitive, &Package{ + Name: packageLockPackageName(name), + Version: v.Version, + Purl: NewPackageUrl(k, v.Version), + Cpes: NewCpes(k, v.Version), + EvidenceLocations: p.evidence, }) } - } else if packageJsonLock.Dependencies != nil { - for k, v := range packageJsonLock.Dependencies { - entries = append(entries, &Package{ - Name: k, - Version: v.Version, - Purl: NewPackageUrl(k, v.Version), - Cpes: NewCpes(k, v.Version), + } else if p.Dependencies != nil { + for k, v := range p.Dependencies { + transitive = append(transitive, &Package{ + Name: k, + Version: v.Version, + Purl: NewPackageUrl(k, v.Version), + Cpes: NewCpes(k, v.Version), + EvidenceLocations: p.evidence, }) } } + return transitive +} - return root, entries, nil +func packageLockPackageName(path string) string { + return strings.TrimPrefix(path, "node_modules/") } diff --git a/providers/os/resources/npm/package_lock_test.go b/providers/os/resources/npm/package_lock_test.go index f1ffa094a3..4c983cda3e 100644 --- a/providers/os/resources/npm/package_lock_test.go +++ b/providers/os/resources/npm/package_lock_test.go @@ -67,11 +67,18 @@ func TestPackageLock(t *testing.T) { License: packageLockLicense( []string{"Artistic-2.0"}, ), + Dependencies: map[string]string{ + "@npmcli/arborist": "^1.0.0", + "@npmcli/ci-detect": "^1.2.0", + }, }, "node_modules/@babel/code-frame": { Version: "7.10.4", Resolved: "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + Dependencies: map[string]string{ + "@babel/highlight": "^7.10.4", + }, }, }, Dependencies: map[string]packageLockDependency{ @@ -116,6 +123,9 @@ func TestPackageLock(t *testing.T) { License: packageLockLicense( []string{"Artistic-2.0"}, ), + Dependencies: map[string]string{ + "@isaacs/string-locale-compare": "^1.1.0", + }, }, "node_modules/@isaacs/string-locale-compare": { Version: "1.1.0", @@ -150,36 +160,72 @@ func TestPackageLock(t *testing.T) { } } -func TestPackageJsonLockParser(t *testing.T) { - f, err := os.Open("./testdata/package-lock/workbox-package-lock.json") +func TestPackageJsonLockWithPackages(t *testing.T) { + f, err := os.Open("./testdata/package-lock/lockfile-v2.json") require.NoError(t, err) + defer f.Close() + + info, err := (&PackageLockParser{}).Parse(f, "path/to/package-lock.json") + assert.Nil(t, err) + + root := info.Root() + assert.Equal(t, &Package{ + Name: "npm", + Version: "7.0.0", + Purl: "pkg:npm/npm@7.0.0", + Cpes: []string{"cpe:2.3:a:npm:npm:7.0.0:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/to/package-lock.json"}, + }, root) + + transitive := info.Transitive() + assert.Equal(t, 2, len(transitive)) + p := findPkg(transitive, "@babel/code-frame") + assert.Equal(t, &Package{ + Name: "@babel/code-frame", + Version: "7.10.4", + Purl: "pkg:npm/node-modules/%40babel@7.10.4", + Cpes: []string{"cpe:2.3:a:node_modules\\/\\@babel\\/code-frame:node_modules\\/\\@babel\\/code-frame:7.10.4:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/to/package-lock.json"}, + }, p) + +} + +func TestPackageJsonLockWithDependencies(t *testing.T) { + f, err := os.Open("./testdata/package-lock/workbox-package-lock.json") + require.NoError(t, err) defer f.Close() - root, pkgs, err := (&PackageLockParser{}).Parse(f) + info, err := (&PackageLockParser{}).Parse(f, "path/to/package-lock.json") assert.Nil(t, err) - assert.Equal(t, 1299, len(pkgs)) + root := info.Root() assert.Equal(t, &Package{ - Name: "workbox", - Version: "0.0.0", - Purl: "pkg:npm/workbox@0.0.0", - Cpes: []string{"cpe:2.3:a:workbox:workbox:0.0.0:*:*:*:*:*:*:*"}, + Name: "workbox", + Version: "0.0.0", + Purl: "pkg:npm/workbox@0.0.0", + Cpes: []string{"cpe:2.3:a:workbox:workbox:0.0.0:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/to/package-lock.json"}, }, root) - p := findPkg(pkgs, "@babel/generator") + transitive := info.Transitive() + assert.Equal(t, 1299, len(transitive)) + + p := findPkg(transitive, "@babel/generator") assert.Equal(t, &Package{ - Name: "@babel/generator", - Version: "7.0.0", - Purl: "pkg:npm/%40babel/generator@7.0.0", - Cpes: []string{"cpe:2.3:a:\\@babel\\/generator:\\@babel\\/generator:7.0.0:*:*:*:*:*:*:*"}, + Name: "@babel/generator", + Version: "7.0.0", + Purl: "pkg:npm/%40babel/generator@7.0.0", + Cpes: []string{"cpe:2.3:a:\\@babel\\/generator:\\@babel\\/generator:7.0.0:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/to/package-lock.json"}, }, p) - p = findPkg(pkgs, "@lerna/changed") + p = findPkg(transitive, "@lerna/changed") assert.Equal(t, &Package{ - Name: "@lerna/changed", - Version: "3.3.2", - Purl: "pkg:npm/%40lerna/changed@3.3.2", - Cpes: []string{"cpe:2.3:a:\\@lerna\\/changed:\\@lerna\\/changed:3.3.2:*:*:*:*:*:*:*"}, + Name: "@lerna/changed", + Version: "3.3.2", + Purl: "pkg:npm/%40lerna/changed@3.3.2", + Cpes: []string{"cpe:2.3:a:\\@lerna\\/changed:\\@lerna\\/changed:3.3.2:*:*:*:*:*:*:*"}, + EvidenceLocations: []string{"path/to/package-lock.json"}, }, p) } diff --git a/providers/os/resources/npm/yarn_lock.go b/providers/os/resources/npm/yarn_lock.go index 979f30b4b3..97d4881a18 100644 --- a/providers/os/resources/npm/yarn_lock.go +++ b/providers/os/resources/npm/yarn_lock.go @@ -19,7 +19,9 @@ var ( _ Parser = (*YarnLockParser)(nil) ) -type YarnLockEntry struct { +type yarnLock map[string]yarnLockEntry + +type yarnLockEntry struct { Version string Resolved string Dependencies map[string]string @@ -27,7 +29,7 @@ type YarnLockEntry struct { type YarnLockParser struct{} -func (p *YarnLockParser) Parse(r io.Reader) (*Package, []*Package, error) { +func (p *YarnLockParser) Parse(r io.Reader, filename string) (NpmPackageInfo, error) { var b bytes.Buffer // iterate and convert the format to yaml on the fly @@ -43,26 +45,39 @@ func (p *YarnLockParser) Parse(r io.Reader) (*Package, []*Package, error) { b.Write([]byte("\n")) } if err := scanner.Err(); err != nil { - return nil, nil, err + return nil, err } - var yarnLock map[string]YarnLockEntry + var yarnLock yarnLock err := yaml.Unmarshal(b.Bytes(), &yarnLock) if err != nil { - return nil, nil, err + return nil, err } - entries := []*Package{} + return &yarnLock, nil +} + +func (p *yarnLock) Root() *Package { + // we don't have a root package in yarn.lock + return nil +} + +func (p *yarnLock) Direct() []*Package { + return nil +} + +func (p *yarnLock) Transitive() []*Package { + transitive := []*Package{} // add all dependencies - for k, v := range yarnLock { + for k, v := range *p { name, _, err := parseYarnPackageName(k) if err != nil { log.Error().Str("name", name).Msg("cannot parse yarn package name") continue } - entries = append(entries, &Package{ + transitive = append(transitive, &Package{ Name: name, Version: v.Version, Purl: NewPackageUrl(name, v.Version), @@ -70,7 +85,7 @@ func (p *YarnLockParser) Parse(r io.Reader) (*Package, []*Package, error) { }) } - return nil, entries, nil + return transitive } func parseYarnPackageName(name string) (string, string, error) { diff --git a/providers/os/resources/npm/yarn_lock_test.go b/providers/os/resources/npm/yarn_lock_test.go index 65a2af0323..ec4e66eb9c 100644 --- a/providers/os/resources/npm/yarn_lock_test.go +++ b/providers/os/resources/npm/yarn_lock_test.go @@ -16,11 +16,13 @@ func TestYarnParser(t *testing.T) { require.NoError(t, err) defer f.Close() - _, pkgs, err := (&YarnLockParser{}).Parse(f) + info, err := (&YarnLockParser{}).Parse(f, "/path/to/yarn.lock") assert.Nil(t, err) - assert.Equal(t, 99, len(pkgs)) - p := findPkg(pkgs, "has") + transitive := info.Transitive() + assert.Equal(t, 99, len(transitive)) + + p := findPkg(transitive, "has") assert.Equal(t, &Package{ Name: "has", Version: "1.0.3", @@ -28,7 +30,7 @@ func TestYarnParser(t *testing.T) { Cpes: []string{"cpe:2.3:a:has:has:1.0.3:*:*:*:*:*:*:*"}, }, p) - p = findPkg(pkgs, "iconv-lite") + p = findPkg(transitive, "iconv-lite") assert.Equal(t, &Package{ Name: "iconv-lite", Version: "0.4.24", diff --git a/providers/os/resources/os.lr b/providers/os/resources/os.lr index d7f0a04a5c..847e84ed9d 100644 --- a/providers/os/resources/os.lr +++ b/providers/os/resources/os.lr @@ -1170,6 +1170,40 @@ python.package @defaults("name version") { dependencies() []python.package } +// npm packages +npm.packages { + []npm.package + + init(path? string) + + // optional path to search for packages + path string + + // Root Package (may not exist) + root() npm.package + + // List of direct dependencies + directDependencies() []npm.package + + // Files used to determine the packages + files() []pkgFileInfo +} + +npm.package @defaults("name version") { + // ID is the npm.package unique identifier + id string + // Name of the package + name() string + // Version of the package + version() string + // Package URL + purl() string + // Common Platform Enumeration (CPE) for the package + cpes() []core.cpe + // Package files + files() []pkgFileInfo +} + // macOS specific resources macos { // macOS user defaults diff --git a/providers/os/resources/os.lr.go b/providers/os/resources/os.lr.go index d2dcc64f7e..4fa6a15853 100644 --- a/providers/os/resources/os.lr.go +++ b/providers/os/resources/os.lr.go @@ -366,6 +366,14 @@ func init() { Init: initPythonPackage, Create: createPythonPackage, }, + "npm.packages": { + Init: initNpmPackages, + Create: createNpmPackages, + }, + "npm.package": { + // to override args, implement: initNpmPackage(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) + Create: createNpmPackage, + }, "macos": { // to override args, implement: initMacos(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) Create: createMacos, @@ -1691,6 +1699,39 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "python.package.dependencies": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlPythonPackage).GetDependencies()).ToDataRes(types.Array(types.Resource("python.package"))) }, + "npm.packages.path": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackages).GetPath()).ToDataRes(types.String) + }, + "npm.packages.root": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackages).GetRoot()).ToDataRes(types.Resource("npm.package")) + }, + "npm.packages.directDependencies": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackages).GetDirectDependencies()).ToDataRes(types.Array(types.Resource("npm.package"))) + }, + "npm.packages.files": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackages).GetFiles()).ToDataRes(types.Array(types.Resource("pkgFileInfo"))) + }, + "npm.packages.list": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackages).GetList()).ToDataRes(types.Array(types.Resource("npm.package"))) + }, + "npm.package.id": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetId()).ToDataRes(types.String) + }, + "npm.package.name": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetName()).ToDataRes(types.String) + }, + "npm.package.version": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetVersion()).ToDataRes(types.String) + }, + "npm.package.purl": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetPurl()).ToDataRes(types.String) + }, + "npm.package.cpes": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetCpes()).ToDataRes(types.Array(types.Resource("cpe"))) + }, + "npm.package.files": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlNpmPackage).GetFiles()).ToDataRes(types.Array(types.Resource("pkgFileInfo"))) + }, "macos.userPreferences": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlMacos).GetUserPreferences()).ToDataRes(types.Map(types.String, types.Dict)) }, @@ -3977,6 +4018,58 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlPythonPackage).Dependencies, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) return }, + "npm.packages.__id": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).__id, ok = v.Value.(string) + return + }, + "npm.packages.path": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).Path, ok = plugin.RawToTValue[string](v.Value, v.Error) + return + }, + "npm.packages.root": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).Root, ok = plugin.RawToTValue[*mqlNpmPackage](v.Value, v.Error) + return + }, + "npm.packages.directDependencies": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).DirectDependencies, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, + "npm.packages.files": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).Files, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, + "npm.packages.list": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackages).List, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, + "npm.package.__id": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).__id, ok = v.Value.(string) + return + }, + "npm.package.id": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Id, ok = plugin.RawToTValue[string](v.Value, v.Error) + return + }, + "npm.package.name": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Name, ok = plugin.RawToTValue[string](v.Value, v.Error) + return + }, + "npm.package.version": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Version, ok = plugin.RawToTValue[string](v.Value, v.Error) + return + }, + "npm.package.purl": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Purl, ok = plugin.RawToTValue[string](v.Value, v.Error) + return + }, + "npm.package.cpes": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Cpes, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, + "npm.package.files": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlNpmPackage).Files, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, "macos.__id": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlMacos).__id, ok = v.Value.(string) return @@ -11480,6 +11573,227 @@ func (c *mqlPythonPackage) GetDependencies() *plugin.TValue[[]interface{}] { }) } +// mqlNpmPackages for the npm.packages resource +type mqlNpmPackages struct { + MqlRuntime *plugin.Runtime + __id string + mqlNpmPackagesInternal + Path plugin.TValue[string] + Root plugin.TValue[*mqlNpmPackage] + DirectDependencies plugin.TValue[[]interface{}] + Files plugin.TValue[[]interface{}] + List plugin.TValue[[]interface{}] +} + +// createNpmPackages creates a new instance of this resource +func createNpmPackages(runtime *plugin.Runtime, args map[string]*llx.RawData) (plugin.Resource, error) { + res := &mqlNpmPackages{ + MqlRuntime: runtime, + } + + err := SetAllData(res, args) + if err != nil { + return res, err + } + + if res.__id == "" { + res.__id, err = res.id() + if err != nil { + return nil, err + } + } + + if runtime.HasRecording { + args, err = runtime.ResourceFromRecording("npm.packages", res.__id) + if err != nil || args == nil { + return res, err + } + return res, SetAllData(res, args) + } + + return res, nil +} + +func (c *mqlNpmPackages) MqlName() string { + return "npm.packages" +} + +func (c *mqlNpmPackages) MqlID() string { + return c.__id +} + +func (c *mqlNpmPackages) GetPath() *plugin.TValue[string] { + return &c.Path +} + +func (c *mqlNpmPackages) GetRoot() *plugin.TValue[*mqlNpmPackage] { + return plugin.GetOrCompute[*mqlNpmPackage](&c.Root, func() (*mqlNpmPackage, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.packages", c.__id, "root") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.(*mqlNpmPackage), nil + } + } + + return c.root() + }) +} + +func (c *mqlNpmPackages) GetDirectDependencies() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.DirectDependencies, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.packages", c.__id, "directDependencies") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.directDependencies() + }) +} + +func (c *mqlNpmPackages) GetFiles() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.Files, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.packages", c.__id, "files") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.files() + }) +} + +func (c *mqlNpmPackages) GetList() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.List, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.packages", c.__id, "list") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.list() + }) +} + +// mqlNpmPackage for the npm.package resource +type mqlNpmPackage struct { + MqlRuntime *plugin.Runtime + __id string + // optional: if you define mqlNpmPackageInternal it will be used here + Id plugin.TValue[string] + Name plugin.TValue[string] + Version plugin.TValue[string] + Purl plugin.TValue[string] + Cpes plugin.TValue[[]interface{}] + Files plugin.TValue[[]interface{}] +} + +// createNpmPackage creates a new instance of this resource +func createNpmPackage(runtime *plugin.Runtime, args map[string]*llx.RawData) (plugin.Resource, error) { + res := &mqlNpmPackage{ + MqlRuntime: runtime, + } + + err := SetAllData(res, args) + if err != nil { + return res, err + } + + if res.__id == "" { + res.__id, err = res.id() + if err != nil { + return nil, err + } + } + + if runtime.HasRecording { + args, err = runtime.ResourceFromRecording("npm.package", res.__id) + if err != nil || args == nil { + return res, err + } + return res, SetAllData(res, args) + } + + return res, nil +} + +func (c *mqlNpmPackage) MqlName() string { + return "npm.package" +} + +func (c *mqlNpmPackage) MqlID() string { + return c.__id +} + +func (c *mqlNpmPackage) GetId() *plugin.TValue[string] { + return &c.Id +} + +func (c *mqlNpmPackage) GetName() *plugin.TValue[string] { + return plugin.GetOrCompute[string](&c.Name, func() (string, error) { + return c.name() + }) +} + +func (c *mqlNpmPackage) GetVersion() *plugin.TValue[string] { + return plugin.GetOrCompute[string](&c.Version, func() (string, error) { + return c.version() + }) +} + +func (c *mqlNpmPackage) GetPurl() *plugin.TValue[string] { + return plugin.GetOrCompute[string](&c.Purl, func() (string, error) { + return c.purl() + }) +} + +func (c *mqlNpmPackage) GetCpes() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.Cpes, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.package", c.__id, "cpes") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.cpes() + }) +} + +func (c *mqlNpmPackage) GetFiles() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.Files, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("npm.package", c.__id, "files") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.files() + }) +} + // mqlMacos for the macos resource type mqlMacos struct { MqlRuntime *plugin.Runtime diff --git a/providers/os/resources/os.lr.manifest.yaml b/providers/os/resources/os.lr.manifest.yaml index 1a7f6dc58c..757dd6c27e 100644 --- a/providers/os/resources/os.lr.manifest.yaml +++ b/providers/os/resources/os.lr.manifest.yaml @@ -489,6 +489,26 @@ resources: options: {} path: {} min_mondoo_version: 5.15.0 + npm.package: + fields: + cpes: {} + dependencies: {} + files: {} + id: {} + name: {} + purl: {} + version: {} + min_mondoo_version: latest + npm.packages: + fields: + "": {} + directDependencies: {} + files: {} + list: {} + path: {} + root: {} + transitive: {} + min_mondoo_version: latest ntp.conf: fields: content: {}