From cbba7cfff96a2b89b07c99848f8c18214c932ec5 Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Thu, 15 Feb 2024 14:48:45 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20refactor=20npm=20parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- providers/os/resources/npm/npm.go | 14 + providers/os/resources/npm/package_json.go | 172 +++++++++++ .../os/resources/npm/package_json_test.go | 278 ++++++++++++++++++ providers/os/resources/npm/package_lock.go | 55 ++++ .../os/resources/npm/package_lock_test.go | 38 +++ providers/os/resources/npm/packagejson.go | 104 ------- .../os/resources/npm/packagejson_test.go | 65 ---- .../npm/testdata/package-json/author.json | 8 + .../testdata/package-json/author_shorten.json | 4 + .../npm/testdata/package-json/bugs.json | 6 + .../package-json/bundle_dependencies.json | 5 + .../testdata/package-json/contributors.json | 7 + .../package-json/contributors_shorten.json | 5 + .../testdata/package-json/cpu_exclude.json | 3 + .../testdata/package-json/cpu_include.json | 3 + .../testdata/package-json/dependencies.json | 16 + .../package-json/dev_dependencies.json | 12 + .../npm/testdata/package-json/engines.json | 5 + .../{ => package-json}/express-package.json | 0 .../npm/testdata/package-json/homepage.json | 3 + .../package-json/license_deprecated_01.json | 6 + .../package-json/license_deprecated_02.json | 12 + .../testdata/package-json/license_spdx.json | 3 + .../package-json/license_spdx_expression.json | 3 + .../npm/testdata/package-json/os_exclude.json | 3 + .../npm/testdata/package-json/os_include.json | 3 + .../package-json/peer_dependencies.json | 7 + .../package-json/peer_dependencies_meta.json | 13 + .../npm/testdata/package-json/private.json | 3 + .../npm/testdata/package-json/repository.json | 6 + .../package-json/repository_bitbucket.json | 3 + .../package-json/repository_directory.json | 7 + .../testdata/package-json/repository_gh.json | 3 + .../npm/testdata/package-json/workspaces.json | 4 + .../workbox-package-lock.json | 0 .../npm/testdata/{ => yarn-lock}/d3-yarn.lock | 0 .../resources/npm/{yarn.go => yarn_lock.go} | 8 +- .../npm/{yarn_test.go => yarn_lock_test.go} | 21 +- providers/os/resources/os.lr | 32 ++ 39 files changed, 757 insertions(+), 183 deletions(-) create mode 100644 providers/os/resources/npm/npm.go create mode 100644 providers/os/resources/npm/package_json.go create mode 100644 providers/os/resources/npm/package_json_test.go create mode 100644 providers/os/resources/npm/package_lock.go create mode 100644 providers/os/resources/npm/package_lock_test.go delete mode 100644 providers/os/resources/npm/packagejson.go delete mode 100644 providers/os/resources/npm/packagejson_test.go create mode 100644 providers/os/resources/npm/testdata/package-json/author.json create mode 100644 providers/os/resources/npm/testdata/package-json/author_shorten.json create mode 100644 providers/os/resources/npm/testdata/package-json/bugs.json create mode 100644 providers/os/resources/npm/testdata/package-json/bundle_dependencies.json create mode 100644 providers/os/resources/npm/testdata/package-json/contributors.json create mode 100644 providers/os/resources/npm/testdata/package-json/contributors_shorten.json create mode 100644 providers/os/resources/npm/testdata/package-json/cpu_exclude.json create mode 100644 providers/os/resources/npm/testdata/package-json/cpu_include.json create mode 100644 providers/os/resources/npm/testdata/package-json/dependencies.json create mode 100644 providers/os/resources/npm/testdata/package-json/dev_dependencies.json create mode 100644 providers/os/resources/npm/testdata/package-json/engines.json rename providers/os/resources/npm/testdata/{ => package-json}/express-package.json (100%) create mode 100644 providers/os/resources/npm/testdata/package-json/homepage.json create mode 100644 providers/os/resources/npm/testdata/package-json/license_deprecated_01.json create mode 100644 providers/os/resources/npm/testdata/package-json/license_deprecated_02.json create mode 100644 providers/os/resources/npm/testdata/package-json/license_spdx.json create mode 100644 providers/os/resources/npm/testdata/package-json/license_spdx_expression.json create mode 100644 providers/os/resources/npm/testdata/package-json/os_exclude.json create mode 100644 providers/os/resources/npm/testdata/package-json/os_include.json create mode 100644 providers/os/resources/npm/testdata/package-json/peer_dependencies.json create mode 100644 providers/os/resources/npm/testdata/package-json/peer_dependencies_meta.json create mode 100644 providers/os/resources/npm/testdata/package-json/private.json create mode 100644 providers/os/resources/npm/testdata/package-json/repository.json create mode 100644 providers/os/resources/npm/testdata/package-json/repository_bitbucket.json create mode 100644 providers/os/resources/npm/testdata/package-json/repository_directory.json create mode 100644 providers/os/resources/npm/testdata/package-json/repository_gh.json create mode 100644 providers/os/resources/npm/testdata/package-json/workspaces.json rename providers/os/resources/npm/testdata/{ => package-lock}/workbox-package-lock.json (100%) rename providers/os/resources/npm/testdata/{ => yarn-lock}/d3-yarn.lock (100%) rename providers/os/resources/npm/{yarn.go => yarn_lock.go} (88%) rename providers/os/resources/npm/{yarn_test.go => yarn_lock_test.go} (67%) diff --git a/providers/os/resources/npm/npm.go b/providers/os/resources/npm/npm.go new file mode 100644 index 0000000000..57d41e14af --- /dev/null +++ b/providers/os/resources/npm/npm.go @@ -0,0 +1,14 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package npm + +import ( + "io" + + "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" +) + +type Parser interface { + Parse(r io.Reader) ([]*mvd.Package, error) +} diff --git a/providers/os/resources/npm/package_json.go b/providers/os/resources/npm/package_json.go new file mode 100644 index 0000000000..d2d512fe75 --- /dev/null +++ b/providers/os/resources/npm/package_json.go @@ -0,0 +1,172 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package npm + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" +) + +// packageJson allows parsing the package json file +type packageJson struct { + Name string `json:"name"` + Description string `json:"description"` + Version string `json:"version"` + Private bool `json:"private"` + Homepage string `json:"homepage"` + License string `json:"license"` + Author *packageJsonPeople `json:"author"` + Contributors []packageJsonPeople `json:"contributors"` + Dependencies map[string]string `jsonn:"dependencies"` + DevDependencies map[string]string `jsonn:"devDependencies"` + Repository packageJsonRepository `json:"repository"` + Engines map[string]string `jsonn:"engines"` + CPU []string `json:"cpu"` + OS []string `json:"os"` +} + +// packageJsonPeople represents the author of the package +// https://docs.npmjs.com/cli/v10/configuring-npm/package-json#people-fields-author-contributors +type packageJsonPeople struct { + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` +} + +// Explanation: +// +// ^: Asserts the start of the string. +// ([^<]+): Captures one or more characters that are not <. This is for the author's name. +// \s+: Matches one or more whitespace characters. +// <([^>]+)>: Captures the email address within angle brackets. +// \s+: Matches one or more whitespace characters again. +// \(([^)]+)\): Captures the URL within parentheses. +// $: Asserts the end of the string. +var authorPatter = regexp.MustCompile(`^([^<]+)\s+<([^>]+)>\s+\(([^)]+)\)$`) + +// UnmarshalJSON implements the json.Unmarshaler interface +// package.json author can be a string or a structured object +func (a *packageJsonPeople) UnmarshalJSON(b []byte) error { + var authorStr string + type authorStruct packageJsonPeople + author := authorStruct{} + + // try to unmarshal as structured object + err := json.Unmarshal(b, &author) + if err == nil { + a.Name = author.Name + a.Email = author.Email + a.URL = author.URL + return nil + } + + // try to unmarshal as string + err = json.Unmarshal(b, &authorStr) + if err == nil { + matches := authorPatter.FindStringSubmatch(authorStr) + if len(matches) == 4 { + a.Name = matches[1] + a.Email = matches[2] + a.URL = matches[3] + return nil + } + } + + return errors.New("could not unmarshal author: " + string(b)) +} + +func (a *packageJsonPeople) String() string { + b := strings.Builder{} + b.WriteString(a.Name) + if a.Email != "" { + b.WriteString(fmt.Sprintf(" <%s>", a.Email)) + } + if a.URL != "" { + b.WriteString(fmt.Sprintf(" (%s)", a.URL)) + } + return b.String() +} + +type packageJsonRepository struct { + Type string `json:"type"` + URL string `json:"url"` + Directory string `json:"directory"` +} + +func (r *packageJsonRepository) UnmarshalJSON(b []byte) error { + var repositoryStr string + type repositoryStruct packageJsonRepository + repo := repositoryStruct{} + + // try to unmarshal as structured object + err := json.Unmarshal(b, &repo) + if err == nil { + r.Type = repo.Type + r.URL = repo.URL + r.Directory = repo.Directory + return nil + } + + // try to unmarshal as string + err = json.Unmarshal(b, &repositoryStr) + if err == nil { + // handle case where the type is provided as prefix like `bitbucket:` + parts := strings.SplitN(repositoryStr, ":", 2) + if len(parts) == 2 { + r.Type = parts[0] + r.URL = parts[1] + return nil + } else { + r.Type = "github" + r.URL = repositoryStr + return nil + } + } + + return errors.New("could not unmarshal repository: " + string(b)) +} + +type PackageJsonParser struct{} + +func (p *PackageJsonParser) Parse(r io.Reader) ([]*mvd.Package, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + var packageJson packageJson + err = json.Unmarshal(data, &packageJson) + if err != nil { + return nil, err + } + + entries := []*mvd.Package{} + + // add own package + entries = append(entries, &mvd.Package{ + Name: packageJson.Name, + Version: packageJson.Version, + Format: "npm", + Namespace: "nodejs", + }) + + // add all dependencies + + for k, v := range packageJson.Dependencies { + entries = append(entries, &mvd.Package{ + Name: k, + Version: v, + Format: "npm", + Namespace: "nodejs", + }) + } + + return entries, nil +} diff --git a/providers/os/resources/npm/package_json_test.go b/providers/os/resources/npm/package_json_test.go new file mode 100644 index 0000000000..e1f9ba5c98 --- /dev/null +++ b/providers/os/resources/npm/package_json_test.go @@ -0,0 +1,278 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package npm + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" +) + +func TestPackageJson(t *testing.T) { + tests := []struct { + Fixture string + Expected packageJson + }{ + { + Fixture: "./testdata/package-json/author.json", + Expected: packageJson{ + Name: "author.js", + Author: &packageJsonPeople{ + Name: "Barney Rubble", + Email: "b@rubble.com", + URL: "http://barnyrubble.tumblr.com/", + }, + }, + }, + { + Fixture: "./testdata/package-json/author_shorten.json", + Expected: packageJson{ + Name: "author-shorten.js", + Author: &packageJsonPeople{ + Name: "Barney Rubble", + Email: "b@rubble.com", + URL: "http://barnyrubble.tumblr.com/", + }, + }, + }, + { + Fixture: "./testdata/package-json/bugs.json", + Expected: packageJson{ + // we do not parse bugs, so it should be empty + }, + }, + { + Fixture: "./testdata/package-json/bundle_dependencies.json", + Expected: packageJson{ + Name: "awesome-web-framework", + Version: "1.0.0", + // we do not parse bundle dependencies, so it should be empty + }, + }, + { + Fixture: "./testdata/package-json/contributors.json", + Expected: packageJson{ + Contributors: []packageJsonPeople{ + { + Name: "Barney Rubble", + Email: "b@rubble.com", + URL: "http://barnyrubble.tumblr.com/", + }, + }, + }, + }, + { + Fixture: "./testdata/package-json/contributors_shorten.json", + Expected: packageJson{ + Contributors: []packageJsonPeople{ + { + Name: "Barney Rubble", + Email: "b@rubble.com", + URL: "http://barnyrubble.tumblr.com/", + }, + }, + }, + }, + { + Fixture: "./testdata/package-json/cpu_exclude.json", + Expected: packageJson{ + CPU: []string{"!arm", "!mips"}, + }, + }, + { + Fixture: "./testdata/package-json/cpu_include.json", + Expected: packageJson{ + CPU: []string{"x64", "ia32"}, + }, + }, + { + Fixture: "./testdata/package-json/dependencies.json", + Expected: packageJson{ + Dependencies: map[string]string{ + "foo": "1.0.0 - 2.9999.9999", + "bar": ">=1.0.2 <2.1.2", + "baz": ">1.0.2 <=2.3.4", + "boo": "2.0.1", + "qux": "<1.0.0 || >=2.3.1 <2.4.5 || >=2.5.2 <3.0.0", + "asd": "http://asdf.com/asdf.tar.gz", + "til": "~1.2", + "elf": "~1.2.3", + "two": "2.x", + "thr": "3.3.x", + "lat": "latest", + "dyl": "file:../dyl", + }, + }, + }, + { + Fixture: "./testdata/package-json/dev_dependencies.json", + Expected: packageJson{ + Name: "ethopia-waza", + Description: "a delightfully fruity coffee varietal", + Version: "1.2.3", + DevDependencies: map[string]string{ + "coffee-script": "~1.6.3", + }, + }, + }, + { + Fixture: "./testdata/package-json/engines.json", + Expected: packageJson{ + Engines: map[string]string{ + "node": ">=0.10.3 <15", + }, + }, + }, + { + Fixture: "./testdata/package-json/homepage.json", + Expected: packageJson{ + Homepage: "https://github.com/owner/project#readme", + }, + }, + { + Fixture: "./testdata/package-json/license_deprecated_01.json", + Expected: packageJson{ + // we ignore those licenses for now + License: "", + }, + }, + { + Fixture: "./testdata/package-json/license_deprecated_02.json", + Expected: packageJson{ + // we ignore those licenses for now + License: "", + }, + }, + { + Fixture: "./testdata/package-json/license_spdx.json", + Expected: packageJson{ + License: "BSD-3-Clause", + }, + }, + { + Fixture: "./testdata/package-json/license_spdx_expression.json", + Expected: packageJson{ + License: "(MIT OR Apache-2.0)", + }, + }, + { + Fixture: "./testdata/package-json/os_exclude.json", + Expected: packageJson{ + OS: []string{"!win32"}, + }, + }, + { + Fixture: "./testdata/package-json/os_include.json", + Expected: packageJson{ + OS: []string{"darwin", "linux"}, + }, + }, + { + Fixture: "./testdata/package-json/peer_dependencies.json", + Expected: packageJson{ + Name: "tea-latte", + Version: "1.3.5", + // we do not parse peerDependencies so it should be empty + }, + }, + { + Fixture: "./testdata/package-json/peer_dependencies_meta.json", + Expected: packageJson{ + Name: "tea-latte", + Version: "1.3.5", + // we do not parse peerDependenciesMeta so it should be empty + }, + }, + { + Fixture: "./testdata/package-json/private.json", + Expected: packageJson{ + Private: true, + }, + }, + { + Fixture: "./testdata/package-json/repository.json", + Expected: packageJson{ + Repository: packageJsonRepository{ + Type: "git", + URL: "https://github.com/npm/cli.git", + }, + }, + }, + { + Fixture: "./testdata/package-json/repository_bitbucket.json", + Expected: packageJson{ + Repository: packageJsonRepository{ + Type: "bitbucket", + URL: "user/repo", + }, + }, + }, + { + Fixture: "./testdata/package-json/repository_directory.json", + Expected: packageJson{ + Repository: packageJsonRepository{ + Type: "git", + URL: "https://github.com/facebook/react.git", + Directory: "packages/react-dom", + }, + }, + }, + { + Fixture: "./testdata/package-json/repository_gh.json", + Expected: packageJson{ + Repository: packageJsonRepository{ + Type: "github", + URL: "npm/npm", + }, + }, + }, + { + Fixture: "./testdata/package-json/workspaces.json", + Expected: packageJson{ + Name: "workspace-example", + // we do not parse workspaces so it should be empty + }, + }, + } + + for i := range tests { + test := tests[i] + t.Run(test.Fixture, func(t *testing.T) { + f, err := os.Open(tests[i].Fixture) + require.NoError(t, err) + + pkg := packageJson{} + json.NewDecoder(f).Decode(&pkg) + assert.Equal(t, tests[i].Expected, pkg) + }) + } +} + +func TestPackageJsonParser(t *testing.T) { + f, err := os.Open("./testdata/package-json/express-package.json") + require.NoError(t, err) + defer f.Close() + + pkgs, err := (&PackageJsonParser{}).Parse(f) + assert.Nil(t, err) + assert.Equal(t, 31, len(pkgs)) + + assert.Contains(t, pkgs, &mvd.Package{ + Name: "path-to-regexp", + Version: "0.1.7", + Format: "npm", + Namespace: "nodejs", + }) + + assert.Contains(t, pkgs, &mvd.Package{ + Name: "range-parser", + Version: "~1.2.0", + Format: "npm", + Namespace: "nodejs", + }) +} diff --git a/providers/os/resources/npm/package_lock.go b/providers/os/resources/npm/package_lock.go new file mode 100644 index 0000000000..a789e71bca --- /dev/null +++ b/providers/os/resources/npm/package_lock.go @@ -0,0 +1,55 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package npm + +import ( + "io" + + "encoding/json" + "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" +) + +type PackageJsonLockEntry struct { + Version string `json:"version"` + Dev bool `json:"dev"` +} + +// PackageJsonLock is the struct to represent the package.lock file +type PackageJsonLock struct { + Name string `json:"name"` + Version string `json:"version"` + Dependencies map[string]PackageJsonLockEntry `jsonn:"dependencies"` +} + +type PackageLockParser struct{} + +func (p *PackageLockParser) Parse(r io.Reader) ([]*mvd.Package, error) { + var packageJsonLock PackageJsonLock + err := json.NewDecoder(r).Decode(&packageJsonLock) + if err != nil { + return nil, err + } + + entries := []*mvd.Package{} + + // add own package + entries = append(entries, &mvd.Package{ + Name: packageJsonLock.Name, + Version: packageJsonLock.Version, + Format: "npm", + Namespace: "nodejs", + }) + + // add all dependencies + for k, v := range packageJsonLock.Dependencies { + entries = append(entries, &mvd.Package{ + Name: k, + Version: v.Version, + Format: "npm", + Namespace: "nodejs", + }) + } + + return entries, nil +} diff --git a/providers/os/resources/npm/package_lock_test.go b/providers/os/resources/npm/package_lock_test.go new file mode 100644 index 0000000000..8dc4712530 --- /dev/null +++ b/providers/os/resources/npm/package_lock_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package npm + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" +) + +func TestPackageJsonLockParser(t *testing.T) { + f, err := os.Open("./testdata/package-lock/workbox-package-lock.json") + require.NoError(t, err) + + defer f.Close() + + pkgs, err := (&PackageLockParser{}).Parse(f) + assert.Nil(t, err) + assert.Equal(t, 1300, len(pkgs)) + + assert.Contains(t, pkgs, &mvd.Package{ + Name: "@babel/generator", + Version: "7.0.0", + Format: "npm", + Namespace: "nodejs", + }) + + assert.Contains(t, pkgs, &mvd.Package{ + Name: "@lerna/changed", + Version: "3.3.2", + Format: "npm", + Namespace: "nodejs", + }) +} diff --git a/providers/os/resources/npm/packagejson.go b/providers/os/resources/npm/packagejson.go deleted file mode 100644 index f7a9bdacdd..0000000000 --- a/providers/os/resources/npm/packagejson.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package npm - -import ( - "encoding/json" - "io" - - "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" -) - -// PackageJson allows parsing the package json file -type PackageJson struct { - Name string `json:"name"` - Description string `json:"description"` - Version string `json:"version"` - License string `jsonn:"license"` - Dependencies map[string]string `jsonn:"dependencies"` - DevDependencies map[string]string `jsonn:"devDependencies"` -} - -type PackageJsonLockEntry struct { - Version string `json:"version"` - Dev bool `json:"dev"` -} - -// PackageJsonLock is the struct to represent the package.lock file -type PackageJsonLock struct { - Name string `json:"name"` - Version string `json:"version"` - Dependencies map[string]PackageJsonLockEntry `jsonn:"dependencies"` -} - -func ParsePackageJson(r io.Reader) ([]*mvd.Package, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var packageJson PackageJson - err = json.Unmarshal(data, &packageJson) - if err != nil { - return nil, err - } - - entries := []*mvd.Package{} - - // add own package - entries = append(entries, &mvd.Package{ - Name: packageJson.Name, - Version: packageJson.Version, - Format: "npm", - Namespace: "nodejs", - }) - - // add all dependencies - - for k, v := range packageJson.Dependencies { - entries = append(entries, &mvd.Package{ - Name: k, - Version: v, - Format: "npm", - Namespace: "nodejs", - }) - } - - return entries, nil -} - -func ParsePackageJsonLock(r io.Reader) ([]*mvd.Package, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var packageJsonLock PackageJsonLock - err = json.Unmarshal(data, &packageJsonLock) - if err != nil { - return nil, err - } - - entries := []*mvd.Package{} - - // add own package - entries = append(entries, &mvd.Package{ - Name: packageJsonLock.Name, - Version: packageJsonLock.Version, - Format: "npm", - Namespace: "nodejs", - }) - - // add all dependencies - for k, v := range packageJsonLock.Dependencies { - entries = append(entries, &mvd.Package{ - Name: k, - Version: v.Version, - Format: "npm", - Namespace: "nodejs", - }) - } - - return entries, nil -} diff --git a/providers/os/resources/npm/packagejson_test.go b/providers/os/resources/npm/packagejson_test.go deleted file mode 100644 index 1533909204..0000000000 --- a/providers/os/resources/npm/packagejson_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package npm_test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" - "go.mondoo.com/cnquery/v10/providers/os/resources/npm" -) - -func TestPackageJsonParser(t *testing.T) { - data, err := os.Open("./testdata/express-package.json") - if err != nil { - t.Fatal(err) - } - - pkgs, err := npm.ParsePackageJson(data) - assert.Nil(t, err) - assert.Equal(t, 31, len(pkgs)) - - assert.Contains(t, pkgs, &mvd.Package{ - Name: "path-to-regexp", - Version: "0.1.7", - Format: "npm", - Namespace: "nodejs", - }) - - // "range-parser": "~1.2.0", - // TODO: we need to be better at version ranges - assert.Contains(t, pkgs, &mvd.Package{ - Name: "range-parser", - Version: "~1.2.0", - Format: "npm", - Namespace: "nodejs", - }) -} - -func TestPackageJsonLockParser(t *testing.T) { - data, err := os.Open("./testdata/workbox-package-lock.json") - if err != nil { - t.Fatal(err) - } - - pkgs, err := npm.ParsePackageJsonLock(data) - assert.Nil(t, err) - assert.Equal(t, 1300, len(pkgs)) - - assert.Contains(t, pkgs, &mvd.Package{ - Name: "@babel/generator", - Version: "7.0.0", - Format: "npm", - Namespace: "nodejs", - }) - - assert.Contains(t, pkgs, &mvd.Package{ - Name: "@lerna/changed", - Version: "3.3.2", - Format: "npm", - Namespace: "nodejs", - }) -} diff --git a/providers/os/resources/npm/testdata/package-json/author.json b/providers/os/resources/npm/testdata/package-json/author.json new file mode 100644 index 0000000000..9fc4b805e4 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/author.json @@ -0,0 +1,8 @@ +{ + "name": "author.js", + "author": { + "name": "Barney Rubble", + "email": "b@rubble.com", + "url": "http://barnyrubble.tumblr.com/" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/author_shorten.json b/providers/os/resources/npm/testdata/package-json/author_shorten.json new file mode 100644 index 0000000000..e9c7059d5f --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/author_shorten.json @@ -0,0 +1,4 @@ +{ + "name": "author-shorten.js", + "author": "Barney Rubble (http://barnyrubble.tumblr.com/)" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/bugs.json b/providers/os/resources/npm/testdata/package-json/bugs.json new file mode 100644 index 0000000000..b61e24034b --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/bugs.json @@ -0,0 +1,6 @@ +{ + "bugs": { + "url": "https://github.com/owner/project/issues", + "email": "project@hostname.com" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/bundle_dependencies.json b/providers/os/resources/npm/testdata/package-json/bundle_dependencies.json new file mode 100644 index 0000000000..1a0854dd88 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/bundle_dependencies.json @@ -0,0 +1,5 @@ +{ + "name": "awesome-web-framework", + "version": "1.0.0", + "bundleDependencies": ["renderized", "super-streams"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/contributors.json b/providers/os/resources/npm/testdata/package-json/contributors.json new file mode 100644 index 0000000000..36b60ce62d --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/contributors.json @@ -0,0 +1,7 @@ +{ + "contributors": [{ + "name": "Barney Rubble", + "email": "b@rubble.com", + "url": "http://barnyrubble.tumblr.com/" + }] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/contributors_shorten.json b/providers/os/resources/npm/testdata/package-json/contributors_shorten.json new file mode 100644 index 0000000000..5a2edd7003 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/contributors_shorten.json @@ -0,0 +1,5 @@ +{ + "contributors": [ + "Barney Rubble (http://barnyrubble.tumblr.com/)" + ] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/cpu_exclude.json b/providers/os/resources/npm/testdata/package-json/cpu_exclude.json new file mode 100644 index 0000000000..5db50aab9d --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/cpu_exclude.json @@ -0,0 +1,3 @@ +{ + "cpu": ["!arm", "!mips"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/cpu_include.json b/providers/os/resources/npm/testdata/package-json/cpu_include.json new file mode 100644 index 0000000000..808d8e6b29 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/cpu_include.json @@ -0,0 +1,3 @@ +{ + "cpu": ["x64", "ia32"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/dependencies.json b/providers/os/resources/npm/testdata/package-json/dependencies.json new file mode 100644 index 0000000000..c29a518448 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/dependencies.json @@ -0,0 +1,16 @@ +{ + "dependencies": { + "foo": "1.0.0 - 2.9999.9999", + "bar": ">=1.0.2 <2.1.2", + "baz": ">1.0.2 <=2.3.4", + "boo": "2.0.1", + "qux": "<1.0.0 || >=2.3.1 <2.4.5 || >=2.5.2 <3.0.0", + "asd": "http://asdf.com/asdf.tar.gz", + "til": "~1.2", + "elf": "~1.2.3", + "two": "2.x", + "thr": "3.3.x", + "lat": "latest", + "dyl": "file:../dyl" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/dev_dependencies.json b/providers/os/resources/npm/testdata/package-json/dev_dependencies.json new file mode 100644 index 0000000000..bf14c051d1 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/dev_dependencies.json @@ -0,0 +1,12 @@ +{ + "name": "ethopia-waza", + "description": "a delightfully fruity coffee varietal", + "version": "1.2.3", + "devDependencies": { + "coffee-script": "~1.6.3" + }, + "scripts": { + "prepare": "coffee -o lib/ -c src/waza.coffee" + }, + "main": "lib/waza.js" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/engines.json b/providers/os/resources/npm/testdata/package-json/engines.json new file mode 100644 index 0000000000..9766d3d065 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/engines.json @@ -0,0 +1,5 @@ +{ + "engines": { + "node": ">=0.10.3 <15" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/express-package.json b/providers/os/resources/npm/testdata/package-json/express-package.json similarity index 100% rename from providers/os/resources/npm/testdata/express-package.json rename to providers/os/resources/npm/testdata/package-json/express-package.json diff --git a/providers/os/resources/npm/testdata/package-json/homepage.json b/providers/os/resources/npm/testdata/package-json/homepage.json new file mode 100644 index 0000000000..edb770b6e9 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/homepage.json @@ -0,0 +1,3 @@ +{ + "homepage": "https://github.com/owner/project#readme" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/license_deprecated_01.json b/providers/os/resources/npm/testdata/package-json/license_deprecated_01.json new file mode 100644 index 0000000000..5aac68e504 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/license_deprecated_01.json @@ -0,0 +1,6 @@ +{ + "license" : { + "type" : "ISC", + "url" : "https://opensource.org/licenses/ISC" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/license_deprecated_02.json b/providers/os/resources/npm/testdata/package-json/license_deprecated_02.json new file mode 100644 index 0000000000..6569fbe646 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/license_deprecated_02.json @@ -0,0 +1,12 @@ +{ + "licenses" : [ + { + "type": "MIT", + "url": "https://www.opensource.org/licenses/mit-license.php" + }, + { + "type": "Apache-2.0", + "url": "https://opensource.org/licenses/apache2.0.php" + } + ] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/license_spdx.json b/providers/os/resources/npm/testdata/package-json/license_spdx.json new file mode 100644 index 0000000000..fe00c6a8fd --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/license_spdx.json @@ -0,0 +1,3 @@ +{ + "license": "BSD-3-Clause" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/license_spdx_expression.json b/providers/os/resources/npm/testdata/package-json/license_spdx_expression.json new file mode 100644 index 0000000000..632e4d7bfb --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/license_spdx_expression.json @@ -0,0 +1,3 @@ +{ + "license": "(MIT OR Apache-2.0)" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/os_exclude.json b/providers/os/resources/npm/testdata/package-json/os_exclude.json new file mode 100644 index 0000000000..ec4643724a --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/os_exclude.json @@ -0,0 +1,3 @@ +{ + "os": ["!win32"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/os_include.json b/providers/os/resources/npm/testdata/package-json/os_include.json new file mode 100644 index 0000000000..d31d90b956 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/os_include.json @@ -0,0 +1,3 @@ +{ +"os": ["darwin", "linux"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/peer_dependencies.json b/providers/os/resources/npm/testdata/package-json/peer_dependencies.json new file mode 100644 index 0000000000..7e7266f46b --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/peer_dependencies.json @@ -0,0 +1,7 @@ +{ + "name": "tea-latte", + "version": "1.3.5", + "peerDependencies": { + "tea": "2.x" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/peer_dependencies_meta.json b/providers/os/resources/npm/testdata/package-json/peer_dependencies_meta.json new file mode 100644 index 0000000000..560008befe --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/peer_dependencies_meta.json @@ -0,0 +1,13 @@ +{ + "name": "tea-latte", + "version": "1.3.5", + "peerDependencies": { + "tea": "2.x", + "soy-milk": "1.2" + }, + "peerDependenciesMeta": { + "soy-milk": { + "optional": true + } + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/private.json b/providers/os/resources/npm/testdata/package-json/private.json new file mode 100644 index 0000000000..83a4a45227 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/private.json @@ -0,0 +1,3 @@ +{ + "private": true +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/repository.json b/providers/os/resources/npm/testdata/package-json/repository.json new file mode 100644 index 0000000000..48131fc7bf --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/repository.json @@ -0,0 +1,6 @@ +{ + "repository": { + "type": "git", + "url": "https://github.com/npm/cli.git" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/repository_bitbucket.json b/providers/os/resources/npm/testdata/package-json/repository_bitbucket.json new file mode 100644 index 0000000000..5d341cda2b --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/repository_bitbucket.json @@ -0,0 +1,3 @@ +{ + "repository": "bitbucket:user/repo" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/repository_directory.json b/providers/os/resources/npm/testdata/package-json/repository_directory.json new file mode 100644 index 0000000000..1542be72be --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/repository_directory.json @@ -0,0 +1,7 @@ +{ + "repository": { + "type": "git", + "url": "https://github.com/facebook/react.git", + "directory": "packages/react-dom" + } +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/repository_gh.json b/providers/os/resources/npm/testdata/package-json/repository_gh.json new file mode 100644 index 0000000000..564f51b0d9 --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/repository_gh.json @@ -0,0 +1,3 @@ +{ + "repository": "npm/npm" +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/package-json/workspaces.json b/providers/os/resources/npm/testdata/package-json/workspaces.json new file mode 100644 index 0000000000..823a91167d --- /dev/null +++ b/providers/os/resources/npm/testdata/package-json/workspaces.json @@ -0,0 +1,4 @@ +{ + "name": "workspace-example", + "workspaces": ["./packages/*"] +} \ No newline at end of file diff --git a/providers/os/resources/npm/testdata/workbox-package-lock.json b/providers/os/resources/npm/testdata/package-lock/workbox-package-lock.json similarity index 100% rename from providers/os/resources/npm/testdata/workbox-package-lock.json rename to providers/os/resources/npm/testdata/package-lock/workbox-package-lock.json diff --git a/providers/os/resources/npm/testdata/d3-yarn.lock b/providers/os/resources/npm/testdata/yarn-lock/d3-yarn.lock similarity index 100% rename from providers/os/resources/npm/testdata/d3-yarn.lock rename to providers/os/resources/npm/testdata/yarn-lock/d3-yarn.lock diff --git a/providers/os/resources/npm/yarn.go b/providers/os/resources/npm/yarn_lock.go similarity index 88% rename from providers/os/resources/npm/yarn.go rename to providers/os/resources/npm/yarn_lock.go index eccbfe0168..dc1ad2303c 100644 --- a/providers/os/resources/npm/yarn.go +++ b/providers/os/resources/npm/yarn_lock.go @@ -22,7 +22,9 @@ type YarnLockEntry struct { Dependencies map[string]string } -func ParseYarnLock(r io.Reader) ([]*mvd.Package, error) { +type YarnLockParser struct{} + +func (p *YarnLockParser) Parse(r io.Reader) ([]*mvd.Package, error) { var b bytes.Buffer // iterate and convert the format to yaml on the fly @@ -52,7 +54,7 @@ func ParseYarnLock(r io.Reader) ([]*mvd.Package, error) { // add all dependencies for k, v := range yarnLock { - name, _, err := ParseYarnPackageName(k) + name, _, err := parseYarnPackageName(k) if err != nil { log.Error().Str("name", name).Msg("cannot parse yarn package name") continue @@ -68,7 +70,7 @@ func ParseYarnLock(r io.Reader) ([]*mvd.Package, error) { return entries, nil } -func ParseYarnPackageName(name string) (string, string, error) { +func parseYarnPackageName(name string) (string, string, error) { // a yarn package line may include may items pkgNames := strings.Split(name, ",") diff --git a/providers/os/resources/npm/yarn_test.go b/providers/os/resources/npm/yarn_lock_test.go similarity index 67% rename from providers/os/resources/npm/yarn_test.go rename to providers/os/resources/npm/yarn_lock_test.go index c82346df36..b011883177 100644 --- a/providers/os/resources/npm/yarn_test.go +++ b/providers/os/resources/npm/yarn_lock_test.go @@ -1,24 +1,23 @@ // Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 -package npm_test +package npm import ( + "github.com/stretchr/testify/require" "os" "testing" "github.com/stretchr/testify/assert" "go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream/mvd" - "go.mondoo.com/cnquery/v10/providers/os/resources/npm" ) func TestYarnParser(t *testing.T) { - data, err := os.Open("./testdata/d3-yarn.lock") - if err != nil { - t.Fatal(err) - } + f, err := os.Open("./testdata/yarn-lock/d3-yarn.lock") + require.NoError(t, err) + defer f.Close() - pkgs, err := npm.ParseYarnLock(data) + pkgs, err := (&YarnLockParser{}).Parse(f) assert.Nil(t, err) assert.Equal(t, 99, len(pkgs)) @@ -42,22 +41,22 @@ func TestParsePackagename(t *testing.T) { var version string var err error - name, version, err = npm.ParseYarnPackageName("source-map-support@~0.5.10") + name, version, err = parseYarnPackageName("source-map-support@~0.5.10") assert.Nil(t, err) assert.Equal(t, "source-map-support", name) assert.Equal(t, "~0.5.10", version) - name, version, err = npm.ParseYarnPackageName("@types/node@*") + name, version, err = parseYarnPackageName("@types/node@*") assert.Nil(t, err) assert.Equal(t, "@types/node", name) assert.Equal(t, "*", version) - name, version, err = npm.ParseYarnPackageName("@babel/code-frame@^7.0.0-beta.47") + name, version, err = parseYarnPackageName("@babel/code-frame@^7.0.0-beta.47") assert.Nil(t, err) assert.Equal(t, "@babel/code-frame", name) assert.Equal(t, "^7.0.0-beta.47", version) - name, version, err = npm.ParseYarnPackageName("has@^1.0.1, has@^1.0.3, has@~1.0.3") + name, version, err = parseYarnPackageName("has@^1.0.1, has@^1.0.3, has@~1.0.3") assert.Nil(t, err) assert.Equal(t, "has", name) assert.Equal(t, "^1.0.1", version) diff --git a/providers/os/resources/os.lr b/providers/os/resources/os.lr index d7f0a04a5c..fa739cae47 100644 --- a/providers/os/resources/os.lr +++ b/providers/os/resources/os.lr @@ -1151,6 +1151,7 @@ python.package @defaults("name version") { // Name of the package name() string // File containing the package metadata + // TODO: deprecate and use files instead file file // Version of the package version() string @@ -1161,6 +1162,7 @@ python.package @defaults("name version") { // Author email of the package authorEmail() string // Short package description + // TODO: deprecate and use description() instead summary() string // Package URL purl() string @@ -1170,6 +1172,36 @@ python.package @defaults("name version") { dependencies() []python.package } +nodejs { + init(path? string) + + // List of all discovered packages + packages() []python.package +} + +nodejs.package @defaults("name version") { + // ID is the nodejs.package unique identifier + id string + // Name of the package + name() string + // Version of the package + version() string + // License of the package + license() string + // Author of the package + // TODO: align with python.package author + author() string + // Short package description + description() string + // Package URL + purl() string + // Common Platform Enumeration (CPE) for the package + cpes() []core.cpe + + // Package files + files() []pkgFileInfo +} + // macOS specific resources macos { // macOS user defaults