From e34e974c2d62f584da672d39adc79962b0eb1570 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Wed, 16 Dec 2020 16:17:34 +0530 Subject: [PATCH] fix: Split collector into separate module according to file type (#175) --- src/collector.ts | 283 +++++------------- src/collector/go.mod.ts | 161 ++++++++++ src/collector/package.json.ts | 21 ++ .../pom.xml.ts} | 9 +- src/collector/requirements.txt.ts | 48 +++ src/server.ts | 15 +- src/types.ts | 90 ------ src/utils.ts | 3 +- .../go.mod.test.ts} | 6 +- .../package.json.test.ts} | 42 ++- .../pom.xml.test.ts} | 4 +- .../requirements.txt.test.ts} | 42 +-- test/consumer.test.ts | 5 +- 13 files changed, 354 insertions(+), 375 deletions(-) create mode 100644 src/collector/go.mod.ts create mode 100644 src/collector/package.json.ts rename src/{maven.collector.ts => collector/pom.xml.ts} (95%) create mode 100644 src/collector/requirements.txt.ts delete mode 100644 src/types.ts rename test/{gomod.collector.test.ts => collector/go.mod.test.ts} (99%) rename test/{npm.collector.test.ts => collector/package.json.test.ts} (77%) rename test/{maven.collector.test.ts => collector/pom.xml.test.ts} (98%) rename test/{pypi.collector.test.ts => collector/requirements.txt.test.ts} (75%) diff --git a/src/collector.ts b/src/collector.ts index d9a16184..62135526 100644 --- a/src/collector.ts +++ b/src/collector.ts @@ -3,230 +3,87 @@ * Licensed under the Apache-2.0 License. See License.txt in the project root for license information. * ------------------------------------------------------------------------------------------ */ 'use strict'; -import { Stream } from 'stream'; -import * as jsonAst from 'json-to-ast'; -import { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IPositionedString, IDependencyCollector, Dependency } from './types'; -import { stream_from_string, getGoLangImportsCmd } from './utils'; -import { config } from './config'; -import { exec } from 'child_process'; -import { parse, DocumentCstNode } from "@xml-tools/parser"; -import { buildAst, accept, XMLElement, XMLDocument } from "@xml-tools/ast"; -/* Please note :: There was issue with semverRegex usage in the code. During run time, it extracts - * version with 'v' prefix, but this is not be behavior of semver in CLI and test environment. - * At the moment, using regex directly to extract version information without 'v' prefix. */ -//import semverRegex = require('semver-regex'); -function semVerRegExp(line: string): RegExpExecArray { - const regExp = /(?<=^v?|\sv?)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*)(?:\.(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*))*)?(?:\+[\da-z-]+(?:\.[\da-z-]+)*)?(?=$|\s)/ig - return regExp.exec(line); +/* Determine what is the value */ +enum ValueType { + Invalid, + String, + Integer, + Float, + Array, + Object, + Boolean, + Null +}; + +/* Value variant */ +interface IVariant { + type: ValueType; + object: any; } -class NaivePyParser { - constructor(contents: string) { - this.dependencies = NaivePyParser.parseDependencies(contents); - } - - dependencies: Array; - - static parseDependencies(contents:string): Array { - const requirements = contents.split("\n"); - return requirements.reduce((dependencies, req, index) => { - // skip any text after # - if (req.includes('#')) { - req = req.split('#')[0]; - } - const parsedRequirement: Array = req.split(/[==,>=,<=]+/); - const pkgName:string = (parsedRequirement[0] || '').trim(); - // skip empty lines - if (pkgName.length > 0) { - const version = (parsedRequirement[1] || '').trim(); - const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 }); - entry.value = new Variant(ValueType.String, version); - entry.value_position = { line: index + 1, column: req.indexOf(version) + 1 }; - dependencies.push(new Dependency(entry)); - } - return dependencies; - }, []); - } - - parse(): Array { - return this.dependencies; - } +/* Line and column inside the JSON file */ +interface IPosition { + line: number; + column: number; +}; + +/* Key/Value entry with positions */ +interface IKeyValueEntry { + key: string; + value: IVariant; + key_position: IPosition; + value_position: IPosition; +}; + +class KeyValueEntry implements IKeyValueEntry { + key: string; + value: IVariant; + key_position: IPosition; + value_position: IPosition; + + constructor(k: string, pos: IPosition) { + this.key = k; + this.key_position = pos; + } } -/* Process entries found in the txt files and collect all dependency - * related information */ -class ReqDependencyCollector implements IDependencyCollector { - constructor(public classes: Array = ["dependencies"]) {} - - async collect(contents: string): Promise> { - let parser = new NaivePyParser(contents); - return parser.parse(); - } - +class Variant implements IVariant { + constructor(public type: ValueType, public object: any) {} } -class NaiveGomodParser { - constructor(contents: string, goImports: Set) { - this.dependencies = NaiveGomodParser.parseDependencies(contents, goImports); - } - - dependencies: Array; - - static getReplaceMap(line: string, index: number): any{ - // split the replace statements by '=>' - const parts: Array = line.replace('replace', '').replace('(', '').replace(')', '').trim().split('=>'); - const replaceWithVersion = semVerRegExp(parts[1]); - - // Skip lines without final version string - if (replaceWithVersion && replaceWithVersion.length > 0) { - const replaceTo: Array = (parts[0] || '').trim().split(' '); - const replaceToVersion = semVerRegExp(replaceTo[1]); - const replaceWith: Array = (parts[1] || '').trim().split(' '); - const replaceWithIndex = line.lastIndexOf(parts[1]); - const replaceEntry: IKeyValueEntry = new KeyValueEntry(replaceWith[0].trim(), { line: 0, column: 0 }); - replaceEntry.value = new Variant(ValueType.String, 'v' + replaceWithVersion[0]); - replaceEntry.value_position = { line: index + 1, column: (replaceWithIndex + replaceWithVersion.index) }; - const replaceDependency = new Dependency(replaceEntry); - const isReplaceToVersion: boolean = replaceToVersion && replaceToVersion.length > 0; - return {key: replaceTo[0].trim() + (isReplaceToVersion ? ('@v' + replaceToVersion[0]) : ''), value: replaceDependency}; - } - return null; - } - - static applyReplaceMap(dep: IDependency, replaceMap: Map): IDependency { - let replaceDependency = replaceMap.get(dep.name.value + "@" + dep.version.value); - if (replaceDependency === undefined) { - replaceDependency = replaceMap.get(dep.name.value); - if(replaceDependency === undefined) { - return dep; - } - } - return replaceDependency; - } - - static parseDependencies(contents:string, goImports: Set): Array { - let replaceMap = new Map(); - let goModDeps = contents.split("\n").reduce((dependencies, line, index) => { - // skip any text after '//' - if (line.includes("//")) { - line = line.split("//")[0]; - } - if (line.includes("=>")) { - let replaceEntry = NaiveGomodParser.getReplaceMap(line, index); - if (replaceEntry) { - replaceMap.set(replaceEntry.key, replaceEntry.value); - } - } else { - // Not using semver directly, look at comment on import statement. - //const version = semverRegex().exec(line) - const version = semVerRegExp(line); - // Skip lines without version string - if (version && version.length > 0) { - const parts: Array = line.replace('require', '').replace('(', '').replace(')', '').trim().split(' '); - const pkgName: string = (parts[0] || '').trim(); - // Ignore line starting with replace clause and empty package - if (pkgName.length > 0) { - const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 }); - entry.value = new Variant(ValueType.String, 'v' + version[0]); - entry.value_position = { line: index + 1, column: version.index }; - // Push all direct and indirect modules present in go.mod (manifest) - dependencies.push(new Dependency(entry)); - } - } - } - return dependencies; - }, []); - - let goPackageDeps = []; - - goImports.forEach(importStatement => { - let exactMatchDep: Dependency = null; - let moduleMatchDep: Dependency = null; - goModDeps.forEach(goModDep => { - if (importStatement == goModDep.name.value) { - // Software stack uses the module - exactMatchDep = goModDep; - } else if (importStatement.startsWith(goModDep.name.value + "/")) { - // Find longest module name that matches the import statement - if (moduleMatchDep == null) { - moduleMatchDep = goModDep; - } else if (moduleMatchDep.name.value.length < goModDep.name.value.length) { - moduleMatchDep = goModDep; - } - } - }); - - if (exactMatchDep == null && moduleMatchDep != null) { - // Software stack uses a package from the module - let replaceDependency = NaiveGomodParser.applyReplaceMap(moduleMatchDep, replaceMap); - if (replaceDependency !== moduleMatchDep) { - importStatement = importStatement.replace(moduleMatchDep.name.value, replaceDependency.name.value); - } - const entry: IKeyValueEntry = new KeyValueEntry(importStatement + '@' + replaceDependency.name.value, replaceDependency.name.position); - entry.value = new Variant(ValueType.String, replaceDependency.version.value); - entry.value_position = replaceDependency.version.position; - goPackageDeps.push(new Dependency(entry)); - } - }); - - goModDeps = goModDeps.map(goModDep => NaiveGomodParser.applyReplaceMap(goModDep, replaceMap)); - - // Return modules present in go.mod and packages used in imports. - return [...goModDeps, ...goPackageDeps]; - } - - parse(): Array { - return this.dependencies; - } +/* String value with position */ +interface IPositionedString { + value: string; + position: IPosition; } -/* Process entries found in the go.mod file and collect all dependency - * related information */ -class GomodDependencyCollector implements IDependencyCollector { - constructor(private manifestFile: string, public classes: Array = ["dependencies"]) { - this.manifestFile = manifestFile; - } - - async collect(contents: string): Promise> { - let promiseExec = new Promise>((resolve, reject) => { - const vscodeRootpath = this.manifestFile.replace("file://", "").replace("/go.mod", "") - exec(getGoLangImportsCmd(), - { shell: process.env["SHELL"], windowsHide: true, cwd: vscodeRootpath, maxBuffer: 1024 * 1200 }, (error, stdout, stderr) => { - if (error) { - console.error(`Command failed, environment SHELL: [${process.env["SHELL"]}] PATH: [${process.env["PATH"]}] CWD: [${process.env["CWD"]}]`) - if (error.code == 127) { // Invalid command, go executable not found - reject(`Unable to locate '${config.golang_executable}'`); - } else { - reject(`Unable to execute '${config.golang_executable} list' command, run '${config.golang_executable} mod tidy' to know more`); - } - } else { - resolve(new Set(stdout.toString().split("\n"))); - } - }); - }); - const goImports: Set = await promiseExec; - let parser = new NaiveGomodParser(contents, goImports); - return parser.parse(); - } - +/* Dependency specification */ +interface IDependency { + name: IPositionedString; + version: IPositionedString; } -class PackageJsonCollector implements IDependencyCollector { - constructor(public classes: Array = ["dependencies"]) {} +/* Dependency collector interface */ +interface IDependencyCollector { + classes: Array; + collect(contents: string): Promise>; +} - async collect(contents: string): Promise> { - const ast = jsonAst(contents); - return ast.children. - filter(c => this.classes.includes(c.key.value)). - flatMap(c => c.value.children). - map(c => { - let entry: IKeyValueEntry = new KeyValueEntry(c.key.value, {line: c.key.loc.start.line, column: c.key.loc.start.column + 1}); - entry.value = new Variant(ValueType.String, c.value.value); - entry.value_position = {line: c.value.loc.start.line, column: c.value.loc.start.column + 1}; - return new Dependency(entry); - }); - } +/* Dependency class that can be created from `IKeyValueEntry` */ +class Dependency implements IDependency { + name: IPositionedString; + version: IPositionedString; + constructor(dependency: IKeyValueEntry) { + this.name = { + value: dependency.key, + position: dependency.key_position + }; + this.version = { + value: dependency.value.object, + position: dependency.value_position + }; + } } -export { IDependencyCollector, PackageJsonCollector, ReqDependencyCollector, GomodDependencyCollector, IPositionedString, IDependency }; +export { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IPositionedString, IDependencyCollector, Dependency }; diff --git a/src/collector/go.mod.ts b/src/collector/go.mod.ts new file mode 100644 index 00000000..0a6b2f4f --- /dev/null +++ b/src/collector/go.mod.ts @@ -0,0 +1,161 @@ +'use strict'; + +import { IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IDependencyCollector, Dependency } from '../collector'; +import { getGoLangImportsCmd } from '../utils'; +import { config } from '../config'; +import { exec } from 'child_process'; + +/* Please note :: There was issue with semverRegex usage in the code. During run time, it extracts + * version with 'v' prefix, but this is not be behavior of semver in CLI and test environment. + * At the moment, using regex directly to extract version information without 'v' prefix. */ +//import semverRegex = require('semver-regex'); +function semVerRegExp(line: string): RegExpExecArray { + const regExp = /(?<=^v?|\sv?)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*)(?:\.(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*))*)?(?:\+[\da-z-]+(?:\.[\da-z-]+)*)?(?=$|\s)/ig + return regExp.exec(line); +} + +class NaiveGomodParser { + constructor(contents: string, goImports: Set) { + this.dependencies = NaiveGomodParser.parseDependencies(contents, goImports); + } + + dependencies: Array; + + static getReplaceMap(line: string, index: number): any{ + // split the replace statements by '=>' + const parts: Array = line.replace('replace', '').replace('(', '').replace(')', '').trim().split('=>'); + const replaceWithVersion = semVerRegExp(parts[1]); + + // Skip lines without final version string + if (replaceWithVersion && replaceWithVersion.length > 0) { + const replaceTo: Array = (parts[0] || '').trim().split(' '); + const replaceToVersion = semVerRegExp(replaceTo[1]); + const replaceWith: Array = (parts[1] || '').trim().split(' '); + const replaceWithIndex = line.lastIndexOf(parts[1]); + const replaceEntry: IKeyValueEntry = new KeyValueEntry(replaceWith[0].trim(), { line: 0, column: 0 }); + replaceEntry.value = new Variant(ValueType.String, 'v' + replaceWithVersion[0]); + replaceEntry.value_position = { line: index + 1, column: (replaceWithIndex + replaceWithVersion.index) }; + const replaceDependency = new Dependency(replaceEntry); + const isReplaceToVersion: boolean = replaceToVersion && replaceToVersion.length > 0; + return {key: replaceTo[0].trim() + (isReplaceToVersion ? ('@v' + replaceToVersion[0]) : ''), value: replaceDependency}; + } + return null; + } + + static applyReplaceMap(dep: IDependency, replaceMap: Map): IDependency { + let replaceDependency = replaceMap.get(dep.name.value + "@" + dep.version.value); + if (replaceDependency === undefined) { + replaceDependency = replaceMap.get(dep.name.value); + if(replaceDependency === undefined) { + return dep; + } + } + return replaceDependency; + } + + static parseDependencies(contents:string, goImports: Set): Array { + let replaceMap = new Map(); + let goModDeps = contents.split("\n").reduce((dependencies, line, index) => { + // skip any text after '//' + if (line.includes("//")) { + line = line.split("//")[0]; + } + if (line.includes("=>")) { + let replaceEntry = NaiveGomodParser.getReplaceMap(line, index); + if (replaceEntry) { + replaceMap.set(replaceEntry.key, replaceEntry.value); + } + } else { + // Not using semver directly, look at comment on import statement. + //const version = semverRegex().exec(line) + const version = semVerRegExp(line); + // Skip lines without version string + if (version && version.length > 0) { + const parts: Array = line.replace('require', '').replace('(', '').replace(')', '').trim().split(' '); + const pkgName: string = (parts[0] || '').trim(); + // Ignore line starting with replace clause and empty package + if (pkgName.length > 0) { + const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 }); + entry.value = new Variant(ValueType.String, 'v' + version[0]); + entry.value_position = { line: index + 1, column: version.index }; + // Push all direct and indirect modules present in go.mod (manifest) + dependencies.push(new Dependency(entry)); + } + } + } + return dependencies; + }, []); + + let goPackageDeps = []; + + goImports.forEach(importStatement => { + let exactMatchDep: Dependency = null; + let moduleMatchDep: Dependency = null; + goModDeps.forEach(goModDep => { + if (importStatement == goModDep.name.value) { + // Software stack uses the module + exactMatchDep = goModDep; + } else if (importStatement.startsWith(goModDep.name.value + "/")) { + // Find longest module name that matches the import statement + if (moduleMatchDep == null) { + moduleMatchDep = goModDep; + } else if (moduleMatchDep.name.value.length < goModDep.name.value.length) { + moduleMatchDep = goModDep; + } + } + }); + + if (exactMatchDep == null && moduleMatchDep != null) { + // Software stack uses a package from the module + let replaceDependency = NaiveGomodParser.applyReplaceMap(moduleMatchDep, replaceMap); + if (replaceDependency !== moduleMatchDep) { + importStatement = importStatement.replace(moduleMatchDep.name.value, replaceDependency.name.value); + } + const entry: IKeyValueEntry = new KeyValueEntry(importStatement + '@' + replaceDependency.name.value, replaceDependency.name.position); + entry.value = new Variant(ValueType.String, replaceDependency.version.value); + entry.value_position = replaceDependency.version.position; + goPackageDeps.push(new Dependency(entry)); + } + }); + + goModDeps = goModDeps.map(goModDep => NaiveGomodParser.applyReplaceMap(goModDep, replaceMap)); + + // Return modules present in go.mod and packages used in imports. + return [...goModDeps, ...goPackageDeps]; + } + + parse(): Array { + return this.dependencies; + } +} + +/* Process entries found in the go.mod file and collect all dependency + * related information */ +export class DependencyCollector implements IDependencyCollector { + constructor(private manifestFile: string, public classes: Array = ["dependencies"]) { + this.manifestFile = manifestFile; + } + + async collect(contents: string): Promise> { + let promiseExec = new Promise>((resolve, reject) => { + const vscodeRootpath = this.manifestFile.replace("file://", "").replace("/go.mod", "") + exec(getGoLangImportsCmd(), + { shell: process.env["SHELL"], windowsHide: true, cwd: vscodeRootpath, maxBuffer: 1024 * 1200 }, (error, stdout, stderr) => { + if (error) { + console.error(`Command failed, environment SHELL: [${process.env["SHELL"]}] PATH: [${process.env["PATH"]}] CWD: [${process.env["CWD"]}]`) + if (error.code == 127) { // Invalid command, go executable not found + reject(`Unable to locate '${config.golang_executable}'`); + } else { + reject(`Unable to execute '${config.golang_executable} list' command, run '${config.golang_executable} mod tidy' to know more`); + } + } else { + resolve(new Set(stdout.toString().split("\n"))); + } + }); + }); + const goImports: Set = await promiseExec; + let parser = new NaiveGomodParser(contents, goImports); + return parser.parse(); + } + +} diff --git a/src/collector/package.json.ts b/src/collector/package.json.ts new file mode 100644 index 00000000..7bfe596c --- /dev/null +++ b/src/collector/package.json.ts @@ -0,0 +1,21 @@ +'use strict'; + +import * as jsonAst from 'json-to-ast'; +import { IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IDependencyCollector, Dependency } from '../collector'; + +export class DependencyCollector implements IDependencyCollector { + constructor(public classes: Array = ["dependencies"]) {} + + async collect(contents: string): Promise> { + const ast = jsonAst(contents); + return ast.children. + filter(c => this.classes.includes(c.key.value)). + flatMap(c => c.value.children). + map(c => { + let entry: IKeyValueEntry = new KeyValueEntry(c.key.value, {line: c.key.loc.start.line, column: c.key.loc.start.column + 1}); + entry.value = new Variant(ValueType.String, c.value.value); + entry.value_position = {line: c.value.loc.start.line, column: c.value.loc.start.column + 1}; + return new Dependency(entry); + }); + } +} diff --git a/src/maven.collector.ts b/src/collector/pom.xml.ts similarity index 95% rename from src/maven.collector.ts rename to src/collector/pom.xml.ts index b13d548d..0d2d2d98 100644 --- a/src/maven.collector.ts +++ b/src/collector/pom.xml.ts @@ -1,9 +1,9 @@ 'use strict'; -import { IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IDependencyCollector, Dependency, IPositionedString, IPosition } from './types'; +import { IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IDependencyCollector, Dependency, IPositionedString, IPosition } from '../collector'; import { parse, DocumentCstNode } from "@xml-tools/parser"; import { buildAst, accept, XMLElement, XMLDocument } from "@xml-tools/ast"; -export class PomXmlDependencyCollector implements IDependencyCollector { +export class DependencyCollector implements IDependencyCollector { private xmlDocAst: XMLDocument; constructor(public classes: Array = ["dependencies"]) {} @@ -52,8 +52,7 @@ export class PomXmlDependencyCollector implements IDependencyCollector { } }; const validElementNames = ['groupId', 'artifactId', 'version']; - const dependencies = dependenciesNode?. - subElements. + const dependencies = dependenciesNode.subElements. filter(e => e.name === 'dependency'). // must include all validElementNames filter(e => e.subElements.filter(e => validElementNames.includes(e.name)).length == validElementNames.length). @@ -62,7 +61,7 @@ export class PomXmlDependencyCollector implements IDependencyCollector { map(e => new PomDependency(e)). filter(d => d.isValid()). map(d => d.toDependency()); - return dependencies || []; + return dependencies; } private createPropertySubstitution(e: XMLElement): Map { diff --git a/src/collector/requirements.txt.ts b/src/collector/requirements.txt.ts new file mode 100644 index 00000000..f9a7647f --- /dev/null +++ b/src/collector/requirements.txt.ts @@ -0,0 +1,48 @@ +'use strict'; + +import { IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IDependencyCollector, Dependency } from '../collector'; + +class NaivePyParser { + constructor(contents: string) { + this.dependencies = NaivePyParser.parseDependencies(contents); + } + + dependencies: Array; + + static parseDependencies(contents:string): Array { + const requirements = contents.split("\n"); + return requirements.reduce((dependencies, req, index) => { + // skip any text after # + if (req.includes('#')) { + req = req.split('#')[0]; + } + const parsedRequirement: Array = req.split(/[==,>=,<=]+/); + const pkgName:string = (parsedRequirement[0] || '').trim(); + // skip empty lines + if (pkgName.length > 0) { + const version = (parsedRequirement[1] || '').trim(); + const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 }); + entry.value = new Variant(ValueType.String, version); + entry.value_position = { line: index + 1, column: req.indexOf(version) + 1 }; + dependencies.push(new Dependency(entry)); + } + return dependencies; + }, []); + } + + parse(): Array { + return this.dependencies; + } +} + +/* Process entries found in the txt files and collect all dependency + * related information */ +export class DependencyCollector implements IDependencyCollector { + constructor(public classes: Array = ["dependencies"]) {} + + async collect(contents: string): Promise> { + let parser = new NaivePyParser(contents); + return parser.parse(); + } + +} diff --git a/src/server.ts b/src/server.ts index 721ff364..70d7dde7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,8 +8,11 @@ import * as fs from 'fs'; import { IPCMessageReader, IPCMessageWriter, createConnection, IConnection, TextDocuments, InitializeResult, CodeLens, CodeAction, CodeActionKind} from 'vscode-languageserver'; -import { IDependencyCollector, PackageJsonCollector, ReqDependencyCollector, GomodDependencyCollector } from './collector'; -import { PomXmlDependencyCollector } from './maven.collector'; +import { DependencyCollector as GoMod } from './collector/go.mod'; +import { DependencyCollector as PackageJson } from './collector/package.json'; +import { DependencyCollector as PomXml } from './collector/pom.xml'; +import { DependencyCollector as RequirementsTxt } from './collector/requirements.txt'; +import { IDependencyCollector } from './collector'; import { SecurityEngine, DiagnosticsPipeline, codeActionsMap } from './consumers'; import { NoopVulnerabilityAggregator, GolangVulnerabilityAggregator } from './aggregators'; import { AnalyticsSource } from './vulnerability'; @@ -291,20 +294,20 @@ const sendDiagnostics = async (ecosystem: string, diagnosticFilePath: string, co }; files.on(EventStream.Diagnostics, "^package\\.json$", (uri, name, contents) => { - sendDiagnostics('npm', uri, contents, new PackageJsonCollector()); + sendDiagnostics('npm', uri, contents, new PackageJson()); }); files.on(EventStream.Diagnostics, "^pom\\.xml$", (uri, name, contents) => { - sendDiagnostics('maven', uri, contents, new PomXmlDependencyCollector()); + sendDiagnostics('maven', uri, contents, new PomXml()); }); files.on(EventStream.Diagnostics, "^requirements\\.txt$", (uri, name, contents) => { - sendDiagnostics('pypi', uri, contents, new ReqDependencyCollector()); + sendDiagnostics('pypi', uri, contents, new RequirementsTxt()); }); files.on(EventStream.Diagnostics, "^go\\.mod$", (uri, name, contents) => { connection.console.log("Using golang executable: " + config.golang_executable); - sendDiagnostics('golang', uri, contents, new GomodDependencyCollector(uri)); + sendDiagnostics('golang', uri, contents, new GoMod(uri)); }); let checkDelay; diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index 149ce6bb..00000000 --- a/src/types.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* Determine what is the value */ -enum ValueType { - Invalid, - String, - Integer, - Float, - Array, - Object, - Boolean, - Null -}; - -/* Generic token interface, although currently we're going to use only IToken */ -interface IToken { - value: T; - line: number; - pos: number; -} - -/* Value variant */ -interface IVariant { - type: ValueType; - object: any; -} - -/* Line and column inside the JSON file */ -interface IPosition { - line: number; - column: number; -}; - -/* Key/Value entry with positions */ -interface IKeyValueEntry { - key: string; - value: IVariant; - key_position: IPosition; - value_position: IPosition; -}; - -class KeyValueEntry implements IKeyValueEntry { - key: string; - value: IVariant; - key_position: IPosition; - value_position: IPosition; - - constructor(k: string, pos: IPosition) { - this.key = k; - this.key_position = pos; - } -} - -class Variant implements IVariant { - constructor(public type: ValueType, public object: any) {} -} - -/* String value with position */ -interface IPositionedString { - value: string; - position: IPosition; -} - -/* Dependency specification */ -interface IDependency { - name: IPositionedString; - version: IPositionedString; -} - -/* Dependency collector interface */ -interface IDependencyCollector { - classes: Array; - collect(contents: string): Promise>; -} - -/* Dependency class that can be created from `IKeyValueEntry` */ -class Dependency implements IDependency { - name: IPositionedString; - version: IPositionedString; - constructor(dependency: IKeyValueEntry) { - this.name = { - value: dependency.key, - position: dependency.key_position - }; - this.version = { - value: dependency.value.object, - position: dependency.value_position - }; - } -} - -export { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IPositionedString, IDependencyCollector, Dependency }; diff --git a/src/utils.ts b/src/utils.ts index 355b1dee..a2483dc2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -4,9 +4,8 @@ * ------------------------------------------------------------------------------------------ */ 'use strict'; import { Stream, Readable } from 'stream'; -import { IPosition } from './types'; -import { IPositionedString } from './collector'; import { Position, Range } from 'vscode-languageserver'; +import { IPositionedString, IPosition } from './collector'; import { config } from './config'; export let stream_from_string = (s: string): Stream => { diff --git a/test/gomod.collector.test.ts b/test/collector/go.mod.test.ts similarity index 99% rename from test/gomod.collector.test.ts rename to test/collector/go.mod.test.ts index a22ea7cc..ee2f371b 100644 --- a/test/gomod.collector.test.ts +++ b/test/collector/go.mod.test.ts @@ -1,12 +1,12 @@ import { expect } from 'chai'; -import { GomodDependencyCollector } from '../src/collector'; -import { getGoLangImportsCmd } from '../src/utils'; +import { DependencyCollector } from '../../src/collector/go.mod'; +import { getGoLangImportsCmd } from '../../src/utils'; const fake = require('fake-exec'); describe('Golang go.mod parser test', () => { const fakeSourceRoot = "/tmp/fake/path/to/goproject/source"; - const collector: GomodDependencyCollector = new GomodDependencyCollector(fakeSourceRoot); + const collector = new DependencyCollector(fakeSourceRoot); it('tests valid go.mod', async () => { fake(getGoLangImportsCmd(), `github.com/alecthomas/units diff --git a/test/npm.collector.test.ts b/test/collector/package.json.test.ts similarity index 77% rename from test/npm.collector.test.ts rename to test/collector/package.json.test.ts index 4e9dc3da..ab674366 100644 --- a/test/npm.collector.test.ts +++ b/test/collector/package.json.test.ts @@ -1,8 +1,8 @@ import { expect } from 'chai'; -import { PackageJsonCollector } from '../src/collector'; +import { DependencyCollector } from '../../src/collector/package.json'; describe('npm package.json parser test', () => { - const collector = new PackageJsonCollector(); + const collector = new DependencyCollector(); it('tests empty package.json', async () => { const deps = await collector.collect(` @@ -28,15 +28,14 @@ describe('npm package.json parser test', () => { "hello": "1.0" } }`); - expect(deps.length).equal(1); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: "hello", position: {line: 4, column: 14}}, version: {value: "1.0", position: {line: 4, column: 23}} - }); + }]); }); it('tests single dependency as devDependencies', async () => { - let collector = new PackageJsonCollector(["devDependencies"]); + let collector = new DependencyCollector(["devDependencies"]); let deps = await collector.collect(`{ "devDependencies": { "hello": "1.0" @@ -45,13 +44,12 @@ describe('npm package.json parser test', () => { "foo": "10.1.1" } }`); - expect(deps.length).equal(1); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: "hello", position: {line: 3, column: 14}}, version: {value: "1.0", position: {line: 3, column: 23}} - }); + }]); - collector = new PackageJsonCollector(["devDependencies", "dependencies"]); + collector = new DependencyCollector(["devDependencies", "dependencies"]); deps = await collector.collect(`{ "devDependencies": { "hello": "1.0" @@ -60,15 +58,13 @@ describe('npm package.json parser test', () => { "foo": "10.1.1" } }`); - expect(deps.length).equal(2); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: "hello", position: {line: 3, column: 14}}, version: {value: "1.0", position: {line: 3, column: 23}} - }); - expect(deps[1]).is.eql({ + },{ name: {value: "foo", position: {line: 6, column: 14}}, version: {value: "10.1.1", position: {line: 6, column: 21}} - }); + }]); }); @@ -80,11 +76,10 @@ describe('npm package.json parser test', () => { "1.0" } }`); - expect(deps.length).equal(1); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: "hello", position: {line: 4, column: 14}}, version: {value: "1.0", position: {line: 5, column: 16}} - }); + }]); }); it('tests 3 dependencies with spaces', async () => { @@ -100,18 +95,15 @@ describe('npm package.json parser test', () => { " 10.0.1" } }`); - expect(deps.length).equal(3); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: "hello", position: {line: 4, column: 13}}, version: {value: "1.0", position: {line: 4, column: 37}} - }); - expect(deps[1]).is.eql({ + },{ name: {value: "world", position: {line: 5, column: 16}}, version: {value: "^1.0", position: {line: 5, column: 24}} - }); - expect(deps[2]).is.eql({ + },{ name: {value: "foo", position: {line: 8, column: 10}}, version: {value: " 10.0.1", position: {line: 10, column: 12}} - }); + }]); }); }); diff --git a/test/maven.collector.test.ts b/test/collector/pom.xml.test.ts similarity index 98% rename from test/maven.collector.test.ts rename to test/collector/pom.xml.test.ts index 4a455282..92e61fdd 100644 --- a/test/maven.collector.test.ts +++ b/test/collector/pom.xml.test.ts @@ -1,9 +1,9 @@ import { expect } from 'chai'; -import { PomXmlDependencyCollector } from '../src/maven.collector'; +import { DependencyCollector } from '../../src/collector/pom.xml'; import parse = require("@xml-tools/parser"); describe('Maven pom.xml parser test', () => { - const collector:PomXmlDependencyCollector = new PomXmlDependencyCollector(); + const collector = new DependencyCollector(); it('tests valid pom.xml', async () => { const deps = await collector.collect( diff --git a/test/pypi.collector.test.ts b/test/collector/requirements.txt.test.ts similarity index 75% rename from test/pypi.collector.test.ts rename to test/collector/requirements.txt.test.ts index b83ad47b..228fa32a 100644 --- a/test/pypi.collector.test.ts +++ b/test/collector/requirements.txt.test.ts @@ -1,8 +1,8 @@ import { expect } from 'chai'; -import { ReqDependencyCollector } from '../src/collector'; +import { DependencyCollector } from '../../src/collector/requirements.txt'; describe('PyPi requirements.txt parser test', () => { - const collector:ReqDependencyCollector = new ReqDependencyCollector(); + const collector = new DependencyCollector(); it('tests valid requirements.txt', async () => { const deps = await collector.collect(`a==1 @@ -10,23 +10,19 @@ describe('PyPi requirements.txt parser test', () => { c>=10.1 d<=20.1.2.3.4.5.6.7.8 `); - expect(deps.length).equal(4); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: 'a', position: {line: 0, column: 0}}, version: {value: '1', position: {line: 1, column: 4}} - }); - expect(deps[1]).is.eql({ + },{ name: {value: 'b', position: {line: 0, column: 0}}, version: {value: '2.1.1', position: {line: 2, column: 16}} - }); - expect(deps[2]).is.eql({ + },{ name: {value: 'c', position: {line: 0, column: 0}}, version: {value: '10.1', position: {line: 3, column: 16}} - }); - expect(deps[3]).is.eql({ + },{ name: {value: 'd', position: {line: 0, column: 0}}, version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 4, column: 16}} - }); + }]); }); it('tests requirements.txt with comments', async () => { @@ -37,19 +33,16 @@ describe('PyPi requirements.txt parser test', () => { d<=20.1.2.3.4.5.6.7.8 # done `); - expect(deps.length).equal(3); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: 'a', position: {line: 0, column: 0}}, version: {value: '1', position: {line: 2, column: 16}} - }); - expect(deps[1]).is.eql({ + },{ name: {value: 'c', position: {line: 0, column: 0}}, version: {value: '', position: {line: 4, column: 1}} // column shouldn't matter for empty versions - }); - expect(deps[2]).is.eql({ + },{ name: {value: 'd', position: {line: 0, column: 0}}, version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 5, column: 16}} - }); + }]); }); it('tests empty lines', async () => { @@ -58,11 +51,10 @@ describe('PyPi requirements.txt parser test', () => { a==1 `); - expect(deps.length).equal(1); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: 'a', position: {line: 0, column: 0}}, version: {value: '1', position: {line: 3, column: 16}} - }); + }]); }); it('tests deps with spaces before and after comparators', async () => { @@ -72,14 +64,12 @@ describe('PyPi requirements.txt parser test', () => { b <= 10.1 `); - expect(deps.length).equal(2); - expect(deps[0]).is.eql({ + expect(deps).is.eql([{ name: {value: 'a', position: {line: 0, column: 0}}, version: {value: '1', position: {line: 2, column: 24}} - }); - expect(deps[1]).is.eql({ + },{ name: {value: 'b', position: {line: 0, column: 0}}, version: {value: '10.1', position: {line: 4, column: 35}} - }); + }]); }); }); diff --git a/test/consumer.test.ts b/test/consumer.test.ts index 5b5285a3..9ff31321 100644 --- a/test/consumer.test.ts +++ b/test/consumer.test.ts @@ -5,9 +5,9 @@ import { NoopVulnerabilityAggregator, GolangVulnerabilityAggregator } from '../s const config = {}; const diagnosticFilePath = "a/b/c/d"; const dependency = { - "name": { + "name": { "value" : "abc", - "position": { + "position": { "line": 20, "column": 6 } @@ -22,7 +22,6 @@ const dependency = { } describe('Response consumer test', () => { - it('Consume response for free-users', () => { let DiagnosticsEngines = [SecurityEngine]; let diagnostics = [];