From f42291cefe7c0f077b2020fa3ef444a8a050d2e7 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Mon, 29 Jul 2019 17:28:03 +0530 Subject: [PATCH] fix(pypi): Dependency count shown in stack analysis report is incorrect - Add unit test for pypi parser TODO: Similar exercise has to be repeated for npm and maven --- cico_run_tests.sh | 2 + cico_setup.sh | 13 ++++++ package.json | 28 +++++++++++- src/collector.ts | 75 ++++++++++++-------------------- test/mocha.opts | 4 ++ test/pypi.collector.test.ts | 85 +++++++++++++++++++++++++++++++++++++ tsconfig.json | 25 +++++------ 7 files changed, 170 insertions(+), 62 deletions(-) create mode 100644 test/mocha.opts create mode 100644 test/pypi.collector.test.ts diff --git a/cico_run_tests.sh b/cico_run_tests.sh index 35c0967f..6065a4a5 100755 --- a/cico_run_tests.sh +++ b/cico_run_tests.sh @@ -7,3 +7,5 @@ set -ex install_dependencies build_project + +run_unit_tests diff --git a/cico_setup.sh b/cico_setup.sh index fb8db2b6..ece1b2d9 100644 --- a/cico_setup.sh +++ b/cico_setup.sh @@ -51,6 +51,19 @@ build_project() { fi } +run_unit_tests() { + # Exec unit tests + npm test + + if [ $? -eq 0 ]; then + echo 'CICO: unit tests OK' + else + echo 'CICO: unit tests FAIL' + exit 2 + fi +} + + . cico_release.sh load_jenkins_vars diff --git a/package.json b/package.json index 593eb9f9..ce7025fd 100644 --- a/package.json +++ b/package.json @@ -37,17 +37,41 @@ "xml2object": "0.1.2" }, "devDependencies": { - "@types/node": "^6.0.52", "@krux/condition-jenkins": "1.0.1", + "@types/chai": "^4.1.7", + "@types/mocha": "^5.2.7", + "@types/node": "^6.0.52", + "chai": "^4.2.0", + "mocha": "^6.2.0", + "nyc": "^14.1.1", "semantic-release": "8.2.0", - "typescript": "^2.1.4" + "ts-node": "^8.3.0", + "typescript": "^2.9.2" }, "scripts": { "build": "npm run clean && node node_modules/typescript/bin/tsc -p . && cp LICENSE package.json README.md output && npm run dist", "clean": "rm -Rf ca-lsp-server.tar output/", + "test": "nyc mocha", "dist": "cp -r node_modules output/ && cp ./package.json output/ && node -p -e \"require('./package.json').version\" > output/VERSION && rm -rf output/node_modules/typescript/ && tar cvjf ca-lsp-server.tar -C output/ .", "semantic-release": "semantic-release pre && npm run build && cp -r .git output && npm publish output/ && semantic-release post" }, + "nyc": { + "include": [ + "src/**/*.ts" + ], + "extension": [ + ".ts" + ], + "require": [ + "ts-node/register" + ], + "reporter": [ + "text", + "html" + ], + "sourceMap": true, + "instrument": true + }, "release": { "branch": "master", "debug": false, diff --git a/src/collector.ts b/src/collector.ts index 2fd1029e..6a4045ac 100644 --- a/src/collector.ts +++ b/src/collector.ts @@ -72,75 +72,54 @@ class DependencyCollector implements IDependencyCollector { } class NaivePyParser { - constructor(objSam: any) { - this.objSam = objSam; - this.parser = this.createPyParser() + constructor(contents: string) { + this.dependencies = NaivePyParser.parseDependencies(contents); } - objSam: any; - parser: any; - dependencies: Array = []; - isDependency: boolean = false; + dependencies: Array; - createPyParser(): any { - let deps = this.dependencies; - this.objSam.forEach(function(obj) { - let entry: IKeyValueEntry = new KeyValueEntry(obj["pkgName"], {line: 0, column: 0}); - entry.value = new Variant(ValueType.String, obj["version"]); - entry.value_position = {line: obj["line"], column: obj["column"]}; - let dep: IDependency = new Dependency(entry); - deps.push(dep); - }); + static parseDependencies(contents:string): Array { + const requirements = contents.split("\n"); + return requirements.reduce((dependencies, req, index) => { + // skip any text after # + if (req.includes('#')) { + req = req.split('#')[0]; + } + const parsedRequirement: Array = req.split(/[==,>=,<=]+/); + const pkgName:string = (parsedRequirement[0] || '').trim(); + // skip empty lines + if (pkgName.length > 0) { + const version = (parsedRequirement[1] || '').trim(); + const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 }); + entry.value = new Variant(ValueType.String, version); + entry.value_position = { line: index + 1, column: req.indexOf(version) + 1 }; + dependencies.push(new Dependency(entry)); + } + return dependencies; + }, []); } - parse(): Array { + parse(): Array { return this.dependencies; } } -let toObject = (arr:any) => { - let rv: Array = []; - for (let i:number = 0; i < arr.length; ++i){ - if (arr[i] !== undefined){ - // let line: string = arr[i].replace(/\s/g,''); - let line: string = arr[i]; - let lineArr: any; - let lineStr: string; - if(line.indexOf('#')!== -1){ - lineArr = line.split("#"); - lineStr = lineArr[0]; - }else{ - lineStr = line; - } - let subArr: Array = lineStr.split(/[==,>=,<=]+/); - let subObj:any = {}; - subObj["pkgName"] = subArr[0]; - subObj["version"] = subArr[1] || ""; - subObj["line"] = i+1; - subObj["column"] = subArr[0].length +3; - rv.push(subObj); - } - } - return rv; -} /* Process entries found in the txt files and collect all dependency * related information */ class ReqDependencyCollector { constructor(public classes: Array = ["dependencies"]) {} async collect(contents: string): Promise> { - let tempArr = contents.split("\n"); - let objSam = toObject(tempArr); - let parser = new NaivePyParser(objSam); - let dependencies: Array = parser.parse(); - return dependencies; + let parser = new NaivePyParser(contents); + return parser.parse(); } + } class NaivePomXmlSaxParser { constructor(stream: Stream) { this.stream = stream; - this.parser = this.createParser() + this.parser = this.createParser(); } stream: Stream; diff --git a/test/mocha.opts b/test/mocha.opts new file mode 100644 index 00000000..9e3da4e4 --- /dev/null +++ b/test/mocha.opts @@ -0,0 +1,4 @@ +--require ts-node/register +--full-trace +--bail +test/**/*.test.ts diff --git a/test/pypi.collector.test.ts b/test/pypi.collector.test.ts new file mode 100644 index 00000000..b83ad47b --- /dev/null +++ b/test/pypi.collector.test.ts @@ -0,0 +1,85 @@ +import { expect } from 'chai'; +import { ReqDependencyCollector } from '../src/collector'; + +describe('PyPi requirements.txt parser test', () => { + const collector:ReqDependencyCollector = new ReqDependencyCollector(); + + it('tests valid requirements.txt', async () => { + const deps = await collector.collect(`a==1 + b==2.1.1 + c>=10.1 + d<=20.1.2.3.4.5.6.7.8 + `); + expect(deps.length).equal(4); + expect(deps[0]).is.eql({ + name: {value: 'a', position: {line: 0, column: 0}}, + version: {value: '1', position: {line: 1, column: 4}} + }); + expect(deps[1]).is.eql({ + name: {value: 'b', position: {line: 0, column: 0}}, + version: {value: '2.1.1', position: {line: 2, column: 16}} + }); + expect(deps[2]).is.eql({ + name: {value: 'c', position: {line: 0, column: 0}}, + version: {value: '10.1', position: {line: 3, column: 16}} + }); + expect(deps[3]).is.eql({ + name: {value: 'd', position: {line: 0, column: 0}}, + version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 4, column: 16}} + }); + }); + + it('tests requirements.txt with comments', async () => { + const deps = await collector.collect(`# hello world + a==1 # hello + # another comment b==2.1.1 + c # yet another comment >=10.1 + d<=20.1.2.3.4.5.6.7.8 + # done + `); + expect(deps.length).equal(3); + expect(deps[0]).is.eql({ + name: {value: 'a', position: {line: 0, column: 0}}, + version: {value: '1', position: {line: 2, column: 16}} + }); + expect(deps[1]).is.eql({ + name: {value: 'c', position: {line: 0, column: 0}}, + version: {value: '', position: {line: 4, column: 1}} // column shouldn't matter for empty versions + }); + expect(deps[2]).is.eql({ + name: {value: 'd', position: {line: 0, column: 0}}, + version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 5, column: 16}} + }); + }); + + it('tests empty lines', async () => { + const deps = await collector.collect(` + + a==1 + + `); + expect(deps.length).equal(1); + expect(deps[0]).is.eql({ + name: {value: 'a', position: {line: 0, column: 0}}, + version: {value: '1', position: {line: 3, column: 16}} + }); + }); + + it('tests deps with spaces before and after comparators', async () => { + const deps = await collector.collect(` + a ==1 + + b <= 10.1 + + `); + expect(deps.length).equal(2); + expect(deps[0]).is.eql({ + name: {value: 'a', position: {line: 0, column: 0}}, + version: {value: '1', position: {line: 2, column: 24}} + }); + expect(deps[1]).is.eql({ + name: {value: 'b', position: {line: 0, column: 0}}, + version: {value: '10.1', position: {line: 4, column: 35}} + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 1895c14e..39b701ce 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,12 +1,13 @@ -{ - "compilerOptions": { - "target": "ES6", - "module": "commonjs", - "moduleResolution": "node", - "sourceMap": true, - "outDir": "output" - }, - "exclude": [ - "node_modules" - ] -} +{ + "compilerOptions": { + "target": "ES6", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "outDir": "output" + }, + "exclude": [ + "node_modules", + "test" + ] +}