fix(pypi): Dependency count shown in stack analysis report is incorrect

- Add unit test for pypi parser

TODO: Similar exercise has to be repeated for npm and maven

cico_run_tests.sh

@@ -7,3 +7,5 @@ set -ex
 install_dependencies
 
 build_project
+
+run_unit_tests

cico_setup.sh

@@ -51,6 +51,19 @@ build_project() {
   fi
 }
 
+run_unit_tests() {
+    # Exec unit tests
+    npm test
+
+    if [ $? -eq 0 ]; then
+        echo 'CICO: unit tests OK'
+    else
+        echo 'CICO: unit tests FAIL'
+        exit 2
+    fi
+}
+
+
 . cico_release.sh
 
 load_jenkins_vars

package.json

@@ -37,17 +37,41 @@
     "xml2object": "0.1.2"
   },
   "devDependencies": {
-    "@types/node": "^6.0.52",
     "@krux/condition-jenkins": "1.0.1",
+    "@types/chai": "^4.1.7",
+    "@types/mocha": "^5.2.7",
+    "@types/node": "^6.0.52",
+    "chai": "^4.2.0",
+    "mocha": "^6.2.0",
+    "nyc": "^14.1.1",
     "semantic-release": "8.2.0",
-    "typescript": "^2.1.4"
+    "ts-node": "^8.3.0",
+    "typescript": "^2.9.2"
   },
   "scripts": {
     "build": "npm run clean && node node_modules/typescript/bin/tsc -p . && cp LICENSE package.json README.md output && npm run dist",
     "clean": "rm -Rf ca-lsp-server.tar output/",
+    "test": "nyc mocha",
     "dist": "cp -r node_modules output/ && cp ./package.json output/ && node -p -e \"require('./package.json').version\" > output/VERSION && rm -rf output/node_modules/typescript/ && tar cvjf ca-lsp-server.tar -C output/ .",
     "semantic-release": "semantic-release pre && npm run build && cp -r .git output && npm publish output/ && semantic-release post"
   },
+  "nyc": {
+    "include": [
+      "src/**/*.ts"
+    ],
+    "extension": [
+      ".ts"
+    ],
+    "require": [
+      "ts-node/register"
+    ],
+    "reporter": [
+      "text",
+      "html"
+    ],
+    "sourceMap": true,
+    "instrument": true
+  },
   "release": {
     "branch": "master",
     "debug": false,

src/collector.ts

@@ -72,75 +72,54 @@ class DependencyCollector implements IDependencyCollector {
 }
 
 class NaivePyParser {
-    constructor(objSam: any) {
-        this.objSam = objSam;
-        this.parser = this.createPyParser()
+    constructor(contents: string) {
+        this.dependencies = NaivePyParser.parseDependencies(contents);
     }
 
-    objSam: any;
-    parser: any;
-    dependencies: Array<IDependency> = [];
-    isDependency: boolean = false;
+    dependencies: Array<IDependency>;
 
-    createPyParser(): any {
-        let deps = this.dependencies;
-        this.objSam.forEach(function(obj) {
-            let entry: IKeyValueEntry = new KeyValueEntry(obj["pkgName"], {line: 0, column: 0});
-                entry.value = new Variant(ValueType.String, obj["version"]);
-                entry.value_position = {line: obj["line"], column: obj["column"]};
-                let dep: IDependency = new Dependency(entry);
-                deps.push(dep);
-        });
+    static parseDependencies(contents:string): Array<IDependency> {
+        const requirements = contents.split("\n");
+        return requirements.reduce((dependencies, req, index) => {
+            // skip any text after #
+            if (req.includes('#')) {
+                req = req.split('#')[0];
+            }
+            const parsedRequirement: Array<string>  = req.split(/[==,>=,<=]+/);
+            const pkgName:string = (parsedRequirement[0] || '').trim();
+            // skip empty lines
+            if (pkgName.length > 0) {
+                const version = (parsedRequirement[1] || '').trim();
+                const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 });
+                entry.value = new Variant(ValueType.String, version);
+                entry.value_position = { line: index + 1, column: req.indexOf(version) + 1 };
+                dependencies.push(new Dependency(entry));
+            }
+            return dependencies;
+        }, []);
     }
 
-     parse(): Array<IDependency> {
+    parse(): Array<IDependency> {
         return this.dependencies;
     }
 }
 
-let toObject = (arr:any) => {
-  let rv: Array<string> =  [];
-  for (let i:number = 0; i < arr.length; ++i){
-    if (arr[i] !== undefined){
-        // let line: string = arr[i].replace(/\s/g,'');
-        let line: string = arr[i];
-        let lineArr: any;
-        let lineStr: string;
-         if(line.indexOf('#')!== -1){
-            lineArr = line.split("#");
-            lineStr = lineArr[0];
-         }else{
-            lineStr = line;
-         }
-         let subArr: Array<string>  = lineStr.split(/[==,>=,<=]+/);
-         let subObj:any = {};
-         subObj["pkgName"] = subArr[0];
-         subObj["version"] = subArr[1] || "";
-         subObj["line"] = i+1;
-         subObj["column"] = subArr[0].length +3;
-		 rv.push(subObj);
-    }
-  }
-  return rv;
-}
 /* Process entries found in the txt files and collect all dependency
  * related information */
 class ReqDependencyCollector {
     constructor(public classes: Array<string> = ["dependencies"]) {}
 
     async collect(contents: string): Promise<Array<IDependency>> {
-        let tempArr = contents.split("\n");
-        let objSam = toObject(tempArr);
-        let parser = new NaivePyParser(objSam);
-        let dependencies: Array<IDependency> = parser.parse();
-        return dependencies;
+        let parser = new NaivePyParser(contents);
+        return parser.parse();
     }
+
 }
 
 class NaivePomXmlSaxParser {
     constructor(stream: Stream) {
         this.stream = stream;
-        this.parser = this.createParser()
+        this.parser = this.createParser();
     }
 
     stream: Stream;

test/mocha.opts

@@ -0,0 +1,4 @@
+--require ts-node/register
+--full-trace
+--bail
+test/**/*.test.ts

test/pypi.collector.test.ts

@@ -0,0 +1,85 @@
+import { expect } from 'chai';
+import { ReqDependencyCollector } from '../src/collector';
+
+describe('PyPi requirements.txt parser test', () => {
+    const collector:ReqDependencyCollector = new ReqDependencyCollector();
+
+    it('tests valid requirements.txt', async () => {
+        const deps = await collector.collect(`a==1
+            b==2.1.1
+            c>=10.1
+            d<=20.1.2.3.4.5.6.7.8
+        `);
+        expect(deps.length).equal(4);
+        expect(deps[0]).is.eql({
+          name: {value: 'a', position: {line: 0, column: 0}},
+          version: {value: '1', position: {line: 1, column: 4}}
+        });
+        expect(deps[1]).is.eql({
+          name: {value: 'b', position: {line: 0, column: 0}},
+          version: {value: '2.1.1', position: {line: 2, column: 16}}
+        });
+        expect(deps[2]).is.eql({
+          name: {value: 'c', position: {line: 0, column: 0}},
+          version: {value: '10.1', position: {line: 3, column: 16}}
+        });
+        expect(deps[3]).is.eql({
+          name: {value: 'd', position: {line: 0, column: 0}},
+          version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 4, column: 16}}
+        });
+    });
+
+    it('tests requirements.txt with comments', async () => {
+        const deps = await collector.collect(`# hello world
+            a==1 # hello
+            # another comment b==2.1.1
+            c # yet another comment >=10.1
+            d<=20.1.2.3.4.5.6.7.8
+            # done
+        `);
+        expect(deps.length).equal(3);
+        expect(deps[0]).is.eql({
+          name: {value: 'a', position: {line: 0, column: 0}},
+          version: {value: '1', position: {line: 2, column: 16}}
+        });
+        expect(deps[1]).is.eql({
+          name: {value: 'c', position: {line: 0, column: 0}},
+          version: {value: '', position: {line: 4, column: 1}} // column shouldn't matter for empty versions
+        });
+        expect(deps[2]).is.eql({
+          name: {value: 'd', position: {line: 0, column: 0}},
+          version: {value: '20.1.2.3.4.5.6.7.8', position: {line: 5, column: 16}}
+        });
+    });
+
+    it('tests empty lines', async () => {
+        const deps = await collector.collect(`
+
+            a==1
+
+        `);
+        expect(deps.length).equal(1);
+        expect(deps[0]).is.eql({
+          name: {value: 'a', position: {line: 0, column: 0}},
+          version: {value: '1', position: {line: 3, column: 16}}
+        });
+    });
+
+    it('tests deps with spaces before and after comparators', async () => {
+        const deps = await collector.collect(`
+            a        ==1               
+
+                  b        <=     10.1               
+
+        `);
+        expect(deps.length).equal(2);
+        expect(deps[0]).is.eql({
+          name: {value: 'a', position: {line: 0, column: 0}},
+          version: {value: '1', position: {line: 2, column: 24}}
+        });
+        expect(deps[1]).is.eql({
+          name: {value: 'b', position: {line: 0, column: 0}},
+          version: {value: '10.1', position: {line: 4, column: 35}}
+        });
+    });
+});

tsconfig.json

@@ -1,12 +1,13 @@
-{
-	"compilerOptions": {
-		"target": "ES6",
-		"module": "commonjs",
-		"moduleResolution": "node",
-		"sourceMap": true,
-		"outDir": "output"
-	},
-	"exclude": [
-		"node_modules"
-	]
-}
+{
+  "compilerOptions": {
+    "target": "ES6",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "outDir": "output"
+  },
+  "exclude": [
+    "node_modules",
+    "test"
+  ]
+}