feat: [APPAI-1524] Golang CA support from LSP (#148)

* [APPAI-1484] Support for single line require and version string parsing using regex

* [APPAI-1524] Golang CA for modules and packages

* Removed API server directing to local server

* Added unit test case for multiple package use-case

* No more saving data of command to a file, review comments implemented

* Used join for imports to simplify looping

* Optimized import reading and made it async

* Review comments, added test case for module + import testing

* Created package aggregator to merge module and package data

* Created noop aggregator, mock exec for test cases, added semicolon

* Moved package and aggregator into seperate file, added unit test to cover 100% code

* Some more semi-colons missing

* Some more semicolons, removed unwanted changes, foreach for loop

* Removed ecosystem from base class

* Renamed package to vulnerability as per review comment

* Removed unsed class, moved aggregator to pipline from producer

package.json

@@ -34,7 +34,8 @@
     "node-fetch": "^2.6.0",
     "vscode-languageserver": "^5.3.0-next.9",
     "winston": "3.2.1",
-    "xml2object": "0.1.2"
+    "xml2object": "0.1.2",
+    "compare-versions": "3.6.0"
   },
   "devDependencies": {
     "@semantic-release/exec": "^5.0.0",
@@ -47,7 +48,8 @@
     "nyc": "^14.1.1",
     "semantic-release": "^17.1.0",
     "ts-node": "^8.3.0",
-    "typescript": "^3.6.3"
+    "typescript": "^3.6.3",
+    "fake-exec": "^1.1.0"
   },
   "scripts": {
     "build": "tsc -p . && cp package.json LICENSE README.md output",

src/aggregators.ts

@@ -0,0 +1,99 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Red Hat, Dharmendra Patel 2020
+ * Licensed under the Apache-2.0 License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+'use strict';
+import { Vulnerability } from './vulnerability';
+import compareVersions = require('compare-versions');
+
+const severity = ["low", "medium", "high", "critical"];
+
+/* VulnerabilityAggregator */
+interface VulnerabilityAggregator {
+    isNewVulnerability: boolean;
+    aggregate(newVulnerability: Vulnerability): Vulnerability;
+}
+
+/* Noop Vulnerability aggregator class */
+class NoopVulnerabilityAggregator implements VulnerabilityAggregator {
+    isNewVulnerability: boolean;
+
+    aggregate(newVulnerability: Vulnerability): Vulnerability {
+        // Make it a new vulnerability always and set ecosystem for vulnerability.
+        this.isNewVulnerability = true;
+        newVulnerability.ecosystem = "";
+
+        return newVulnerability;
+    }
+}
+
+/* Golang Vulnerability aggregator class */
+class GolangVulnerabilityAggregator implements VulnerabilityAggregator {
+    vulnerabilities: Array<Vulnerability> = Array<Vulnerability>();
+    isNewVulnerability: boolean;
+
+    aggregate(newVulnerability: Vulnerability): Vulnerability {
+        // Set ecosystem for new vulnerability from aggregator
+        newVulnerability.ecosystem = "golang";
+
+        // Check if module / package exists in the list.
+        this.isNewVulnerability = true;
+
+        var existingVulnerabilityIndex = 0
+        this.vulnerabilities.forEach((pckg, index) => {
+            // Module and package can come in any order due to parallel batch requests.
+            // Need handle use case (1) Module first followed by package and (2) Vulnerability first followed by module.
+            if (newVulnerability.name.startsWith(pckg.name + "/") || pckg.name.startsWith(newVulnerability.name + "/")) {
+                // Module / package exists, so aggregate the data and update Diagnostic message and code action.
+                this.mergeVulnerability(index, newVulnerability);
+                this.isNewVulnerability = false;
+                existingVulnerabilityIndex = index;
+            }
+        });
+
+        if (this.isNewVulnerability) {
+            this.vulnerabilities.push(newVulnerability);
+            return newVulnerability;
+        }
+
+        return this.vulnerabilities[existingVulnerabilityIndex];
+    }
+
+    private mergeVulnerability(existingIndex: number, newVulnerability: Vulnerability) {
+        // Between current name and new name, smallest will be the module name.
+        // So, assign the smallest as package name.
+        if (newVulnerability.name.length < this.vulnerabilities[existingIndex].name.length)
+            this.vulnerabilities[existingIndex].name = newVulnerability.name;
+
+        // Merge other informations
+        this.vulnerabilities[existingIndex].packageCount += newVulnerability.packageCount;
+        this.vulnerabilities[existingIndex].vulnerabilityCount += newVulnerability.vulnerabilityCount;
+        this.vulnerabilities[existingIndex].advisoryCount += newVulnerability.advisoryCount;
+        this.vulnerabilities[existingIndex].exploitCount += newVulnerability.exploitCount;
+        this.vulnerabilities[existingIndex].highestSeverity = this.getMaxSeverity(
+            this.vulnerabilities[existingIndex].highestSeverity, newVulnerability.highestSeverity);
+        this.vulnerabilities[existingIndex].recommendedVersion = this.getMaxRecVersion(
+            this.vulnerabilities[existingIndex].recommendedVersion, newVulnerability.recommendedVersion);
+    }
+
+    private getMaxSeverity(oldSeverity: string, newSeverity: string): string {
+        const newSeverityIndex = Math.max(severity.indexOf(oldSeverity), severity.indexOf(newSeverity));
+        return severity[newSeverityIndex];
+    }
+
+    private getMaxRecVersion(oldRecVersion: string, newRecVersion: string): string {
+        // Compute maximium recommended version.
+        var maxRecVersion = oldRecVersion;
+        if (oldRecVersion == "" || oldRecVersion == null) {
+            maxRecVersion = newRecVersion;
+        } else if (newRecVersion != "" && newRecVersion != null) {
+            if (compareVersions(oldRecVersion, newRecVersion) == -1) {
+                maxRecVersion = newRecVersion;
+            }
+        }
+
+        return maxRecVersion;
+    }
+}
+
+export { VulnerabilityAggregator, NoopVulnerabilityAggregator, GolangVulnerabilityAggregator };

src/collector.ts

@@ -8,6 +8,7 @@ import * as Xml2Object from 'xml2object';
 import * as jsonAst from 'json-to-ast';
 import { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType } from './types';
 import { stream_from_string } from './utils';
+import { exec } from 'child_process';
 
 /* Please note :: There was issue with semverRegex usage in the code. During run time, it extracts 
  * version with 'v' prefix, but this is not be behavior of semver in CLI and test environment. 
@@ -96,15 +97,14 @@ class ReqDependencyCollector implements IDependencyCollector {
 }
 
 class NaiveGomodParser {
-    constructor(contents: string) {
-        this.dependencies = NaiveGomodParser.parseDependencies(contents);
+    constructor(contents: string, goImports: Set<string>) {
+        this.dependencies = NaiveGomodParser.parseDependencies(contents, goImports);
     }
 
     dependencies: Array<IDependency>;
 
-    static parseDependencies(contents:string): Array<IDependency> {
-        const gomod = contents.split("\n");
-        return gomod.reduce((dependencies, line, index) => {
+    static parseDependencies(contents:string, goImports: Set<string>): Array<IDependency> {
+        return contents.split("\n").reduce((dependencies, line, index) => {
             // Ignore "replace" lines
             if (!line.includes("=>")) {
                 // skip any text after '//'
@@ -115,16 +115,26 @@ class NaiveGomodParser {
                 //const version = semverRegex().exec(line)
                 regExp.lastIndex = 0;
                 const version = regExp.exec(line);
-      	        // Skip lines without version string
+                // Skip lines without version string
                 if (version && version.length > 0) {
-                    const parts: Array<string>  = line.replace('require', '').replace('(', '').replace(')', '').trim().split(' ');
-                    const pkgName:string = (parts[0] || '').trim();
+                    const parts: Array<string> = line.replace('require', '').replace('(', '').replace(')', '').trim().split(' ');
+                    const pkgName: string = (parts[0] || '').trim();
                     // Ignore line starting with replace clause and empty package
                     if (pkgName.length > 0) {
                         const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 });
                         entry.value = new Variant(ValueType.String, 'v' + version[0]);
                         entry.value_position = { line: index + 1, column: version.index };
                         dependencies.push(new Dependency(entry));
+
+                        // Find packages of this module.
+                        goImports.forEach(pckg => {
+                            if (pckg != pkgName && pckg.startsWith(pkgName + "/")) {
+                                const entry: IKeyValueEntry = new KeyValueEntry(pckg, { line: 0, column: 0 });
+                                entry.value = new Variant(ValueType.String, 'v' + version[0]);
+                                entry.value_position = { line: index + 1, column: version.index };
+                                dependencies.push(new Dependency(entry));
+                            }
+                        })
                     }
                 }
             }
@@ -140,10 +150,24 @@ class NaiveGomodParser {
 /* Process entries found in the go.mod file and collect all dependency
  * related information */
 class GomodDependencyCollector implements IDependencyCollector {
-    constructor(public classes: Array<string> = ["dependencies"]) {}
+    constructor(private manifestFile: string, public classes: Array<string> = ["dependencies"]) {
+        this.manifestFile = manifestFile;
+    }
 
     async collect(contents: string): Promise<Array<IDependency>> {
-        let parser = new NaiveGomodParser(contents);
+        let promiseExec = new Promise<Set<string>>((resolve, reject) => {
+            const vscodeRootpath = this.manifestFile.replace("file://", "").replace("/go.mod", "")
+            exec(`go list -f '{{ join .Imports "\\n" }}' ./...`,
+                   { cwd: vscodeRootpath, maxBuffer: 1024 * 1200 }, (error, stdout, stderr) => {
+                if (error) {
+                    reject(`'go list' command failed with error :: ${stderr}`);
+                } else {
+                    resolve(new Set(stdout.toString().split("\n")));
+                }
+            });
+        });
+        const goImports: Set<string> = await promiseExec;
+        let parser = new NaiveGomodParser(contents, goImports);
         return parser.parse();
     }