Skip to content

Commit

Permalink
chore: remove effective-pom generation (#212)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ilona Shishov <[email protected]>
  • Loading branch information
@IlonaShishov
IlonaShishov authored Sep 27, 2023
1 parent 3275aec commit 56975d1
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 289 deletions.
119 changes: 32 additions & 87 deletions src/providers/pom.xml.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,10 @@ import { VERSION_TEMPLATE } from '../utils';

export class DependencyProvider implements IDependencyProvider {
private xmlDocAst: XMLDocument;
private originalDeps: Array<XMLElement>;
ecosystem: string;

constructor(originalContents: string, enforceVersions: boolean, public classes: Array<string> = ['dependencies']) {
constructor(public classes: Array<string> = ['dependencies']) {
this.ecosystem = 'maven';
const { cst, tokenVector } = parse(originalContents);
const originalXmlDocAst = buildAst(cst as DocumentCstNode, tokenVector);
if (originalXmlDocAst.rootElement) {
this.originalDeps = this.getXMLDependencies(originalXmlDocAst, enforceVersions);
}
}

private findRootNodes(document: XMLDocument, rootElementName: string): Array<XMLElement> {
Expand All @@ -37,7 +31,7 @@ export class DependencyProvider implements IDependencyProvider {
this.xmlDocAst = buildAst(cst as DocumentCstNode, tokenVector);
}

private mapToDependency(dependenciesNode: XMLElement[]): Array<IDependency> {
private mapToDependency(deps: XMLElement[]): Array<IDependency> {
class PomDependency {
public element: XMLElement;
public groupId: XMLElement;
Expand All @@ -55,121 +49,72 @@ export class DependencyProvider implements IDependencyProvider {
return [this.groupId, this.artifactId].find(e => !e.textContents[0]?.text) === undefined;
}

isValidWithVersion(): boolean {
// none should have a empty text.
return [this.groupId, this.artifactId, this.version].find(e => !e.textContents[0]?.text) === undefined;
}

};

const toDependency = (resolved: PomDependency, original: PomDependency): Dependency => {
const toDependency = (d: PomDependency): Dependency => {
const dep: IKeyValueEntry = new KeyValueEntry(
`${original.groupId.textContents[0].text}/${original.artifactId.textContents[0].text}`,
{ line: original.element.position.startLine, column: original.element.position.startColumn }
`${d.groupId.textContents[0].text}/${d.artifactId.textContents[0].text}`,
{ line: d.element.position.startLine, column: d.element.position.startColumn }
);
dep.context_range = {
start: { line: original.element.position.startLine - 1, character: original.element.position.startColumn - 1 },
end: { line: original.element.position.endLine - 1, character: original.element.position.endColumn }
start: { line: d.element.position.startLine - 1, character: d.element.position.startColumn - 1 },
end: { line: d.element.position.endLine - 1, character: d.element.position.endColumn }
};
dep.value = new Variant(ValueType.String, resolved.version.textContents[0].text);

if (original.version) {
const versionVal = original.version.textContents[0];
if (d.version && d.version.textContents.length > 0) {
dep.value = new Variant(ValueType.String, d.version.textContents[0].text);
const versionVal = d.version.textContents[0];
dep.value_position = { line: versionVal.position.startLine, column: versionVal.position.startColumn };
} else {
dep.value = new Variant(ValueType.String, '');
dep.value_position = { line: 0, column: 0 };
dep.context = dependencyTemplate(original.element);
dep.context = dependencyTemplate(d.element);
}
return new Dependency(dep);
};

const dependencyTemplate = (original: XMLElement): string => {
const dependencyTemplate = (dep: XMLElement): string => {
let template = '<dependency>';
let hasVersion = false;
let idx = 0;
let margin = original.textContents[idx].text;
original.subElements.forEach(e => {
if (e.name === 'version') {
template += `${original.textContents[idx++].text}<${e.name}>${VERSION_TEMPLATE}</${e.name}>`;
} else {
template += `${original.textContents[idx++].text}<${e.name}>${e.textContents[0].text}</${e.name}>`;
let margin = dep.textContents[idx].text;
dep.subElements.forEach(e => {
if (e.name !== 'version') {
template += `${dep.textContents[idx++].text}<${e.name}>${e.textContents[0].text}</${e.name}>`;
}
});
if (!hasVersion) {
template += `${margin}<version>${VERSION_TEMPLATE}</version>`;
}
template += `${original.textContents[idx].text}</dependency>`;
template += `${margin}<version>${VERSION_TEMPLATE}</version>`;
template += `${dep.textContents[idx].text}</dependency>`;
return template;
};

const getMapKey = (element: PomDependency): string => {
return `${element.groupId.textContents[0].text}/${element.artifactId.textContents[0].text}`;
};
const purgeTestDeps = (nodes: XMLElement[]): Array<PomDependency> => nodes
// no test dependencies
.filter(e => !e.subElements.find(e => (e.name === 'scope' && e.textContents[0]?.text === 'test')))
.map(e => new PomDependency(e));

const buildDependencyMap = (original: Array<PomDependency>, resolved: Array<PomDependency>): Array<PomDependency> => {
let result = new Array<PomDependency>();
if (original) {
const visited = new Map<string, number>();
let resolvedIdx = 0;
original.forEach(o => {
const k = getMapKey(o);
let r = visited.get(k);
if(r === undefined) {
r = resolvedIdx++;
visited.set(k, r);
}
if(resolved[r] !== undefined) {
result.push(resolved[r]);
} else {
result.push(null);
}
});
}
return result;
};
const validDeps = purgeTestDeps(deps).filter(e => e.isValid());

if (this.originalDeps) {
const toPomDep = (nodes: XMLElement[]): Array<PomDependency> => nodes
// no test dependencies
.filter(e => !e.subElements.find(e => (e.name === 'scope' && e.textContents[0].text === 'test')))
.map(e => new PomDependency(e));

const resolvedDeps = toPomDep(dependenciesNode).filter(e => e.isValidWithVersion());
const origDeps = toPomDep(this.originalDeps).filter(e => e.isValid());

const resolvedMap = buildDependencyMap(origDeps, resolvedDeps);
const result = new Array();
origDeps.forEach((d, idx) => {
if(resolvedMap[idx] !== null) {
result.push(toDependency(resolvedMap[idx], d));
}
});
return result;
}
return new Array();
const result = new Array();
validDeps.forEach((d) => {
result.push(toDependency(d));
});
return result;
}

async collect(contents: string): Promise<Array<IDependency>> {
this.parseXml(contents);
const deps = this.getXMLDependencies(this.xmlDocAst, true);
const deps = this.getXMLDependencies(this.xmlDocAst);
return this.mapToDependency(deps);
}

private getXMLDependencies(doc: XMLDocument, enforceVersions: boolean): Array<XMLElement> {
private getXMLDependencies(doc: XMLDocument): Array<XMLElement> {
let validElementNames = ['groupId', 'artifactId'];
if(enforceVersions) {
validElementNames.push('version');
}

return this.findRootNodes(doc, 'dependencies')
//must not be a dependency under dependencyManagement
.filter(e => {
const parentElement = e.parent as XMLElement | undefined;

if (parentElement) {
return parentElement.name !== 'dependencyManagement';
}
return true;
return parentElement?.name !== 'dependencyManagement';
})
.map(node => node.subElements)
.flat(1)
Expand Down
54 changes: 8 additions & 46 deletions src/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ const fetchVulnerabilities = async (fileType: string, reqData: any) => {
'EXHORT_MVN_PATH': globalSettings.mvnExecutable,
'EXHORT_NPM_PATH': globalSettings.npmExecutable,
'EXHORT_GO_PATH': globalSettings.goExecutable,
'EXHORT_DEV_MODE': config.exhort_dev_mode,
'EXHORT_DEV_MODE': config.exhort_dev_mode
};
if (globalSettings.exhortSnykToken !== '') {
options['EXHORT_SNYK_TOKEN'] = globalSettings.exhortSnykToken;
Expand Down Expand Up @@ -233,7 +233,7 @@ const fetchVulnerabilities = async (fileType: string, reqData: any) => {
}
};

const sendDiagnostics = async (diagnosticFilePath: string, originalContents: string, effectiveContents: string, provider: IDependencyProvider) => {
const sendDiagnostics = async (diagnosticFilePath: string, contents: string, provider: IDependencyProvider) => {

// get dependencies from response before firing diagnostics.
const getDepsAndRunPipeline = response => {
Expand All @@ -256,7 +256,7 @@ const sendDiagnostics = async (diagnosticFilePath: string, originalContents: str
let deps = null;
try {
const start = new Date().getTime();
deps = await provider.collect(effectiveContents ? effectiveContents : originalContents);
deps = await provider.collect(contents);
const end = new Date().getTime();
connection.console.log(`manifest parse took ${end - start} ms, found ${deps.length} deps`);
} catch (error) {
Expand All @@ -269,9 +269,7 @@ const sendDiagnostics = async (diagnosticFilePath: string, originalContents: str
}

// map dependencies
const regexVersion = new RegExp(/^(~|\^)?([a-zA-Z0-9]+\.)?([a-zA-Z0-9]+\.)?([a-zA-Z0-9]+\.)?([a-zA-Z0-9]+)$/);
const validPackages = provider.ecosystem === 'golang' ? deps : deps.filter(d => regexVersion.test(d.version.value.trim()));
const pkgMap = new DependencyMap(validPackages);
const pkgMap = new DependencyMap(deps);

// init aggregator
let packageAggregator = provider.ecosystem === 'maven' ? new MavenVulnerabilityAggregator(provider) : new NoopVulnerabilityAggregator(provider);
Expand All @@ -282,7 +280,7 @@ const sendDiagnostics = async (diagnosticFilePath: string, originalContents: str
const start = new Date().getTime();

// fetch vulnerabilities
const request = fetchVulnerabilities(path.basename(diagnosticFilePath), originalContents).then(getDepsAndRunPipeline);
const request = fetchVulnerabilities(path.basename(diagnosticFilePath), contents).then(getDepsAndRunPipeline);
await request;

// report results
Expand All @@ -297,52 +295,16 @@ const sendDiagnostics = async (diagnosticFilePath: string, originalContents: str
});
};

function sendDiagnosticsWithEffectivePom(uri, originalContents: string) {
let tempTarget = uri.replace('file://', '').replaceAll('%20', ' ').replace('pom.xml', '');
const effectivePomPath = path.join(tempTarget, 'target', 'effective-pom.xml');
const tmpPomPath = path.join(tempTarget, 'target', 'in-memory-pom.xml');
if (!fs.existsSync(path.dirname(tmpPomPath))) {
fs.mkdirSync(path.dirname(tmpPomPath), { recursive: true});
}
fs.writeFile(tmpPomPath, originalContents, (error) => {
if (error) {
server.connection.sendNotification('caError', error);
} else {
try {
execSync(`${globalSettings.mvnExecutable} help:effective-pom -Doutput='${effectivePomPath}' --quiet -f '${tmpPomPath}'`);
try {
const effectiveContents = fs.readFileSync(effectivePomPath, 'utf8');
sendDiagnostics(uri, originalContents, effectiveContents, new PomXml(originalContents, false));
} catch (error) {
server.connection.sendNotification('caError', error.message);
}
} catch (error) {
// Ignore. Non parseable pom and fall back to original content
server.connection.sendNotification('caSimpleWarning', 'Full component analysis cannot be performed until the Pom is valid.');
connection.console.info('Unable to parse effective pom. Cause: ' + error.message);
sendDiagnostics(uri, originalContents, null, new PomXml(originalContents, true));
} finally {
if (fs.existsSync(tmpPomPath)) {
fs.rmSync(tmpPomPath);
}
if (fs.existsSync(effectivePomPath)) {
fs.rmSync(effectivePomPath);
}
}
}
});
}

files.on(EventStream.Diagnostics, '^package\\.json$', (uri, name, contents) => {
sendDiagnostics(uri, contents, null, new PackageJson());
sendDiagnostics(uri, contents, new PackageJson());
});

files.on(EventStream.Diagnostics, '^pom\\.xml$', (uri, name, contents) => {
sendDiagnosticsWithEffectivePom(uri, contents);
sendDiagnostics(uri, contents, new PomXml());
});

files.on(EventStream.Diagnostics, '^go\\.mod$', (uri, name, contents) => {
sendDiagnostics(uri, contents, null, new GoMod());
sendDiagnostics(uri, contents, new GoMod());
});


Expand Down
6 changes: 3 additions & 3 deletions test/aggregators.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ describe('Noop vulnerability aggregator tests', () => {
describe('Maven vulnerability aggregator tests', () => {

it('Test Maven aggregator with one vulnerability', async () => {
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml('', false));
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml());
// let vulnerability = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, null, '', '', null, '');
let vulnerability = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, '');
let aggVulnerability = mavenVulnerabilityAggregator.aggregate(vulnerability);
Expand All @@ -62,7 +62,7 @@ describe('Maven vulnerability aggregator tests', () => {
});

it('Test Maven aggregator with two identical vulnerability', async () => {
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml('', false));
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml());
// let vulnerability1 = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, null, '', '', null, '');
let vulnerability1 = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, '');
let aggVulnerability1 = mavenVulnerabilityAggregator.aggregate(vulnerability1);
Expand All @@ -77,7 +77,7 @@ describe('Maven vulnerability aggregator tests', () => {
});

it('Test Maven aggregator with two different vulnerability', async () => {
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml('', false));
let mavenVulnerabilityAggregator = new MavenVulnerabilityAggregator(new PomXml());
// let vulnerability1 = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, null, '', '', null, '');
let vulnerability1 = new Vulnerability(dummyRange, 'pkg:maven/[email protected]', 0, '');
let aggVulnerability1 = mavenVulnerabilityAggregator.aggregate(vulnerability1);
Expand Down
Loading

0 comments on commit 56975d1

Please sign in to comment.