From d47ae983d7e3b25d817706d1664bb09ab17405d4 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 10 Jan 2024 13:17:37 +0100 Subject: [PATCH 1/2] :tada: merge OpenVAS XML and CSV formats to have only one parser for OpenVAS --- .../en/integrations/parsers/file/openvas.md | 5 + .../integrations/parsers/file/openvas_csv.md | 5 - .../integrations/parsers/file/openvas_xml.md | 5 - .../{openvas_xml => openvas}/__init__.py | 0 dojo/tools/{openvas_csv => openvas}/parser.py | 159 ++++++++++++------ dojo/tools/openvas_csv/__init__.py | 0 dojo/tools/openvas_xml/parser.py | 68 -------- .../{openvas_csv => openvas}/many_vuln.csv | 0 .../{openvas_xml => openvas}/many_vuln.xml | 0 .../{openvas_xml => openvas}/no_vuln.xml | 0 .../{openvas_csv => openvas}/one_vuln.csv | 0 .../{openvas_xml => openvas}/one_vuln.xml | 0 ...s_csv_parser.py => test_openvas_parser.py} | 50 +++++- unittests/tools/test_openvas_xml_parser.py | 43 ----- 14 files changed, 154 insertions(+), 181 deletions(-) create mode 100644 docs/content/en/integrations/parsers/file/openvas.md delete mode 100644 docs/content/en/integrations/parsers/file/openvas_csv.md delete mode 100644 docs/content/en/integrations/parsers/file/openvas_xml.md rename dojo/tools/{openvas_xml => openvas}/__init__.py (100%) rename dojo/tools/{openvas_csv => openvas}/parser.py (67%) mode change 100644 => 100755 delete mode 100644 dojo/tools/openvas_csv/__init__.py delete mode 100755 dojo/tools/openvas_xml/parser.py rename unittests/scans/{openvas_csv => openvas}/many_vuln.csv (100%) rename unittests/scans/{openvas_xml => openvas}/many_vuln.xml (100%) rename unittests/scans/{openvas_xml => openvas}/no_vuln.xml (100%) rename unittests/scans/{openvas_csv => openvas}/one_vuln.csv (100%) rename unittests/scans/{openvas_xml => openvas}/one_vuln.xml (100%) rename unittests/tools/{test_openvas_csv_parser.py => test_openvas_parser.py} (50%) delete mode 100644 unittests/tools/test_openvas_xml_parser.py diff --git a/docs/content/en/integrations/parsers/file/openvas.md b/docs/content/en/integrations/parsers/file/openvas.md new file mode 100644 index 0000000000..6cfbcd5efe --- /dev/null +++ b/docs/content/en/integrations/parsers/file/openvas.md @@ -0,0 +1,5 @@ +--- +title: "OpenVAS Parser" +toc_hide: true +--- +You can eiter upload the exported results of an OpenVAS Scan in a .csv or .xml format. \ No newline at end of file diff --git a/docs/content/en/integrations/parsers/file/openvas_csv.md b/docs/content/en/integrations/parsers/file/openvas_csv.md deleted file mode 100644 index 621f055a41..0000000000 --- a/docs/content/en/integrations/parsers/file/openvas_csv.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: "OpenVAS CSV" -toc_hide: true ---- -Import OpenVAS Scan in CSV format. Export as CSV Results on OpenVAS. diff --git a/docs/content/en/integrations/parsers/file/openvas_xml.md b/docs/content/en/integrations/parsers/file/openvas_xml.md deleted file mode 100644 index c361a1c44b..0000000000 --- a/docs/content/en/integrations/parsers/file/openvas_xml.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: "OpenVAS XML" -toc_hide: true ---- -Import Greenbone OpenVAS Scan in XML format. Export as XML Results on OpenVAS. diff --git a/dojo/tools/openvas_xml/__init__.py b/dojo/tools/openvas/__init__.py similarity index 100% rename from dojo/tools/openvas_xml/__init__.py rename to dojo/tools/openvas/__init__.py diff --git a/dojo/tools/openvas_csv/parser.py b/dojo/tools/openvas/parser.py old mode 100644 new mode 100755 similarity index 67% rename from dojo/tools/openvas_csv/parser.py rename to dojo/tools/openvas/parser.py index 04d6166b23..9a8c9b79ad --- a/dojo/tools/openvas_csv/parser.py +++ b/dojo/tools/openvas/parser.py @@ -1,10 +1,10 @@ import csv import hashlib import io - from dateutil.parser import parse - -from dojo.models import Endpoint, Finding +from xml.dom import NamespaceErr +from defusedxml import ElementTree as ET +from dojo.models import Finding, Endpoint class ColumnMappingStrategy(object): @@ -194,7 +194,7 @@ def map_column_value(self, finding, column_value): finding.duplicate = self.evaluate_bool_value(column_value) -class OpenVASCsvParser(object): +class OpenVASParser(object): def create_chain(self): date_column_strategy = DateColumnMappingStrategy() title_column_strategy = TitleColumnMappingStrategy() @@ -240,62 +240,115 @@ def read_column_names(self, row): return column_names def get_scan_types(self): - return ["OpenVAS CSV"] + return ["OpenVAS Parser"] def get_label_for_scan_types(self, scan_type): return scan_type # no custom label for now def get_description_for_scan_types(self, scan_type): - return "Import OpenVAS Scan in CSV format. Export as CSV Results on OpenVAS." + return "Import CSV or XML output of Greenbone OpenVAS report." + + def convert_cvss_score(self, raw_value): + val = float(raw_value) + if val == 0.0: + return "Info" + elif val < 4.0: + return "Low" + elif val < 7.0: + return "Medium" + elif val < 9.0: + return "High" + else: + return "Critical" def get_findings(self, filename, test): - column_names = dict() - dupes = dict() - chain = self.create_chain() + if str(filename.name).endswith('.csv'): + column_names = dict() + dupes = dict() + chain = self.create_chain() + + content = filename.read() + if isinstance(content, bytes): + content = content.decode("utf-8") + reader = csv.reader(io.StringIO(content), delimiter=",", quotechar='"') + + row_number = 0 + for row in reader: + finding = Finding(test=test) + finding.unsaved_endpoints = [Endpoint()] + + if row_number == 0: + column_names = self.read_column_names(row) + row_number += 1 + continue + + column_number = 0 + for column in row: + chain.process_column( + column_names[column_number], column, finding + ) + column_number += 1 + + if finding is not None and row_number > 0: + if finding.title is None: + finding.title = "" + if finding.description is None: + finding.description = "" + + key = hashlib.sha256( + ( + str(finding.unsaved_endpoints[0]) + + "|" + + finding.severity + + "|" + + finding.title + + "|" + + finding.description + ).encode("utf-8") + ).hexdigest() + + if key not in dupes: + dupes[key] = finding - content = filename.read() - if isinstance(content, bytes): - content = content.decode("utf-8") - reader = csv.reader(io.StringIO(content), delimiter=",", quotechar='"') - - row_number = 0 - for row in reader: - finding = Finding(test=test) - finding.unsaved_endpoints = [Endpoint()] - - if row_number == 0: - column_names = self.read_column_names(row) row_number += 1 - continue - - column_number = 0 - for column in row: - chain.process_column( - column_names[column_number], column, finding + return list(dupes.values()) + elif str(filename.name).endswith('.xml'): + findings = [] + tree = ET.parse(filename) + root = tree.getroot() + if "report" not in root.tag: + raise NamespaceErr( + "This doesn't seem to be a valid Greenbone OpenVAS XML file." + ) + report = root.find("report") + results = report.find("results") + for result in results: + for finding in result: + if finding.tag == "name": + title = finding.text + description = [f"**Name**: {finding.text}"] + if finding.tag == "host": + title = title + "_" + finding.text + description.append(f"**Host**: {finding.text}") + if finding.tag == "port": + title = title + "_" + finding.text + description.append(f"**Port**: {finding.text}") + if finding.tag == "nvt": + description.append(f"**NVT**: {finding.text}") + if finding.tag == "severity": + severity = self.convert_cvss_score(finding.text) + description.append(f"**Severity**: {finding.text}") + if finding.tag == "qod": + description.append(f"**QOD**: {finding.text}") + if finding.tag == "description": + description.append(f"**Description**: {finding.text}") + + finding = Finding( + title=str(title), + description="\n".join(description), + severity=severity, + dynamic_finding=True, + static_finding=False ) - column_number += 1 - - if finding is not None and row_number > 0: - if finding.title is None: - finding.title = "" - if finding.description is None: - finding.description = "" - - key = hashlib.sha256( - ( - str(finding.unsaved_endpoints[0]) - + "|" - + finding.severity - + "|" - + finding.title - + "|" - + finding.description - ).encode("utf-8") - ).hexdigest() - - if key not in dupes: - dupes[key] = finding - - row_number += 1 - - return list(dupes.values()) + findings.append(finding) + return findings diff --git a/dojo/tools/openvas_csv/__init__.py b/dojo/tools/openvas_csv/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dojo/tools/openvas_xml/parser.py b/dojo/tools/openvas_xml/parser.py deleted file mode 100755 index 65449e8c81..0000000000 --- a/dojo/tools/openvas_xml/parser.py +++ /dev/null @@ -1,68 +0,0 @@ -from xml.dom import NamespaceErr -from defusedxml import ElementTree as ET -from dojo.models import Finding - - -class OpenVASXMLParser(object): - def get_scan_types(self): - return ["OpenVAS XML"] - - def get_label_for_scan_types(self, scan_type): - return scan_type # no custom label for now - - def get_description_for_scan_types(self, scan_type): - return "Import XML output of Greenbone OpenVAS XML report." - - def convert_cvss_score(self, raw_value): - val = float(raw_value) - if val == 0.0: - return "Info" - elif val < 4.0: - return "Low" - elif val < 7.0: - return "Medium" - elif val < 9.0: - return "High" - else: - return "Critical" - - def get_findings(self, file, test): - findings = [] - tree = ET.parse(file) - root = tree.getroot() - if "report" not in root.tag: - raise NamespaceErr( - "This doesn't seem to be a valid Greenbone OpenVAS xml file." - ) - report = root.find("report") - results = report.find("results") - for result in results: - for finding in result: - if finding.tag == "name": - title = finding.text - description = [f"**Name**: {finding.text}"] - if finding.tag == "host": - title = title + "_" + finding.text - description.append(f"**Host**: {finding.text}") - if finding.tag == "port": - title = title + "_" + finding.text - description.append(f"**Port**: {finding.text}") - if finding.tag == "nvt": - description.append(f"**NVT**: {finding.text}") - if finding.tag == "severity": - severity = self.convert_cvss_score(finding.text) - description.append(f"**Severity**: {finding.text}") - if finding.tag == "qod": - description.append(f"**QOD**: {finding.text}") - if finding.tag == "description": - description.append(f"**Description**: {finding.text}") - - finding = Finding( - title=str(title), - description="\n".join(description), - severity=severity, - dynamic_finding=True, - static_finding=False - ) - findings.append(finding) - return findings diff --git a/unittests/scans/openvas_csv/many_vuln.csv b/unittests/scans/openvas/many_vuln.csv similarity index 100% rename from unittests/scans/openvas_csv/many_vuln.csv rename to unittests/scans/openvas/many_vuln.csv diff --git a/unittests/scans/openvas_xml/many_vuln.xml b/unittests/scans/openvas/many_vuln.xml similarity index 100% rename from unittests/scans/openvas_xml/many_vuln.xml rename to unittests/scans/openvas/many_vuln.xml diff --git a/unittests/scans/openvas_xml/no_vuln.xml b/unittests/scans/openvas/no_vuln.xml similarity index 100% rename from unittests/scans/openvas_xml/no_vuln.xml rename to unittests/scans/openvas/no_vuln.xml diff --git a/unittests/scans/openvas_csv/one_vuln.csv b/unittests/scans/openvas/one_vuln.csv similarity index 100% rename from unittests/scans/openvas_csv/one_vuln.csv rename to unittests/scans/openvas/one_vuln.csv diff --git a/unittests/scans/openvas_xml/one_vuln.xml b/unittests/scans/openvas/one_vuln.xml similarity index 100% rename from unittests/scans/openvas_xml/one_vuln.xml rename to unittests/scans/openvas/one_vuln.xml diff --git a/unittests/tools/test_openvas_csv_parser.py b/unittests/tools/test_openvas_parser.py similarity index 50% rename from unittests/tools/test_openvas_csv_parser.py rename to unittests/tools/test_openvas_parser.py index 6931edaea6..a3fdf35453 100644 --- a/unittests/tools/test_openvas_csv_parser.py +++ b/unittests/tools/test_openvas_parser.py @@ -1,16 +1,15 @@ from ..dojo_test_case import DojoTestCase -from dojo.tools.openvas_csv.parser import OpenVASCsvParser +from dojo.tools.openvas.parser import OpenVASParser from dojo.models import Test, Engagement, Product -class TestOpenVASUploadCsvParser(DojoTestCase): - +class TestOpenVASParser(DojoTestCase): def test_openvas_csv_one_vuln(self): - with open("unittests/scans/openvas_csv/one_vuln.csv") as f: + with open("unittests/scans/openvas/one_vuln.csv") as f: test = Test() test.engagement = Engagement() test.engagement.product = Product() - parser = OpenVASCsvParser() + parser = OpenVASParser() findings = parser.get_findings(f, test) for finding in findings: for endpoint in finding.unsaved_endpoints: @@ -27,11 +26,11 @@ def test_openvas_csv_one_vuln(self): self.assertEqual(22, findings[0].unsaved_endpoints[0].port) def test_openvas_csv_many_vuln(self): - with open("unittests/scans/openvas_csv/many_vuln.csv") as f: + with open("unittests/scans/openvas/many_vuln.csv") as f: test = Test() test.engagement = Engagement() test.engagement.product = Product() - parser = OpenVASCsvParser() + parser = OpenVASParser() findings = parser.get_findings(f, test) for finding in findings: for endpoint in finding.unsaved_endpoints: @@ -48,3 +47,40 @@ def test_openvas_csv_many_vuln(self): self.assertEqual("LOGSRV", endpoint.host) self.assertEqual("tcp", endpoint.protocol) self.assertEqual(9200, endpoint.port) + + def test_openvas_xml_no_vuln(self): + with open("unittests/scans/openvas/no_vuln.xml") as f: + test = Test() + test.engagement = Engagement() + test.engagement.product = Product() + parser = OpenVASParser() + findings = parser.get_findings(f, test) + self.assertEqual(0, len(findings)) + + def test_openvas_xml_one_vuln(self): + with open("unittests/scans/openvas/one_vuln.xml") as f: + test = Test() + test.engagement = Engagement() + test.engagement.product = Product() + parser = OpenVASParser() + findings = parser.get_findings(f, test) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + self.assertEqual(1, len(findings)) + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Mozilla Firefox Security Update (mfsa_2023-32_2023-36) - Windows_10.0.101.2_general/tcp", finding.title) + self.assertEqual("Critical", finding.severity) + + def test_openvas_xml_many_vuln(self): + with open("unittests/scans/openvas/many_vuln.xml") as f: + test = Test() + test.engagement = Engagement() + test.engagement.product = Product() + parser = OpenVASParser() + findings = parser.get_findings(f, test) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + self.assertEqual(44, len(findings)) diff --git a/unittests/tools/test_openvas_xml_parser.py b/unittests/tools/test_openvas_xml_parser.py deleted file mode 100644 index 40004d6e0b..0000000000 --- a/unittests/tools/test_openvas_xml_parser.py +++ /dev/null @@ -1,43 +0,0 @@ -from ..dojo_test_case import DojoTestCase -from dojo.tools.openvas_xml.parser import OpenVASXMLParser -from dojo.models import Test, Engagement, Product - - -class TestOpenVASUploadXMLParser(DojoTestCase): - - def test_openvas_xml_no_vuln(self): - with open("unittests/scans/openvas_xml/no_vuln.xml") as f: - test = Test() - test.engagement = Engagement() - test.engagement.product = Product() - parser = OpenVASXMLParser() - findings = parser.get_findings(f, test) - self.assertEqual(0, len(findings)) - - def test_openvas_xml_one_vuln(self): - with open("unittests/scans/openvas_xml/one_vuln.xml") as f: - test = Test() - test.engagement = Engagement() - test.engagement.product = Product() - parser = OpenVASXMLParser() - findings = parser.get_findings(f, test) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - self.assertEqual(1, len(findings)) - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Mozilla Firefox Security Update (mfsa_2023-32_2023-36) - Windows_10.0.101.2_general/tcp", finding.title) - self.assertEqual("Critical", finding.severity) - - def test_openvas_xml_many_vuln(self): - with open("unittests/scans/openvas_xml/many_vuln.xml") as f: - test = Test() - test.engagement = Engagement() - test.engagement.product = Product() - parser = OpenVASXMLParser() - findings = parser.get_findings(f, test) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - self.assertEqual(44, len(findings)) From 840bd45c683d73bcd94b06307ad8ea17f5cd775b Mon Sep 17 00:00:00 2001 From: manuelsommer <47991713+manuel-sommer@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:53:26 +0100 Subject: [PATCH 2/2] Update docs/content/en/integrations/parsers/file/openvas.md Co-authored-by: Charles Neill <1749665+cneill@users.noreply.github.com> --- docs/content/en/integrations/parsers/file/openvas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/en/integrations/parsers/file/openvas.md b/docs/content/en/integrations/parsers/file/openvas.md index 6cfbcd5efe..ab93b2498f 100644 --- a/docs/content/en/integrations/parsers/file/openvas.md +++ b/docs/content/en/integrations/parsers/file/openvas.md @@ -2,4 +2,4 @@ title: "OpenVAS Parser" toc_hide: true --- -You can eiter upload the exported results of an OpenVAS Scan in a .csv or .xml format. \ No newline at end of file +You can either upload the exported results of an OpenVAS Scan in a .csv or .xml format. \ No newline at end of file