From 0445b9cda48d04f55c60cb33bc82b46b9601e180 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sat, 30 Nov 2024 22:30:06 +0100 Subject: [PATCH 1/6] bump ruff 0.8.1 --- dojo/management/commands/rename_mend_findings.py | 4 ++-- dojo/pipeline.py | 2 +- dojo/tools/appcheck_web_application_scanner/engines/base.py | 2 +- dojo/tools/burp_enterprise/parser.py | 2 +- dojo/tools/burp_graphql/parser.py | 2 +- dojo/tools/crashtest_security/parser.py | 2 +- dojo/tools/microfocus_webinspect/parser.py | 2 +- dojo/tools/nexpose/parser.py | 2 +- dojo/tools/npm_audit/parser.py | 2 +- dojo/tools/qualys_webapp/parser.py | 2 +- dojo/tools/sarif/parser.py | 2 +- dojo/tools/sonarqube/soprasteria_helper.py | 2 +- dojo/tools/trivy_operator/uniform_vulnid.py | 4 ++-- dojo/tools/veracode/xml_parser.py | 2 +- dojo/tools/wapiti/parser.py | 2 +- dojo/user/validators.py | 4 ++-- requirements-lint.txt | 2 +- tests/Import_scanner_test.py | 6 +++--- 18 files changed, 23 insertions(+), 23 deletions(-) diff --git a/dojo/management/commands/rename_mend_findings.py b/dojo/management/commands/rename_mend_findings.py index 1620e5ce93b..f99f35a8027 100644 --- a/dojo/management/commands/rename_mend_findings.py +++ b/dojo/management/commands/rename_mend_findings.py @@ -33,8 +33,8 @@ def rename_mend_finding(): logger.info("######## Updating Hashcodes - deduplication is done in the background upon finding save ########") for finding in findings: logger.info("Updating Mend Finding with id: %d", finding.id) - lib_name_begin = re.search("\\*\\*Library Filename\\*\\* : ", finding.description).span(0)[1] - lib_name_end = re.search("\\*\\*Library Description\\*\\*", finding.description).span(0)[0] + lib_name_begin = re.search(r"\*\*Library Filename\*\* : ", finding.description).span(0)[1] + lib_name_end = re.search(r"\*\*Library Description\*\*", finding.description).span(0)[0] lib_name = finding.description[lib_name_begin:lib_name_end - 1] if finding.cve is None: finding.title = "CVE-None | " + lib_name diff --git a/dojo/pipeline.py b/dojo/pipeline.py index befabc0e836..91dc1500089 100644 --- a/dojo/pipeline.py +++ b/dojo/pipeline.py @@ -107,7 +107,7 @@ def update_azure_groups(backend, uid, user=None, social=None, *args, **kwargs): def is_group_id(group): - return bool(re.search("^[a-zA-Z0-9]{8,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{12,}$", group)) + return bool(re.search(r"^[a-zA-Z0-9]{8,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{12,}$", group)) def assign_user_to_groups(user, group_names, social_provider): diff --git a/dojo/tools/appcheck_web_application_scanner/engines/base.py b/dojo/tools/appcheck_web_application_scanner/engines/base.py index e07433c2946..84523b90435 100644 --- a/dojo/tools/appcheck_web_application_scanner/engines/base.py +++ b/dojo/tools/appcheck_web_application_scanner/engines/base.py @@ -205,7 +205,7 @@ def parse_initial_date(self, finding: Finding, value: str) -> None: ##### # For parsing CVEs ##### - CVE_PATTERN = re.compile("CVE-[0-9]+-[0-9]+", re.IGNORECASE) + CVE_PATTERN = re.compile(r"CVE-[0-9]+-[0-9]+", re.IGNORECASE) def is_cve(self, c: str) -> bool: return bool(c and isinstance(c, str) and self.CVE_PATTERN.fullmatch(c)) diff --git a/dojo/tools/burp_enterprise/parser.py b/dojo/tools/burp_enterprise/parser.py index 052d8a80f84..58b2a5a6ea6 100644 --- a/dojo/tools/burp_enterprise/parser.py +++ b/dojo/tools/burp_enterprise/parser.py @@ -162,7 +162,7 @@ def _set_or_append_content(self, finding_details: dict, header: str, div_element cleaned_item = item.split(":")[0] if ( finding_details["cwe"] is None - and (cwe_search := re.search("CWE-([0-9]*)", cleaned_item, re.IGNORECASE)) + and (cwe_search := re.search(r"CWE-([0-9]*)", cleaned_item, re.IGNORECASE)) ): finding_details["cwe"] = int(cwe_search.group(1)) if "vulnerability_ids" not in finding_details: diff --git a/dojo/tools/burp_graphql/parser.py b/dojo/tools/burp_graphql/parser.py index 9b37760e2a8..11df852dc54 100644 --- a/dojo/tools/burp_graphql/parser.py +++ b/dojo/tools/burp_graphql/parser.py @@ -219,7 +219,7 @@ def parse_evidence(self, evidence): def get_cwe(self, cwe_html): # Match only the first CWE! - cweSearch = re.search("CWE-([0-9]*)", cwe_html, re.IGNORECASE) + cweSearch = re.search(r"CWE-([0-9]*)", cwe_html, re.IGNORECASE) if cweSearch: return cweSearch.group(1) return 0 diff --git a/dojo/tools/crashtest_security/parser.py b/dojo/tools/crashtest_security/parser.py index deedb916b81..a12c194723a 100644 --- a/dojo/tools/crashtest_security/parser.py +++ b/dojo/tools/crashtest_security/parser.py @@ -185,7 +185,7 @@ def get_items(self, tree, test): title = re.sub(r" \([0-9]*\)$", "", title) # Attache CVEs - vulnerability_id = re.findall("CVE-\\d{4}-\\d{4,10}", title)[0] if "CVE" in title else None + vulnerability_id = re.findall(r"CVE-\d{4}-\d{4,10}", title)[0] if "CVE" in title else None description = failure.get("message") severity = failure.get("type").capitalize() diff --git a/dojo/tools/microfocus_webinspect/parser.py b/dojo/tools/microfocus_webinspect/parser.py index bf4475580d0..df1b4f84bac 100644 --- a/dojo/tools/microfocus_webinspect/parser.py +++ b/dojo/tools/microfocus_webinspect/parser.py @@ -111,7 +111,7 @@ def convert_severity(val): @staticmethod def get_cwe(val): # Match only the first CWE! - cweSearch = re.search("CWE-(\\d+)", val, re.IGNORECASE) + cweSearch = re.search(r"CWE-(\d+)", val, re.IGNORECASE) if cweSearch: return int(cweSearch.group(1)) return 0 diff --git a/dojo/tools/nexpose/parser.py b/dojo/tools/nexpose/parser.py index d6b63c66c8a..08916d42901 100644 --- a/dojo/tools/nexpose/parser.py +++ b/dojo/tools/nexpose/parser.py @@ -265,7 +265,7 @@ def get_items(self, tree, vulns, test): "severity": "Info", "tags": [ re.sub( - "[^A-Za-z0-9]+", + r"[^A-Za-z0-9]+", "-", service.get("name").lower(), ).rstrip("-"), diff --git a/dojo/tools/npm_audit/parser.py b/dojo/tools/npm_audit/parser.py index 6296477a971..186f133e6ab 100644 --- a/dojo/tools/npm_audit/parser.py +++ b/dojo/tools/npm_audit/parser.py @@ -66,7 +66,7 @@ def censor_path_hashes(path): if not path: return None - return re.sub("[a-f0-9]{64}", "censored_by_npm_audit", path) + return re.sub(r"[a-f0-9]{64}", "censored_by_npm_audit", path) def get_item(item_node, test): diff --git a/dojo/tools/qualys_webapp/parser.py b/dojo/tools/qualys_webapp/parser.py index 825d55b531a..989e5ba48cd 100644 --- a/dojo/tools/qualys_webapp/parser.py +++ b/dojo/tools/qualys_webapp/parser.py @@ -34,7 +34,7 @@ def truncate_str(value: str, maxlen: int): # Parse 'CWE-XXXX' format to strip just the numbers def get_cwe(cwe): - cweSearch = re.search("CWE-([0-9]*)", cwe, re.IGNORECASE) + cweSearch = re.search(r"CWE-([0-9]*)", cwe, re.IGNORECASE) if cweSearch: return cweSearch.group(1) return 0 diff --git a/dojo/tools/sarif/parser.py b/dojo/tools/sarif/parser.py index a9532b21134..414cb0c36ab 100644 --- a/dojo/tools/sarif/parser.py +++ b/dojo/tools/sarif/parser.py @@ -156,7 +156,7 @@ def get_message_from_multiformatMessageString(data, rule): def cve_try(val): # Match only the first CVE! - cveSearch = re.search("(CVE-[0-9]+-[0-9]+)", val, re.IGNORECASE) + cveSearch = re.search(r"(CVE-[0-9]+-[0-9]+)", val, re.IGNORECASE) if cveSearch: return cveSearch.group(1).upper() return None diff --git a/dojo/tools/sonarqube/soprasteria_helper.py b/dojo/tools/sonarqube/soprasteria_helper.py index 2e7259e6376..63b59607e6a 100644 --- a/dojo/tools/sonarqube/soprasteria_helper.py +++ b/dojo/tools/sonarqube/soprasteria_helper.py @@ -41,7 +41,7 @@ def get_references(self, rule_name, vuln_details): def get_cwe(self, vuln_references): # Match only the first CWE! - cweSearch = re.search("CWE-([0-9]*)", vuln_references, re.IGNORECASE) + cweSearch = re.search(r"CWE-([0-9]*)", vuln_references, re.IGNORECASE) if cweSearch: return cweSearch.group(1) return 0 diff --git a/dojo/tools/trivy_operator/uniform_vulnid.py b/dojo/tools/trivy_operator/uniform_vulnid.py index b3aae5055e4..b03ef9acbed 100644 --- a/dojo/tools/trivy_operator/uniform_vulnid.py +++ b/dojo/tools/trivy_operator/uniform_vulnid.py @@ -8,12 +8,12 @@ def return_uniformed_vulnid(self, vulnid): if "cve" in vulnid.lower(): return vulnid if "khv" in vulnid.lower(): - temp = re.compile("([a-zA-Z-_]+)([0-9]+)") + temp = re.compile(r"([a-zA-Z-_]+)([0-9]+)") number = str(temp.match(vulnid).groups()[1]).zfill(3) avd_category = str(temp.match(vulnid.lower()).groups()[0]) return avd_category.upper() + number if "ksv" in vulnid.lower() or "kcv" in vulnid.lower(): - temp = re.compile("([a-zA-Z-_]+)([0-9]+)") + temp = re.compile(r"([a-zA-Z-_]+)([0-9]+)") number = str(temp.match(vulnid).groups()[1]).zfill(4) avd_category = str(temp.match(vulnid.lower().replace("_", "").replace("-", "")).groups()[0].replace("avd", "")) return "AVD-" + avd_category.upper() + "-" + number diff --git a/dojo/tools/veracode/xml_parser.py b/dojo/tools/veracode/xml_parser.py index 17061402d6f..1e53b5545c4 100644 --- a/dojo/tools/veracode/xml_parser.py +++ b/dojo/tools/veracode/xml_parser.py @@ -271,7 +271,7 @@ def __xml_dynamic_flaw_to_finding( @staticmethod def _get_cwe(val): # Match only the first CWE! - cweSearch = re.search("CWE-(\\d+)", val, re.IGNORECASE) + cweSearch = re.search(r"CWE-(\d+)", val, re.IGNORECASE) if cweSearch: return int(cweSearch.group(1)) return None diff --git a/dojo/tools/wapiti/parser.py b/dojo/tools/wapiti/parser.py index 335281b9701..591ae3a390b 100644 --- a/dojo/tools/wapiti/parser.py +++ b/dojo/tools/wapiti/parser.py @@ -104,7 +104,7 @@ def get_findings(self, file, test): @staticmethod def get_cwe(val): # Match only the first CWE! - cweSearch = re.search("CWE-(\\d+)", val, re.IGNORECASE) + cweSearch = re.search(r"CWE-(\d+)", val, re.IGNORECASE) if cweSearch: return int(cweSearch.group(1)) return None diff --git a/dojo/user/validators.py b/dojo/user/validators.py index 83ee954419e..f6b665bc1c2 100644 --- a/dojo/user/validators.py +++ b/dojo/user/validators.py @@ -45,7 +45,7 @@ def get_help_text(self): class UppercaseValidator: def validate(self, password, user=None): - if not re.findall("[A-Z]", password) and get_system_setting("uppercase_character_required"): + if not re.findall(r"[A-Z]", password) and get_system_setting("uppercase_character_required"): raise ValidationError( self.get_help_text(), code="password_no_upper") @@ -57,7 +57,7 @@ def get_help_text(self): class LowercaseValidator: def validate(self, password, user=None): - if not re.findall("[a-z]", password) and get_system_setting("lowercase_character_required"): + if not re.findall(r"[a-z]", password) and get_system_setting("lowercase_character_required"): raise ValidationError( self.get_help_text(), code="password_no_lower") diff --git a/requirements-lint.txt b/requirements-lint.txt index 6821d390595..8f161d317ec 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1 +1 @@ -ruff==0.7.4 +ruff==0.8.1 \ No newline at end of file diff --git a/tests/Import_scanner_test.py b/tests/Import_scanner_test.py index 737b48bdcf0..ba10f7a889b 100644 --- a/tests/Import_scanner_test.py +++ b/tests/Import_scanner_test.py @@ -145,8 +145,8 @@ def test_engagement_import_scan_result(self): options_text = [scan.strip() for scan in options_text] mod_options = options_text - mod_options = [re.sub(" Scanner", "", scan) for scan in mod_options] - mod_options = [re.sub(" Scan", "", scan) for scan in mod_options] + mod_options = [scan.replace(" Scanner", "") for scan in mod_options] + mod_options = [scan.replace(" Scan", "") for scan in mod_options] mod_options = [scan.lower().replace("-", " ").replace(".", "") for scan in mod_options] acronyms = [] @@ -199,7 +199,7 @@ def test_engagement_import_scan_result(self): driver.find_element(By.ID, "id_file").send_keys(test_location) driver.find_element(By.CSS_SELECTOR, "input.btn.btn-primary").click() EngagementTXT = "".join(driver.find_element(By.TAG_NAME, "BODY").text).split("\n") - reg = re.compile("processed, a total of") + reg = re.compile(r"processed, a total of") matches = list(filter(reg.search, EngagementTXT)) if len(matches) != 1: failed_tests += [test.upper() + " - " + case + ": Not imported"] From 5a96e22fead58bedfc9d9636c6a8953a4177e327 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sat, 30 Nov 2024 22:39:54 +0100 Subject: [PATCH 2/6] fix ruff --- dojo/tools/factory.py | 4 ++-- tests/Import_scanner_test.py | 13 ++++++++----- unittests/test_factory.py | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dojo/tools/factory.py b/dojo/tools/factory.py index 04930321c50..2cb23a2fc3c 100644 --- a/dojo/tools/factory.py +++ b/dojo/tools/factory.py @@ -114,8 +114,8 @@ def requires_tool_type(scan_type): # iterate through the modules in the current package -package_dir = str(Path(__file__).resolve().parent) -for module_name in os.listdir(package_dir): +package_dir = Path(str(Path(__file__).resolve().parent)) +for module_name in any(package_dir.iterdir()): # check if it's dir if Path(os.path.join(package_dir, module_name)).is_dir(): try: diff --git a/tests/Import_scanner_test.py b/tests/Import_scanner_test.py index ba10f7a889b..7f2e3abd9bd 100644 --- a/tests/Import_scanner_test.py +++ b/tests/Import_scanner_test.py @@ -28,9 +28,10 @@ def setUp(self): git.Repo.clone_from("https://github.com/DefectDojo/sample-scan-files", self.repo_path) self.remove_items = ["__init__.py", "__init__.pyc", "factory.py", "factory.pyc", "factory.py", "LICENSE", "README.md", ".gitignore", ".git", "__pycache__"] - tool_path = dir_path[:-5] + "dojo/tools" - tools = sorted(os.listdir(tool_path)) - tests = sorted(os.listdir(self.repo_path)) + tool_path = Path(dir_path[:-5] + "dojo/tools") + tools = sorted(any(tool_path.iterdir())) + p = Path(self.repo_path) + tests = sorted(any(p.iterdir())) self.tools = [i for i in tools if i not in self.remove_items] self.tests = [i for i in tests if i not in self.remove_items] @@ -43,7 +44,8 @@ def test_check_test_file(self): missing_tests += ["\nNO TEST FILES"] for test in self.tests: - cases = sorted(os.listdir(self.repo_path + "/" + test)) + p = Path(self.repo_path + "/" + test) + cases = sorted(any(p.iterdir())) cases = [i for i in cases if i not in self.remove_items] if len(cases) == 0 and tool not in missing_tests: missing_tests += [test] @@ -180,7 +182,8 @@ def test_engagement_import_scan_result(self): failed_tests = [] for test in self.tests: - cases = sorted(os.listdir(self.repo_path + "/" + test)) + p = Path(self.repo_path + "/" + test) + cases = sorted(any(p.iterdir())) cases = [i for i in cases if i not in self.remove_items] if len(cases) == 0: failed_tests += [test.upper() + ": No test cases"] diff --git a/unittests/test_factory.py b/unittests/test_factory.py index 43fb5b54771..8aa165cfe9c 100644 --- a/unittests/test_factory.py +++ b/unittests/test_factory.py @@ -64,8 +64,8 @@ def test_get_parser_test_active_in_db(self): def test_parser_name_matches_module(self): """Test to ensure that parsers' class names match their module names""" - package_dir = "dojo/tools" - module_names = os.listdir(package_dir) + package_dir = Path("dojo/tools") + module_names = any(package_dir.iterdir()) missing_parsers = [] excluded_parsers = [ "wizcli_common_parsers", # common class for other wizcli parsers, there is not parsing here From 5376248c7ef33fec6b35028051933b3cf841e5bb Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Thu, 5 Dec 2024 15:06:46 +0100 Subject: [PATCH 3/6] fix unittests --- dojo/tools/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dojo/tools/factory.py b/dojo/tools/factory.py index 2cb23a2fc3c..b69fea12ac0 100644 --- a/dojo/tools/factory.py +++ b/dojo/tools/factory.py @@ -114,8 +114,8 @@ def requires_tool_type(scan_type): # iterate through the modules in the current package -package_dir = Path(str(Path(__file__).resolve().parent)) -for module_name in any(package_dir.iterdir()): +package_dir = str(Path(__file__).resolve().parent) +for module_name in os.listdir(package_dir): # noqa: PTH208 # check if it's dir if Path(os.path.join(package_dir, module_name)).is_dir(): try: From 724e1b32549f0921c318376a9699d6ff26f0efd4 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 15 Dec 2024 16:04:28 +0100 Subject: [PATCH 4/6] update --- requirements-lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-lint.txt b/requirements-lint.txt index 8f161d317ec..b5dbf8b3e69 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1 +1 @@ -ruff==0.8.1 \ No newline at end of file +ruff==0.8.3 \ No newline at end of file From 7941abcae7eb3bb592e73242444338d9da66bb54 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 15 Dec 2024 16:08:08 +0100 Subject: [PATCH 5/6] update --- requirements-lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-lint.txt b/requirements-lint.txt index b5dbf8b3e69..8f161d317ec 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1 +1 @@ -ruff==0.8.3 \ No newline at end of file +ruff==0.8.1 \ No newline at end of file From bb43bee88f9279119c855461284bbb60ee5ae461 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 18 Dec 2024 11:14:22 +0100 Subject: [PATCH 6/6] fix unittest --- unittests/test_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_factory.py b/unittests/test_factory.py index 8aa165cfe9c..5d8b4040dd5 100644 --- a/unittests/test_factory.py +++ b/unittests/test_factory.py @@ -65,7 +65,7 @@ def test_get_parser_test_active_in_db(self): def test_parser_name_matches_module(self): """Test to ensure that parsers' class names match their module names""" package_dir = Path("dojo/tools") - module_names = any(package_dir.iterdir()) + module_names = package_dir.iterdir() missing_parsers = [] excluded_parsers = [ "wizcli_common_parsers", # common class for other wizcli parsers, there is not parsing here