diff --git a/docs/content/en/usage/features.md b/docs/content/en/usage/features.md index 5f99f34023f..7fad563b138 100644 --- a/docs/content/en/usage/features.md +++ b/docs/content/en/usage/features.md @@ -244,7 +244,7 @@ The environment variable will override the settings in `settings.dist.py`, repla The available algorithms are: -DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL +DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL (value for `DD_DEDUPLICATION_ALGORITHM_PER_PARSER`: `unique_id_from_tool`) : The deduplication occurs based on finding.unique_id_from_tool which is a unique technical id existing in the source tool. Few scanners populate this @@ -266,12 +266,12 @@ DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL able to recognise that findings found in previous scans are actually the same as the new findings. -DEDUPE_ALGO_HASH_CODE +DEDUPE_ALGO_HASH_CODE (value for `DD_DEDUPLICATION_ALGORITHM_PER_PARSER`: `hash_code`) : The deduplication occurs based on finding.hash_code. The hash_code itself is configurable for each scanner in parameter `HASHCODE_FIELDS_PER_SCANNER`. -DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE +DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE (value for `DD_DEDUPLICATION_ALGORITHM_PER_PARSER`: `unique_id_from_tool_or_hash_code`) : A finding is a duplicate with another if they have the same unique_id_from_tool OR the same hash_code. @@ -284,7 +284,7 @@ DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE cross-parser deduplication -DEDUPE_ALGO_LEGACY +DEDUPE_ALGO_LEGACY (value for `DD_DEDUPLICATION_ALGORITHM_PER_PARSER`: `legacy`) : This is algorithm that was in place before the configuration per parser was made possible, and also the default one for backward compatibility reasons. diff --git a/dojo/settings/.settings.dist.py.sha256sum b/dojo/settings/.settings.dist.py.sha256sum index 24fb6aafce2..220f2b35098 100644 --- a/dojo/settings/.settings.dist.py.sha256sum +++ b/dojo/settings/.settings.dist.py.sha256sum @@ -1 +1 @@ -09169f6d20ebf2f37347156111c3670a5b207c3530583a53ed9ac59ae4221188 +f09caa2d4e41f44b7cd6ecf2f1400817d4776e703bd039c8d857f1356382e1f3 diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index a0145954308..e9100b681b5 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1296,6 +1296,12 @@ def saml2_attrib_map_format(dict): if len(env("DD_HASHCODE_FIELDS_PER_SCANNER")) > 0: env_hashcode_fields_per_scanner = json.loads(env("DD_HASHCODE_FIELDS_PER_SCANNER")) for key, value in env_hashcode_fields_per_scanner.items(): + if not isinstance(value, list): + msg = f"Fields definition '{value}' for hashcode calculation of '{key}' is not valid. It needs to be list of strings but it is {type(value)}." + raise TypeError(msg) + if not all(isinstance(field, str) for field in value): + msg = f"Fields for hashcode calculation for {key} are not valid. It needs to be list of strings. Some of fields are not string." + raise AttributeError(msg) if key in HASHCODE_FIELDS_PER_SCANNER: logger.info(f"Replacing {key} with value {value} (previously set to {HASHCODE_FIELDS_PER_SCANNER[key]}) from env var DD_HASHCODE_FIELDS_PER_SCANNER") HASHCODE_FIELDS_PER_SCANNER[key] = value @@ -1377,6 +1383,13 @@ def saml2_attrib_map_format(dict): # Makes it possible to deduplicate on a technical id (same parser) and also on some functional fields (cross-parsers deduplication) DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE = "unique_id_from_tool_or_hash_code" +DEDUPE_ALGOS = [ + DEDUPE_ALGO_LEGACY, + DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL, + DEDUPE_ALGO_HASH_CODE, + DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE, +] + # Allows to deduplicate with endpoints if endpoints is not included in the hashcode. # Possible values are: scheme, host, port, path, query, fragment, userinfo, and user. For a details description see https://hyperlink.readthedocs.io/en/latest/api.html#attributes. # Example: @@ -1526,6 +1539,9 @@ def saml2_attrib_map_format(dict): if len(env("DD_DEDUPLICATION_ALGORITHM_PER_PARSER")) > 0: env_dedup_algorithm_per_parser = json.loads(env("DD_DEDUPLICATION_ALGORITHM_PER_PARSER")) for key, value in env_dedup_algorithm_per_parser.items(): + if value not in DEDUPE_ALGOS: + msg = f"DEDUP algorithm '{value}' for '{key}' is not valid. Use one of following values: {', '.join(DEDUPE_ALGOS)}" + raise AttributeError(msg) if key in DEDUPLICATION_ALGORITHM_PER_PARSER: logger.info(f"Replacing {key} with value {value} (previously set to {DEDUPLICATION_ALGORITHM_PER_PARSER[key]}) from env var DD_DEDUPLICATION_ALGORITHM_PER_PARSER") DEDUPLICATION_ALGORITHM_PER_PARSER[key] = value