Merge branch 'dev' into ruffpth120

.github/workflows/build-docker-images-for-testing.yml

@@ -49,8 +49,8 @@ jobs:
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
         timeout-minutes: 10
-        uses:  actions/upload-artifact@v3
+        uses:  actions/upload-artifact@v4
         with:
-          name: ${{ matrix.docker-image }}
+          name: built-docker-image-${{ matrix.docker-image }}-${{ matrix.os }}
           path: ${{ matrix.docker-image }}-${{ matrix.os }}_img
           retention-days: 1

.github/workflows/fetch-oas.yml

@@ -51,7 +51,7 @@ jobs:
         run: docker compose down
 
       - name: Upload oas.${{ matrix.file-type }} as artifact
-        uses:  actions/upload-artifact@v3
+        uses:  actions/upload-artifact@v4
         with:
           name: oas-${{ matrix.file-type }}
           path: oas.${{ matrix.file-type }}

.github/workflows/integration-tests.yml

@@ -45,14 +45,18 @@ jobs:
 
       # load docker images from build jobs
       - name: Load images from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
+        with:
+          path: built-docker-image
+          pattern: built-docker-image-*
+          merge-multiple: true
 
       - name: Load docker images
         timeout-minutes: 10
         run: |-
-             docker load -i nginx/nginx-${{ matrix.os }}_img
-             docker load -i django/django-${{ matrix.os }}_img
-             docker load -i integration-tests/integration-tests-debian_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/integration-tests-debian_img
              docker images
 
       - name: Set integration-test mode

.github/workflows/k8s-tests.yml

@@ -48,14 +48,18 @@ jobs:
           minikube status
 
       - name: Load images from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
+        with:
+          path: built-docker-image
+          pattern: built-docker-image-*
+          merge-multiple: true
 
       - name: Load docker images
         timeout-minutes: 10
         run: |-
              eval $(minikube docker-env)
-             docker load -i nginx/nginx-${{ matrix.os }}_img
-             docker load -i django/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}_img
              docker images
 
       - name: Configure HELM repos

.github/workflows/release-drafter.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Load OAS files from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Upload Release Asset - OpenAPI Specification - YAML
         id: upload-release-asset-yaml

.github/workflows/rest-framework-tests.yml

@@ -20,13 +20,17 @@ jobs:
 
       # load docker images from build jobs
       - name: Load images from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
+        with:
+          path: built-docker-image
+          pattern: built-docker-image-*
+          merge-multiple: true
 
       - name: Load docker images
         timeout-minutes: 10
         run: |-
-             docker load -i nginx/nginx-${{ matrix.os }}_img
-             docker load -i django/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}_img
              docker images
 
       # run tests with docker compose

dojo/api_v2/serializers.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import os
 import re
 from datetime import datetime
 
@@ -803,20 +802,8 @@ class Meta:
 
     def validate(self, data):
         if file := data.get("file"):
-            ext = os.path.splitext(file.name)[1]  # [0] returns path+filename
-            valid_extensions = settings.FILE_UPLOAD_TYPES
-            if ext.lower() not in valid_extensions:
-                if accepted_extensions := f"{', '.join(valid_extensions)}":
-                    msg = (
-                        "Unsupported extension. Supported extensions are as "
-                        f"follows: {accepted_extensions}"
-                    )
-                else:
-                    msg = (
-                        "File uploads are prohibited due to the list of acceptable "
-                        "file extensions being empty"
-                    )
-                raise ValidationError(msg)
+            # the clean will validate the file extensions and raise a Validation error if the extensions are not accepted
+            FileUpload(title=file.name, file=file).clean()
             return data
         return None
 

dojo/decorators.py

@@ -144,22 +144,6 @@ def get_parameter_froms_args_kwargs(args, kwargs, parameter):
     return model_or_id
 
 
-def on_exception_log_kwarg(func):
-    def wrapper(self, *args, **kwargs):
-        try:
-            return func(self, *args, **kwargs)
-
-        except Exception:
-            logger.info(f"exception occured at url: {self.driver.current_url}")
-            logger.info(f"page source: {self.driver.page_source}")
-            f = open("/tmp/selenium_page_source.html", "w", encoding="utf-8")
-            f.writelines(self.driver.page_source)
-            # time.sleep(30)
-            raise
-
-    return wrapper
-
-
 def dojo_ratelimit(key="ip", rate=None, method=UNSAFE, block=False):
     def decorator(fn):
         @wraps(fn)

dojo/importers/options.py

@@ -530,13 +530,15 @@ def validate_tags(
         *args: list,
         **kwargs: dict,
     ) -> list:
-        return self.validate(
+        tags = self.validate(
             "tags",
             expected_types=[list],
             required=False,
             default=[],
             **kwargs,
         )
+        # Force all tags to be lowercase
+        return [tag.lower() for tag in tags]
 
     def validate_test(
         self,

dojo/models.py

@@ -742,6 +742,28 @@ def get_accessible_url(self, obj, obj_id):
 
         return f"access_file/{self.id}/{obj_id}/{obj_type}"
 
+    def clean(self):
+        if not self.title:
+            self.title = "<No Title>"
+
+        valid_extensions = settings.FILE_UPLOAD_TYPES
+
+        # why does this not work with self.file....
+        if self.file:
+            file_name = self.file.url
+        else:
+            file_name = self.title
+        if Path(file_name).suffix.lower() not in valid_extensions:
+            if accepted_extensions := f"{', '.join(valid_extensions)}":
+                msg = (
+                    _("Unsupported extension. Supported extensions are as follows: %s") % accepted_extensions
+                )
+            else:
+                msg = (
+                    _("File uploads are prohibited due to the list of acceptable file extensions being empty")
+                )
+            raise ValidationError(msg)
+
 
 class Product_Type(models.Model):
 

dojo/settings/settings.dist.py

@@ -1745,6 +1745,9 @@ def saml2_attrib_map_format(dict):
     "ELSA": "https://linux.oracle.com/errata/&&.html",  # e.g. https://linux.oracle.com/errata/ELSA-2024-12714.html
     "ELBA": "https://linux.oracle.com/errata/&&.html",  # e.g. https://linux.oracle.com/errata/ELBA-2024-7457.html
     "RXSA": "https://errata.rockylinux.org/",  # e.g. https://errata.rockylinux.org/RXSA-2024:4928
+    "CAPEC": "https://capec.mitre.org/data/definitions/&&.html",  # e.g. https://capec.mitre.org/data/definitions/157.html
+    "CWE": "https://cwe.mitre.org/data/definitions/&&.html",  # e.g. https://cwe.mitre.org/data/definitions/79.html
+    "TEMP": "https://security-tracker.debian.org/tracker/",  # e.g. https://security-tracker.debian.org/tracker/TEMP-0841856-B18BAF
 }
 # List of acceptable file types that can be uploaded to a given object via arbitrary file upload
 FILE_UPLOAD_TYPES = env("DD_FILE_UPLOAD_TYPES")

dojo/templates/notifications/alert/engagement_closed.tpl

@@ -0,0 +1,3 @@
+{% load i18n %}{% blocktranslate trimmed with eng_name=engagement.name eng_product=engagement.product %}
+The engagement "{{ eng_name }}" has been closed in the product "{{ eng_product }}".
+{% endblocktranslate %}

dojo/templates/notifications/mail/engagement_closed.tpl

@@ -0,0 +1,41 @@
+{% load i18n %}
+{% load navigation_tags %}
+{% load display_tags %}
+{% url 'view_product' engagement.product.id as product_url %}
+{% url 'view_engagement' engagement.id as engagement_url %}
+<html>
+    <body>
+        {% autoescape on %}
+            <p>
+                {% trans "Hello" %},
+            </p>
+            <p>
+              {% blocktranslate trimmed with engagement_name=engagement.name engagement_product=engagement.product prod_url=product_url|full_url eng_url=engagement_url|full_url%}
+                The engagement "{{ engagement_name }}" has been closed in the product "{{ engagement_product }}". It can be viewed here: <a href="{{prod_url}}">{{product}}</a> / <a href="{{eng_url}}">{{ engagement_name }}</a>
+              {% endblocktranslate %}
+            </p>
+            <br/>
+            <br/>
+                {% trans "Kind regards" %},<br/>
+            <br/>
+                {% if system_settings.team_name %}
+                    {{ system_settings.team_name }}
+                {% else %}
+                    Defect Dojo
+                {% endif %}
+            <br/>
+            <br/>
+            <p>
+                {% url 'notifications' as notification_url %}
+                {% trans "You can manage your notification settings here" %}: <a href="{{ notification_url|full_url }}">{{ notification_url|full_url }}</a>
+            </p>
+            {% if system_settings.disclaimer and system_settings.disclaimer.strip %}
+                <br/>
+                <div style="background-color:#DADCE2; border:1px #003333; padding:.8em; ">
+                    <span style="font-size:16pt;  font-family: 'Cambria','times new roman','garamond',serif; color:#ff0000;">{% trans "Disclaimer" %}</span><br/>
+                    <p style="font-size:11pt; line-height:10pt; font-family: 'Cambria','times roman',serif;">{{ system_settings.disclaimer }}</p>
+                </div>
+            {% endif %}
+        {% endautoescape %}
+    </body>
+</html>

dojo/templates/notifications/msteams/engagement_closed.tpl

@@ -0,0 +1,44 @@
+{% load i18n %}
+{% load display_tags %}
+{
+    "@context": "https://schema.org/extensions",
+    "@type": "MessageCard",
+    "title": "{% trans "Engagement closed" %}",
+    "summary": "{% trans "Engagement closed" %}",
+    "sections": [
+        {
+            "activityTitle": "DefectDojo",
+            "activityImage": "https://raw.githubusercontent.com/DefectDojo/django-DefectDojo/master/dojo/static/dojo/img/chop.png",
+            "text": "{% trans "An engagement has been closed" %}.",
+            "facts": [
+                {
+                    "name": "{% trans "Product" %}:",
+                    "value": "{{ engagement.product.name }}"
+                },
+                {
+                    "name": "{% trans "Engagement" %}:",
+                    "value": "{{ engagement.name }}"
+                }
+            ]
+        }
+        {% if system_settings.disclaimer and system_settings.disclaimer.strip %}
+            ,{
+                "activityTitle": "{% trans "Disclaimer" %}",
+                "text": "{{ system_settings.disclaimer }}"
+            }
+        {% endif %}
+
+    ],
+    "potentialAction": [
+        {
+            "@type": "OpenUri",
+            "name": "{% trans "View Engagement" %}",
+            "targets": [
+                {
+                    "os": "default",
+                    "uri": "{{ url|full_url }}"
+                }
+            ]
+        }
+    ]
+}

dojo/templates/notifications/slack/engagement_closed.tpl

@@ -0,0 +1,10 @@
+{% load i18n %}
+{% load display_tags %}
+{% blocktranslate trimmed with name=engagement.name eng_product=engagement.product eng_url=url|full_url %}
+The engagement "{{ name }}" has been closed in the product "{{ eng_product }}". It can be viewed here: {{ eng_url }}
+{% endblocktranslate %}
+{% if system_settings.disclaimer and system_settings.disclaimer.strip %}
+
+    {% trans "Disclaimer" %}:
+    {{ system_settings.disclaimer }}
+{% endif %}

dojo/templatetags/display_tags.py

@@ -781,7 +781,12 @@ def vulnerability_url(vulnerability_id):
     for key in settings.VULNERABILITY_URLS:
         if vulnerability_id.upper().startswith(key):
             if "&&" in settings.VULNERABILITY_URLS[key]:
-                return settings.VULNERABILITY_URLS[key].split("&&")[0] + str(vulnerability_id) + settings.VULNERABILITY_URLS[key].split("&&")[1]
+                # Process specific keys specially if need
+                if key in ["CAPEC", "CWE"]:
+                    vuln_id = str(vulnerability_id).replace(f"{key}-", "")
+                else:
+                    vuln_id = str(vulnerability_id)
+                return f'{settings.VULNERABILITY_URLS[key].split("&&")[0]}{vuln_id}{settings.VULNERABILITY_URLS[key].split("&&")[1]}'
             return settings.VULNERABILITY_URLS[key] + str(vulnerability_id)
     return ""
 

dojo/tools/acunetix/parse_acunetix360_json.py

@@ -15,7 +15,7 @@ def get_findings(self, filename, test):
         dupes = {}
         data = json.load(filename)
         dupes = {}
-        scan_date = parser.parse(data["Generated"])
+        scan_date = parser.parse(data["Generated"], dayfirst=True)
         text_maker = html2text.HTML2Text()
         text_maker.body_width = 0
         for item in data["Vulnerabilities"]:
@@ -96,7 +96,7 @@ def get_findings(self, filename, test):
             finding.unsaved_req_resp = [{"req": request, "resp": response}]
             finding.unsaved_endpoints = [Endpoint.from_uri(url)]
             if item.get("FirstSeenDate"):
-                parseddate = parser.parse(item["FirstSeenDate"])
+                parseddate = parser.parse(item["FirstSeenDate"], dayfirst=True)
                 finding.date = parseddate
             if dupe_key in dupes:
                 find = dupes[dupe_key]

dojo/tools/acunetix/parse_acunetix_xml.py

@@ -26,7 +26,7 @@ def get_findings(self, filename, test):
             # get report date
             if scan.findtext("StartTime") and "" != scan.findtext("StartTime"):
                 report_date = dateutil.parser.parse(
-                    scan.findtext("StartTime"),
+                    scan.findtext("StartTime"), dayfirst=True,
                 ).date()
             for item in scan.findall("ReportItems/ReportItem"):
                 finding = Finding(

dojo/tools/awssecurityhub/compliance.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime
 
 from dojo.models import Finding
 
@@ -31,11 +31,11 @@ def get_item(self, finding: dict, test):
             active = False
             if finding.get("LastObservedAt", None):
                 try:
-                    mitigated = datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%S.%fZ")
+                    mitigated = datetime.datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%S.%fZ")
                 except Exception:
-                    mitigated = datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%fZ")
+                    mitigated = datetime.datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%fZ")
             else:
-                mitigated = datetime.utcnow()
+                mitigated = datetime.datetime.now(datetime.UTC)
         else:
             mitigated = None
             is_Mitigated = False

dojo/tools/awssecurityhub/guardduty.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime
 
 from dojo.models import Endpoint, Finding
 
@@ -25,11 +25,11 @@ def get_item(self, finding: dict, test):
             is_Mitigated = True
             if finding.get("LastObservedAt", None):
                 try:
-                    mitigated = datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%S.%fZ")
+                    mitigated = datetime.datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%S.%fZ")
                 except Exception:
-                    mitigated = datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%fZ")
+                    mitigated = datetime.datetime.strptime(finding.get("LastObservedAt"), "%Y-%m-%dT%H:%M:%fZ")
             else:
-                mitigated = datetime.utcnow()
+                mitigated = datetime.datetime.now(datetime.UTC)
         description = f"This is a GuardDuty Finding\n{finding.get('Description', '')}" + "\n"
         description += f"**AWS Finding ARN:** {finding_id}\n"
         if finding.get("SourceUrl"):