Merge branch 'kiuwan-sca' of github.com:mwager/django-DefectDojo into…

… kiuwan-sca # By dependabot[bot] (13) and others # Via GitHub * 'kiuwan-sca' of github.com:mwager/django-DefectDojo: (39 commits) Deprecate Python-jose and migrate okta to python_social_auth (DefectDojo#10117) fix: dockerfile warnings (DefectDojo#10505) Ruff: Add and fix Q000 (DefectDojo#10095) Fix(django): Upgrade of 4.2 (DefectDojo#10553) fix(deps): build python psycopg3 dependency instead of use the pre-build binary (DefectDojo#10491) Bump coverage from 7.5.4 to 7.6.0 (DefectDojo#10560) Bump asteval from 1.0.0 to 1.0.1 (DefectDojo#10561) Bump djangorestframework from 3.14.0 to 3.15.2 (DefectDojo#10431) Bump boto3 from 1.34.142 to 1.34.143 (DefectDojo#10558) Bump django-debug-toolbar from 4.4.5 to 4.4.6 (DefectDojo#10557) Bump boto3 from 1.34.141 to 1.34.142 (DefectDojo#10551) Bump packageurl-python from 0.15.2 to 0.15.3 (DefectDojo#10541) Bump boto3 from 1.34.140 to 1.34.141 (DefectDojo#10542) Update helm lock file Update versions in application files Update versions in application files API: Convert get_filterset calls to get_queryset (DefectDojo#10543) Bump django-debug-toolbar from 4.4.4 to 4.4.5 (DefectDojo#10527) Fix ruff Ruff fix ... # Conflicts: # dojo/settings/.settings.dist.py.sha256sum
mwager · Jul 16, 2024 · b225ff2 · b225ff2
2 parents 7410770 + 0fd219c
commit b225ff2
Show file tree

Hide file tree

Showing 382 changed files with 15,381 additions and 15,431 deletions.
diff --git a/.github/renovate.json b/.github/renovate.json
@@ -6,7 +6,11 @@
   "dependencyDashboardApproval": false,
   "baseBranches": ["dev"],
   "rebaseWhen": "conflicted",
-  "ignorePaths": ["requirements.txt", "components/package.json", "components/package-lock.json", "dojo/components/yarn.lock", "dojo/components/package.json", "Dockerfile**"],
+  "ignorePaths": ["requirements.txt", "requirements-lint.txt", "components/package.json", "components/package-lock.json", "dojo/components/yarn.lock", "dojo/components/package.json", "Dockerfile**"],
+  "ignoreDeps": [
+    "mysql",
+    "rabbitmq"
+  ],
   "packageRules": [{
     "packagePatterns": ["*"],
     "commitMessageExtra": "from {{currentVersion}} to {{#if isMajor}}v{{{newMajor}}}{{else}}{{#if isSingleVersion}}v{{{toVersion}}}{{else}}{{{newValue}}}{{/if}}{{/if}}",

diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
diff --git a/.github/workflows/release-1-create-pr.yml b/.github/workflows/release-1-create-pr.yml
@@ -51,9 +51,9 @@ jobs:
 
       - name: Update version numbers in key files
         run: |
-          sed -ri "s/__version__ = '.*'/__version__ = '${{ github.event.inputs.release_number }}'/" dojo/__init__.py
-          sed -ri "s/\"version\": \".*\"/\"version\": \"${{ github.event.inputs.release_number }}\"/" components/package.json
-          sed -ri "s/appVersion: \".*\"/appVersion: \"${{ github.event.inputs.release_number }}\"/" helm/defectdojo/Chart.yaml
+          sed -ri 's/__version__ = ".*"/__version__ = "${{ github.event.inputs.release_number }}"/' dojo/__init__.py
+          sed -ri 's/"version": ".*"/"version": "${{ github.event.inputs.release_number }}"/' components/package.json
+          sed -ri 's/appVersion: ".*"/appVersion: "${{ github.event.inputs.release_number }}"/' helm/defectdojo/Chart.yaml
 
           if grep "\-dev" helm/defectdojo/Chart.yaml; then
               echo "x.y.z-dev found in Chart.yaml, probably releasing a new minor version"

diff --git a/.github/workflows/release-3-master-into-dev.yml b/.github/workflows/release-3-master-into-dev.yml
@@ -44,9 +44,9 @@ jobs:
 
       - name: Update version numbers in key files
         run: |
-          sed -ri "s/__version__ = '.*'/__version__ = '${{ github.event.inputs.release_number_dev }}'/" dojo/__init__.py
-          sed -ri "s/appVersion: \".*\"/appVersion: \"${{ github.event.inputs.release_number_dev }}\"/" helm/defectdojo/Chart.yaml
-          sed -ri "s/\"version\": \".*\"/\"version\": \"${{ github.event.inputs.release_number_dev }}\"/" components/package.json
+          sed -ri 's/__version__ = ".*"/__version__ = "${{ github.event.inputs.release_number_dev }}"/' dojo/__init__.py
+          sed -ri 's/"version": ".*"/"version": "${{ github.event.inputs.release_number_dev }}"/' components/package.json
+          sed -ri 's/appVersion: ".*"/appVersion: "${{ github.event.inputs.release_number_dev }}"/' helm/defectdojo/Chart.yaml
           CURRENT_CHART_VERSION=$(grep -oP 'version: (\K\S*)?' helm/defectdojo/Chart.yaml | head -1)
           sed -ri "0,/version/s/version: \S+/$(echo "version: $CURRENT_CHART_VERSION" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')-dev/" helm/defectdojo/Chart.yaml
       

diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml
@@ -2,31 +2,13 @@ name: Ruff Linter
 
 on:
   workflow_dispatch:
-  pull_request_target:
   push:
-
+  pull_request:
 jobs:
   ruff-linting:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-        uses: actions/checkout@v4
-        # by default the pull_requst_target event checks out the base branch, i.e. dev
-        # so we need to explicitly checkout the head of the PR
-        # we use fetch-depth 0 to make sure the full history is checked out and we can compare against
-        # the base commit (branch) of the PR
-        # more info https://github.community/t/github-actions-are-severely-limited-on-prs/18179/16
-        # we checkout merge_commit here as this contains all new code from dev also. we don't need to compare against base_commit
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
-          # repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Checkout
-        # for non PR runs we just checkout the default, which is a sha on a branch probably
-        if: github.event_name != 'pull_request' && github.event_name != 'pull_request_target'
         uses: actions/checkout@v4
 
       - name: Install Ruff Linter

diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine
@@ -5,8 +5,8 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
-FROM base as build
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apk update && \
@@ -16,13 +16,15 @@ RUN \
     bind-tools \
     mysql-client \
     mariadb-dev \
-    postgresql14-client \
+    postgresql16-client \
     xmlsec \
     git \
     util-linux \
     curl-dev \
     openssl \
     libffi-dev \
+    python3-dev \
+    libpq-dev \
     && \
     rm -rf /var/cache/apk/* && \
   true
@@ -31,12 +33,12 @@ COPY requirements.txt ./
 # https://github.com/unbit/uwsgi/issues/1318#issuecomment-542238096
 RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM base as django-alpine
+FROM base AS django-alpine
 WORKDIR /app
 ARG uid=1001
 ARG gid=1337
 ARG appuser=defectdojo
-ENV appuser ${appuser}
+ENV appuser=${appuser}
 RUN \
   apk update && \
   apk add --no-cache \
@@ -49,7 +51,7 @@ RUN \
     xmlsec \
     git \
     util-linux \
-    postgresql14-client \
+    postgresql16-client \
     curl-dev \
     openssl \
     # needed for integration-tests
@@ -136,5 +138,5 @@ ENV \
   DD_UWSGI_NUM_OF_THREADS="2"
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]
 
-FROM django-alpine as django-unittests
-COPY unittests/ ./unittests/
+FROM django-alpine AS django-unittests
+COPY unittests/ ./unittests/
diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian
@@ -5,8 +5,8 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as base
-FROM base as build
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -32,12 +32,12 @@ COPY requirements.txt ./
 # https://github.com/unbit/uwsgi/issues/1318#issuecomment-542238096
 RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM base as django
+FROM base AS django
 WORKDIR /app
 ARG uid=1001
 ARG gid=1337
 ARG appuser=defectdojo
-ENV appuser ${appuser}
+ENV appuser=${appuser}
 RUN \
   apt-get -y update && \
   # ugly fix to install postgresql-client without errors
@@ -141,5 +141,5 @@ ENV \
   DD_UWSGI_NUM_OF_THREADS="2"
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]
 
-FROM django as django-unittests
+FROM django AS django-unittests
 COPY unittests/ ./unittests/
diff --git a/Dockerfile.integration-tests-debian b/Dockerfile.integration-tests-debian
@@ -1,8 +1,8 @@
 
 # code: language=Dockerfile
 
-FROM openapitools/openapi-generator-cli:v7.6.0@sha256:f86ca824293602b71b9b66683cc0011f8ff963858bd853621c554ff5cc7dd1d5 as openapitools
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as build
+FROM openapitools/openapi-generator-cli:v7.7.0@sha256:99924315933d49e7b33a7d2074bb2b64fc8def8f74519939036e24eb48f00336 AS openapitools
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -76,4 +76,4 @@ ENV \
   DD_ADMIN_USER=admin \
   DD_ADMIN_PASSWORD='' \
   DD_BASE_URL="http://localhost:8080/"
-CMD ["/entrypoint-integration-tests.sh"]
+CMD ["/entrypoint-integration-tests.sh"]
diff --git a/Dockerfile.nginx-alpine b/Dockerfile.nginx-alpine
@@ -5,8 +5,8 @@
 # Dockerfile.django-alpine to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
-FROM base as build
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apk update && \
@@ -16,13 +16,15 @@ RUN \
     bind-tools \
     mysql-client \
     mariadb-dev \
-    postgresql14-client \
+    postgresql16-client \
     xmlsec \
     git \
     util-linux \
     curl-dev \
     openssl \
     libffi-dev \
+    python3-dev \
+    libpq-dev \
     && \
     rm -rf /var/cache/apk/* && \
   true
@@ -34,7 +36,7 @@ RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 FROM build AS collectstatic
 
 # Node installation from https://github.com/nodejs/docker-node
-ENV NODE_VERSION 20.11.0
+ENV NODE_VERSION=20.11.0
 
 RUN addgroup -g 1000 node \
     && adduser -u 1000 -G node -s /bin/sh -D node \
@@ -103,7 +105,7 @@ RUN addgroup -g 1000 node \
   && node --version \
   && npm --version
 
-ENV YARN_VERSION 1.22.19
+ENV YARN_VERSION=1.22.19
 
 RUN apk add --no-cache --virtual .build-deps-yarn curl gnupg tar \
   && for key in \

diff --git a/Dockerfile.nginx-debian b/Dockerfile.nginx-debian
@@ -5,8 +5,8 @@
 # Dockerfile.django-debian to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as base
-FROM base as build
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apt-get -y update && \

diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml
@@ -1,5 +1,4 @@
 ---
-version: '3.8'
 services:
   nginx:
     image: busybox:1.36.1-musl

diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml
@@ -15,6 +15,7 @@ services:
     environment:
       PYTHONWARNINGS: error  # We are strict about Warnings during testing
       DD_DEBUG: 'True'
+      DD_LOG_LEVEL: 'ERROR'
       DD_TEST_DATABASE_NAME: ${DD_TEST_DATABASE_NAME}
       DD_DATABASE_NAME: ${DD_TEST_DATABASE_NAME}
       DD_DATABASE_ENGINE: ${DD_DATABASE_ENGINE}

diff --git a/docker/entrypoint-unit-tests-devDocker.sh b/docker/entrypoint-unit-tests-devDocker.sh
@@ -53,7 +53,9 @@ EOF
 
 echo "Unit Tests"
 echo "------------------------------------------------------------"
-python3 manage.py test unittests -v 3 --keepdb --no-input
+
+python3 manage.py test unittests -v 3 --keepdb --no-input --failfast --shuffle --parallel --exclude-tag="non-parallel"
+python3 manage.py test unittests -v 3 --keepdb --no-input --failfast --shuffle --tag="non-parallel"
 
 # you can select a single file to "test" unit tests
 # python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3

diff --git a/docker/entrypoint-unit-tests.sh b/docker/entrypoint-unit-tests.sh
@@ -79,4 +79,6 @@ python3 manage.py migrate
 
 echo "Unit Tests"
 echo "------------------------------------------------------------"
-python3 manage.py test unittests -v 3 --keepdb --no-input
+
+python3 manage.py test unittests -v 3 --keepdb --no-input --failfast --shuffle --parallel --exclude-tag="non-parallel"
+python3 manage.py test unittests -v 3 --keepdb --no-input --failfast --shuffle --tag="non-parallel"
diff --git a/docs/config.dev.toml b/docs/config.dev.toml
@@ -85,7 +85,8 @@ weight = 1
       # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html
       style = "dracula"
       # Uncomment if you want your chosen highlight style used for code blocks without a specified language
-      guessSyntax = "true"
+      # Do not uncomment otherwise it breaks mermaid
+      # guessSyntax = "true"
 
 # Everything below this are Site Params
 
@@ -198,3 +199,6 @@ enable = false
 	url = "https://owasp.slack.com/archives/C014H3ZV9U6"
 	icon = "fab fa-slack"
         desc = "Chat with other project developers"
+
+[params.mermaid]
+enable = true
diff --git a/docs/config.master.toml b/docs/config.master.toml
@@ -85,7 +85,8 @@ weight = 1
       # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html
       style = "dracula"
       # Uncomment if you want your chosen highlight style used for code blocks without a specified language
-      guessSyntax = "true"
+      # Do not uncomment otherwise it breaks mermaid
+      # guessSyntax = "true"
 
 # Everything below this are Site Params
 
@@ -198,3 +199,6 @@ enable = false
 	url = "https://owasp.slack.com/archives/C014H3ZV9U6"
 	icon = "fab fa-slack"
         desc = "Chat with other project developers"
+
+[params.mermaid]
+enable = true
diff --git a/docs/content/en/getting_started/upgrading/2.36.md b/docs/content/en/getting_started/upgrading/2.36.md
@@ -2,6 +2,15 @@
 title: 'Upgrading to DefectDojo Version 2.36.x'
 toc_hide: true
 weight: -20240603
-description: No special instructions.
+description: Breaking Change for HELM deployments with PostgreSQL
 ---
-There are no special instructions for upgrading to 2.36.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.36.0) for the contents of the release.
+
+Previous HELM deployments (HELM chart `<=1.6.136`, DefectDojo `<=2.35.4`) used a pinned version of PostgreSQL in versions `11.x`. These are incompatible with Django in version `4.2` (used from DefectDojo version `3.36.0`; HELM chart `1.6.137`). Because of this, it is necessary to upgrade PostgreSQL to version `12.x` or higher. DefectDojo in version `3.36.1` (HELM chart `1.6.138`) uses this new version of PostgreSQL.
+
+Unfortunately, an upgrade of PostgreSQL is not enough because PostgreSQL does not support automatic migration of data structures in the filesystem. Because of this, migration is needed. There are different ways (many of them similar to migration between different database backends (e.g. from MySQL to PostgreSQL)). Please find inspiration and the best fitting way for you in:
+
+- https://github.com/DefectDojo/django-DefectDojo/discussions/9480
+- https://owasp.slack.com/archives/C2P5BA8MN/p1717610931766739?thread_ts=1717587117.831149&cid=C2P5BA8MN
+- https://dev.to/jkostolansky/how-to-upgrade-postgresql-from-11-to-12-2la6
+
+There are no other special instructions for upgrading to 2.36.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.36.0) for the contents of the release.
diff --git a/docs/content/en/integrations/ldap-authentication.md b/docs/content/en/integrations/ldap-authentication.md
@@ -41,7 +41,7 @@ Please check for the latest version of these requirements at the time of impleme
 
 Otherwise add the following to requirements.txt:
 
-```
+```python
 python-ldap==3.4.2
 django-auth-ldap==4.1.0
 ```
@@ -119,7 +119,7 @@ Read the docs for Django Authentication with LDAP here: https://django-auth-ldap
 In order to pass the variables to the settings.dist.py file via docker, it's a good idea to add these to the docker-compose file.
 
 You can do this by adding the following variables to the environment section for the uwsgi image:
-```
+```yaml
 DD_LDAP_SERVER_URI: "${DD_LDAP_SERVER_URI:-ldap://ldap.example.com}"
 DD_LDAP_BIND_DN: "${DD_LDAP_BIND_DN:-}"
 DD_LDAP_BIND_PASSWORD: "${DD_LDAP_BIND_PASSWORD:-}"

diff --git a/docs/content/en/integrations/parsers/file/fortify.md b/docs/content/en/integrations/parsers/file/fortify.md
@@ -20,6 +20,6 @@ per category. To get all issues, copy the [DefaultReportDefinitionAllIssues.xml]
 
 Once this is complete, you can run the following command on your .fpr file to generate the
 required XML:
-```
+```bash
 ./path/to/ReportGenerator -format xml -f /path/to/output.xml -source /path/to/downloaded/artifact.fpr -template DefaultReportDefinitionAllIssues.xml
 ```