Merge branch 'dev' into fix_integrity_settingsdistpy

DefectDojo · Nov 15, 2024 · f4358b6 · f4358b6
2 parents 485d3da + 47fdf73
commit f4358b6
Show file tree

Hide file tree

Showing 1,118 changed files with 514,809 additions and 29,637 deletions.
diff --git a/.flake8 b/.flake8
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -7,7 +7,7 @@ assignees: ''
 
 ---
 **Slack us first!**
-The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp-slack.herokuapp.com/)
+The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp.org/slack/invite)
 If you're confident you've found a bug, or are allergic to Slack, you can submit an issue anyway.
 
 **Be informative**
@@ -36,7 +36,7 @@ A clear and concise description of what you expected to happen.
  - DefectDojo version (see footer) or commit message: [use `git show -s --format="[%ci] %h: %s [%d]"`]
 
 **Logs** 
-Use `docker-compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
+Use `docker compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
 
 **Sample scan files**
 If applicable, add sample scan files to help reproduce your problem.

diff --git a/.github/ISSUE_TEMPLATE/support_request.md b/.github/ISSUE_TEMPLATE/support_request.md
@@ -7,7 +7,7 @@ assignees: ''
 
 ---
 **Slack us first!**
-The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp-slack.herokuapp.com/)
+The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp.org/slack/invite)
 If you're confident you've found a bug, or are allergic to Slack, you can submit an issue anyway.
 
 **Be informative**
@@ -36,7 +36,7 @@ A clear and concise description of what you expected to happen.
  - DefectDojo version (see footer) or commit message: [use `git show -s --format="[%ci] %h: %s [%d]"`]
 
 **Logs** 
-Use `docker-compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
+Use `docker compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
 
 **Sample scan files**
 If applicable, add sample scan files to help reproduce your problem.

diff --git a/.github/renovate.json b/.github/renovate.json
@@ -6,7 +6,8 @@
   "dependencyDashboardApproval": false,
   "baseBranches": ["dev"],
   "rebaseWhen": "conflicted",
-  "ignorePaths": ["requirements.txt", "components/package.json", "components/package-lock.json", "dojo/components/yarn.lock", "dojo/components/package.json", "Dockerfile**"],
+  "ignorePaths": ["requirements.txt", "requirements-lint.txt", "components/package.json", "components/package-lock.json", "dojo/components/yarn.lock", "dojo/components/package.json", "Dockerfile**"],
+  "ignoreDeps": [],
   "packageRules": [{
     "packagePatterns": ["*"],
     "commitMessageExtra": "from {{currentVersion}} to {{#if isMajor}}v{{{newMajor}}}{{else}}{{#if isSingleVersion}}v{{{toVersion}}}{{else}}{{{newValue}}}{{/if}}{{/if}}",

diff --git a/.github/workflows/build-docker-images-for-testing.yml b/.github/workflows/build-docker-images-for-testing.yml
@@ -35,20 +35,22 @@ jobs:
 
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
+        timeout-minutes: 10
+        env:
+          DOCKER_BUILD_CHECKS_ANNOTATIONS: false
         with:
           context: .
           push: false
           tags: defectdojo/defectdojo-${{ matrix.docker-image }}:${{ matrix.os }}
           file: Dockerfile.${{ matrix.docker-image }}-${{ matrix.os }}
           outputs: type=docker,dest=${{ matrix.docker-image }}-${{ matrix.os }}_img
-          cache-from: type=gha,scope=${{ matrix.docker-image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.docker-image }}
-
+
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
-        uses:  actions/upload-artifact@v3
+        timeout-minutes: 10
+        uses:  actions/upload-artifact@v4
         with:
-          name: ${{ matrix.docker-image }}
+          name: built-docker-image-${{ matrix.docker-image }}-${{ matrix.os }}
           path: ${{ matrix.docker-image }}-${{ matrix.os }}_img
-          retention-days: 1
+          retention-days: 1
diff --git a/.github/workflows/fetch-oas.yml b/.github/workflows/fetch-oas.yml
@@ -33,7 +33,7 @@ jobs:
              docker images
 
       - name: Start Dojo
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps -d postgres nginx uwsgi
+        run: docker compose up --no-deps -d postgres nginx uwsgi
         env:
           DJANGO_VERSION: ${{ env.release_version }}-alpine
           NGINX_VERSION: ${{ env.release_version }}-alpine
@@ -44,14 +44,14 @@ jobs:
 
       - name: Logs
         if: always()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env logs --tail="2500"
+        run: docker compose logs --tail="2500"
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env down
+        run: docker compose down
 
       - name: Upload oas.${{ matrix.file-type }} as artifact
-        uses:  actions/upload-artifact@v3
+        uses:  actions/upload-artifact@v4
         with:
           name: oas-${{ matrix.file-type }}
           path: oas.${{ matrix.file-type }}

diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -36,7 +36,6 @@ jobs:
           "tests/tool_config.py",
           "openapi-validatator",
         ]
-        profile: ["mysql-rabbitmq", "postgres-redis"]
         os: [alpine, debian]
       fail-fast: false
 
@@ -46,49 +45,47 @@ jobs:
 
       # load docker images from build jobs
       - name: Load images from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
+        with:
+          path: built-docker-image
+          pattern: built-docker-image-*
+          merge-multiple: true
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
-             docker load -i nginx/nginx-${{ matrix.os }}_img
-             docker load -i django/django-${{ matrix.os }}_img
-             docker load -i integration-tests/integration-tests-debian_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/integration-tests-debian_img
              docker images
 
       - name: Set integration-test mode
         run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
 
-      # phased startup with MySQL and RabbitMQ so we can use the exit code from integrationtest container
-      - name: Start Dojo MySQL + RabbitMQ
-        if: matrix.profile == 'mysql-rabbitmq'
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d mysql nginx celerybeat celeryworker mailhog uwsgi rabbitmq
-        env:
-          DJANGO_VERSION: ${{ matrix.os }}
-          NGINX_VERSION: ${{ matrix.os }}
-
-      - name: Start Dojo PostgreSQL + Redis
-        if: matrix.profile == 'postgres-redis'
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis
+      - name: Start Dojo 
+        run: docker compose up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Initialize
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from initializer initializer
+        timeout-minutes: 10
+        run: docker compose up --no-deps --exit-code-from initializer initializer
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Integration tests
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from integration-tests integration-tests
+        timeout-minutes: 10
+        run: docker compose up --no-deps --exit-code-from integration-tests integration-tests
         env:
           DD_INTEGRATION_TEST_FILENAME: ${{ matrix.test-case }}
           INTEGRATION_TESTS_VERSION: debian
 
       - name: Logs
         if: always()
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env logs --tail="2500"
+        run: docker compose logs --tail="2500"
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env down
+        run: docker compose down
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -5,28 +5,14 @@ on:
 
 env:
   DD_HOSTNAME: defectdojo.default.minikube.local
-  HELM_RABBIT_BROKER_SETTINGS: " \
-    --set redis.enabled=false \
-    --set rabbitmq.enabled=true \
-    --set celery.broker=rabbitmq \
-    --set createRabbitMqSecret=true \
-    "
   HELM_REDIS_BROKER_SETTINGS: " \
     --set redis.enabled=true \
-    --set rabbitmq.enabled=false \
     --set celery.broker=redis \
     --set createRedisSecret=true \
     "
-  HELM_MYSQL_DATABASE_SETTINGS: " \
-    --set database=mysql \
-    --set postgresql.enabled=false \
-    --set mysql.enabled=true \
-    --set createMysqlSecret=true \
-    "
   HELM_PG_DATABASE_SETTINGS: " \
     --set database=postgresql \
     --set postgresql.enabled=true \
-    --set mysql.enabled=false \
     --set createPostgresqlSecret=true \
    "
 jobs:
@@ -42,33 +28,16 @@ jobs:
           # are tested (https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#available-versions)
           - databases: pgsql
             brokers: redis
-            k8s: 'v1.26.11'
-            os: debian
-          - databases: mysql
-            brokers: rabbit
-            k8s: 'v1.26.11'
+            k8s: 'v1.30.3'
             os: debian
-          - databases: pgsql
-            brokers: rabbit
-            k8s: 'v1.29.2'
-            os: debian
-          - databases: mysql
-            brokers: redis
-            k8s: 'v1.29.2'
-            os: debian
-          - databases: pgsql
-            brokers: rabbit
-            k8s: 'v1.29.2'
-            os: alpine
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup Minikube
-        uses: manusa/actions-setup-minikube@v2.11.0
+        uses: manusa/actions-setup-minikube@v2.13.0
         with:
-          minikube version: 'v1.31.2'
+          minikube version: 'v1.33.1'
           kubernetes version: ${{ matrix.k8s }}
           driver: docker
           start args: '--addons=ingress --cni calico'
@@ -79,13 +48,18 @@ jobs:
           minikube status
 
       - name: Load images from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
+        with:
+          path: built-docker-image
+          pattern: built-docker-image-*
+          merge-multiple: true
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              eval $(minikube docker-env)
-             docker load -i nginx/nginx-${{ matrix.os }}_img
-             docker load -i django/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}_img
              docker images
 
       - name: Configure HELM repos
@@ -98,31 +72,33 @@ jobs:
         id: set
         run: |-
               echo "pgsql=${{ env.HELM_PG_DATABASE_SETTINGS }}" >> $GITHUB_ENV
-              echo "mysql=${{ env.HELM_MYSQL_DATABASE_SETTINGS }}" >> $GITHUB_ENV
               echo "redis=${{ env.HELM_REDIS_BROKER_SETTINGS }}" >> $GITHUB_ENV
-              echo "rabbit=${{ env.HELM_RABBIT_BROKER_SETTINGS }}" >> $GITHUB_ENV
 
-      - name: Deploying Djano application with ${{ matrix.databases }} ${{ matrix.brokers }}
+      - name: Deploying Django application with ${{ matrix.databases }} ${{ matrix.brokers }}
+        timeout-minutes: 15 
         run: |-
              helm install \
                --timeout 800s \
+               --wait \
+               --wait-for-jobs \
                defectdojo \
                ./helm/defectdojo \
                --set django.ingress.enabled=true \
                --set imagePullPolicy=Never \
                ${{ env[matrix.databases] }} \
                ${{ env[matrix.brokers] }} \
                --set createSecret=true \
-               --set tag=${{ matrix.os }} \
-               # --set imagePullSecrets=defectdojoregistrykey
+               --set tag=${{ matrix.os }}
 
       - name: Check deployment status
+        if: always()
         run: |-
-             kubectl get pods
-             kubectl get ingress
-             kubectl get services
+             kubectl get all,ingress  # all = pods, services, deployments, replicasets, statefulsets, jobs
+             helm status defectdojo
+             helm history defectdojo
 
       - name: Check Application
+        timeout-minutes: 10
         run: |-
              to_complete () {
                kubectl wait --for=$1 $2 --timeout=500s --selector=$3 2>/tmp/test || true