Merge pull request #10377 from DefectDojo/master-into-dev/2.35.2-2.36…

….0-dev Release: Merge back 2.35.2 into dev from: master-into-dev/2.35.2-2.36.0-dev
DefectDojo · Jun 10, 2024 · 6dbe4ac · 6dbe4ac
2 parents 6b9fd8c + bb7da36
commit 6dbe4ac
Show file tree

Hide file tree

Showing 19 changed files with 52 additions and 23 deletions.
diff --git a/.github/workflows/build-docker-images-for-testing.yml b/.github/workflows/build-docker-images-for-testing.yml
@@ -36,6 +36,7 @@ jobs:
       - name: Build
         id: docker_build
         uses: docker/build-push-action@v5
+        timeout-minutes: 10
         with:
           context: .
           push: false
@@ -47,6 +48,7 @@ jobs:
 
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
+        timeout-minutes: 10
         uses:  actions/upload-artifact@v3
         with:
           name: ${{ matrix.docker-image }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -49,6 +49,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              docker load -i nginx/nginx-${{ matrix.os }}_img
              docker load -i django/django-${{ matrix.os }}_img
@@ -74,12 +75,14 @@ jobs:
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Initialize
+        timeout-minutes: 10
         run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from initializer initializer
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Integration tests
+        timeout-minutes: 10
         run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from integration-tests integration-tests
         env:
           DD_INTEGRATION_TEST_FILENAME: ${{ matrix.test-case }}

diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -82,6 +82,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              eval $(minikube docker-env)
              docker load -i nginx/nginx-${{ matrix.os }}_img
@@ -103,6 +104,7 @@ jobs:
               echo "rabbit=${{ env.HELM_RABBIT_BROKER_SETTINGS }}" >> $GITHUB_ENV
 
       - name: Deploying Djano application with ${{ matrix.databases }} ${{ matrix.brokers }}
+        timeout-minutes: 10 
         run: |-
              helm install \
                --timeout 800s \
@@ -123,6 +125,7 @@ jobs:
              kubectl get services
 
       - name: Check Application
+        timeout-minutes: 10
         run: |-
              to_complete () {
                kubectl wait --for=$1 $2 --timeout=500s --selector=$3 2>/tmp/test || true

diff --git a/.github/workflows/rest-framework-tests.yml b/.github/workflows/rest-framework-tests.yml
@@ -23,6 +23,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              docker load -i nginx/nginx-${{ matrix.os }}_img
              docker load -i django/django-${{ matrix.os }}_img
@@ -38,6 +39,7 @@ jobs:
 
       # no celery or initializer needed for unit tests
       - name: Unit tests
+        timeout-minutes: 10
         run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps --exit-code-from uwsgi uwsgi
         env:
           DJANGO_VERSION: ${{ matrix.os }}

diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine
@@ -76,6 +76,7 @@ COPY \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian
@@ -81,6 +81,7 @@ COPY \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

diff --git a/Dockerfile.integration-tests-debian b/Dockerfile.integration-tests-debian
@@ -62,6 +62,7 @@ COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/ta
 
 COPY docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/entrypoint-integration-tests.sh \
   /
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -54,6 +54,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -75,6 +76,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -95,6 +97,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -119,6 +122,7 @@ services:
       DD_INITIALIZE: "${DD_INITIALIZE:-true}"
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings

diff --git a/docker/entrypoint-celery-beat.sh b/docker/entrypoint-celery-beat.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+
+. /reach_database.sh
+
 umask 0002
 
 id
@@ -16,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
     rm -f /app/dojo/settings/README.md
 fi
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 # do the check with Django stack

diff --git a/docker/entrypoint-celery-worker.sh b/docker/entrypoint-celery-worker.sh
@@ -4,6 +4,7 @@ umask 0002
 id
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null)
@@ -18,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
     rm -f /app/dojo/settings/README.md
 fi
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

diff --git a/docker/entrypoint-initializer.sh b/docker/entrypoint-initializer.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 initialize_data()
 {
@@ -60,12 +61,7 @@ then
 fi
 echo "Initializing."
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 echo "Checking ENABLE_AUDITLOG"

diff --git a/docker/entrypoint-unit-tests-devDocker.sh b/docker/entrypoint-unit-tests-devDocker.sh
@@ -7,6 +7,7 @@ set -e
 set -v
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 cd /app
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
@@ -15,6 +16,8 @@ unset DD_DATABASE_URL
 # Unset the celery broker URL so that we can force the other DD_CELERY_BROKER settings
 unset DD_CELERY_BROKER_URL
 
+wait_for_database_to_be_reachable
+
 python3 manage.py makemigrations dojo
 python3 manage.py migrate
 

diff --git a/docker/entrypoint-unit-tests.sh b/docker/entrypoint-unit-tests.sh
@@ -8,6 +8,7 @@
 
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 cd /app || exit
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
@@ -22,6 +23,8 @@ unset DD_CELERY_BROKER_URL
 #   cp dojo/settings/settings.dist.py dojo/settings/settings.py
 # fi
 
+wait_for_database_to_be_reachable
+
 python3 manage.py spectacular --fail-on-warn > /dev/null || {
     cat <<-EOF
 

diff --git a/docker/reach_database.sh b/docker/reach_database.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+wait_for_database_to_be_reachable() {
+    echo -n "Waiting for database to be reachable "
+    failure_count=0
+    DD_DATABASE_READINESS_TIMEOUT=${DD_DATABASE_READINESS_TIMEOUT:-30}
+    until echo "select 1;" | python3 manage.py dbshell > /dev/null
+    do 
+    echo -n "."
+    failure_count=$((failure_count + 1)) 
+    sleep 1
+    if [ $DD_DATABASE_READINESS_TIMEOUT = $failure_count ]; then
+        exit 1
+    fi
+    done
+}
diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py
@@ -427,7 +427,7 @@ class Meta:
 
 class UserSerializer(serializers.ModelSerializer):
     date_joined = serializers.DateTimeField(read_only=True)
-    last_login = serializers.DateTimeField(read_only=True)
+    last_login = serializers.DateTimeField(read_only=True, allow_null=True)
     password = serializers.CharField(
         write_only=True,
         style={"input_type": "password"},

diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py
@@ -380,7 +380,7 @@ def async_process_findings(
         # After all tasks have been started, time to pull the results
         logger.info('IMPORT_SCAN: Collecting Findings')
         for results in results_list:
-            serial_new_findings = results.get()
+            serial_new_findings = results
             new_findings += [next(deserialize("json", finding)).object for finding in serial_new_findings]
         logger.info('IMPORT_SCAN: All Findings Collected')
         # Indicate that the test is not complete yet as endpoints will still be rolling in.

diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py
@@ -339,7 +339,7 @@ def async_process_findings(
                 serial_reactivated_findings,
                 serial_findings_to_mitigate,
                 serial_untouched_findings,
-            ) = results.get()
+            ) = results
             new_findings += [
                 next(deserialize("json", finding)).object
                 for finding in serial_new_findings

diff --git a/dojo/settings/settings.py b/dojo/settings/settings.py
@@ -21,4 +21,4 @@
         msg = "Change of 'settings.dist.py' file was detected. It is not allowed to edit this file. " \
             "Any customization of variables need to be done via environmental variables or in 'local_settings.py'. " \
             "For more information check https://documentation.defectdojo.com/getting_started/configuration/ "
-        raise ValueError(msg)
+        sys.exit(msg)
diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 appVersion: "2.36.0-dev"
 description: A Helm chart for Kubernetes to install DefectDojo
 name: defectdojo
-version: 1.6.134-dev
+version: 1.6.135-dev
 icon: https://www.defectdojo.org/img/favicon.ico
 maintainers:
   - name: madchap