From c4e7af78145be7c519cb93f9170a98bdde7bdc04 Mon Sep 17 00:00:00 2001 From: DefectDojo release bot Date: Thu, 6 Jun 2024 21:29:48 +0000 Subject: [PATCH 1/6] Update versions in application files --- components/package.json | 2 +- dojo/__init__.py | 2 +- helm/defectdojo/Chart.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/package.json b/components/package.json index f36765a9a80..688f4f5d52f 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.35.1", + "version": "2.36.0-dev", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/dojo/__init__.py b/dojo/__init__.py index 523f51fdd80..423f4050e5c 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = '2.35.1' +__version__ = '2.36.0-dev' __url__ = 'https://github.com/DefectDojo/django-DefectDojo' __docs__ = 'https://documentation.defectdojo.com' diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 8403b51cc38..56203e30611 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.35.1" +appVersion: "2.36.0-dev" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.6.133 +version: 1.6.134-dev icon: https://www.defectdojo.org/img/favicon.ico maintainers: - name: madchap From d43b39cad94fdb7a87bd52cdcbe75b1cbda670c1 Mon Sep 17 00:00:00 2001 From: Cedric Buissart Date: Mon, 10 Jun 2024 21:26:49 +0200 Subject: [PATCH 2/6] last_login is null when user never logged in from the webUI (#10360) --- dojo/api_v2/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 51dd30fa0b0..fe8fe2d60b7 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -427,7 +427,7 @@ class Meta: class UserSerializer(serializers.ModelSerializer): date_joined = serializers.DateTimeField(read_only=True) - last_login = serializers.DateTimeField(read_only=True) + last_login = serializers.DateTimeField(read_only=True, allow_null=True) password = serializers.CharField( write_only=True, style={"input_type": "password"}, From 751ac4998ba8b8dbd9c67be61fd06fc83dcae0f4 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:41:58 -0500 Subject: [PATCH 3/6] Async Importer: Correct typing bug (#10373) --- dojo/importers/default_importer.py | 2 +- dojo/importers/default_reimporter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index 5cf38f38a73..4591fe3633d 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -380,7 +380,7 @@ def async_process_findings( # After all tasks have been started, time to pull the results logger.info('IMPORT_SCAN: Collecting Findings') for results in results_list: - serial_new_findings = results.get() + serial_new_findings = results new_findings += [next(deserialize("json", finding)).object for finding in serial_new_findings] logger.info('IMPORT_SCAN: All Findings Collected') # Indicate that the test is not complete yet as endpoints will still be rolling in. diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index e313afbd1d7..f2e8b86e517 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -339,7 +339,7 @@ def async_process_findings( serial_reactivated_findings, serial_findings_to_mitigate, serial_untouched_findings, - ) = results.get() + ) = results new_findings += [ next(deserialize("json", finding)).object for finding in serial_new_findings From 4573990bb046dc5275e0347586fc5cdac900c074 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:21:14 -0500 Subject: [PATCH 4/6] Entrypoint Scripts: Add container level breakouts to prevent doom loopings (#10374) * Entrypoint Scripts: Add container level breakouts to prevent doom looping * Add GHA timeouts * Accommodate shell check * Accommodate shell check (the sequel) * Accommodate shell check (the trilogy) * Accommodate shell check (the tv series) * Increase timeout for helm tests * Update to a env var * Adding var --- .../build-docker-images-for-testing.yml | 2 ++ .github/workflows/integration-tests.yml | 3 +++ .github/workflows/k8s-tests.yml | 3 +++ .github/workflows/rest-framework-tests.yml | 2 ++ Dockerfile.django-alpine | 1 + Dockerfile.django-debian | 1 + Dockerfile.integration-tests-debian | 1 + docker-compose.yml | 4 ++++ docker/entrypoint-celery-beat.sh | 10 ++++------ docker/entrypoint-celery-worker.sh | 8 ++------ docker/entrypoint-initializer.sh | 8 ++------ docker/entrypoint-unit-tests-devDocker.sh | 3 +++ docker/entrypoint-unit-tests.sh | 3 +++ docker/reach_database.sh | 16 ++++++++++++++++ dojo/settings/settings.py | 2 +- 15 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 docker/reach_database.sh diff --git a/.github/workflows/build-docker-images-for-testing.yml b/.github/workflows/build-docker-images-for-testing.yml index 7253ba132f7..00ffd8aff16 100644 --- a/.github/workflows/build-docker-images-for-testing.yml +++ b/.github/workflows/build-docker-images-for-testing.yml @@ -36,6 +36,7 @@ jobs: - name: Build id: docker_build uses: docker/build-push-action@v5 + timeout-minutes: 10 with: context: . push: false @@ -47,6 +48,7 @@ jobs: # export docker images to be used in next jobs below - name: Upload image ${{ matrix.docker-image }} as artifact + timeout-minutes: 10 uses: actions/upload-artifact@v3 with: name: ${{ matrix.docker-image }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f573efdcc75..36c951cafec 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -49,6 +49,7 @@ jobs: uses: actions/download-artifact@v3 - name: Load docker images + timeout-minutes: 10 run: |- docker load -i nginx/nginx-${{ matrix.os }}_img docker load -i django/django-${{ matrix.os }}_img @@ -74,12 +75,14 @@ jobs: NGINX_VERSION: ${{ matrix.os }} - name: Initialize + timeout-minutes: 10 run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from initializer initializer env: DJANGO_VERSION: ${{ matrix.os }} NGINX_VERSION: ${{ matrix.os }} - name: Integration tests + timeout-minutes: 10 run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from integration-tests integration-tests env: DD_INTEGRATION_TEST_FILENAME: ${{ matrix.test-case }} diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 50778ecf562..f8c94da168f 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -82,6 +82,7 @@ jobs: uses: actions/download-artifact@v3 - name: Load docker images + timeout-minutes: 10 run: |- eval $(minikube docker-env) docker load -i nginx/nginx-${{ matrix.os }}_img @@ -103,6 +104,7 @@ jobs: echo "rabbit=${{ env.HELM_RABBIT_BROKER_SETTINGS }}" >> $GITHUB_ENV - name: Deploying Djano application with ${{ matrix.databases }} ${{ matrix.brokers }} + timeout-minutes: 10 run: |- helm install \ --timeout 800s \ @@ -123,6 +125,7 @@ jobs: kubectl get services - name: Check Application + timeout-minutes: 10 run: |- to_complete () { kubectl wait --for=$1 $2 --timeout=500s --selector=$3 2>/tmp/test || true diff --git a/.github/workflows/rest-framework-tests.yml b/.github/workflows/rest-framework-tests.yml index d50c837efbe..134fbbef319 100644 --- a/.github/workflows/rest-framework-tests.yml +++ b/.github/workflows/rest-framework-tests.yml @@ -23,6 +23,7 @@ jobs: uses: actions/download-artifact@v3 - name: Load docker images + timeout-minutes: 10 run: |- docker load -i nginx/nginx-${{ matrix.os }}_img docker load -i django/django-${{ matrix.os }}_img @@ -38,6 +39,7 @@ jobs: # no celery or initializer needed for unit tests - name: Unit tests + timeout-minutes: 10 run: docker compose --profile mysql-redis --env-file ./docker/environments/mysql-redis.env up --no-deps --exit-code-from uwsgi uwsgi env: DJANGO_VERSION: ${{ matrix.os }} diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine index 0dc5b45aab8..7d712ad6268 100644 --- a/Dockerfile.django-alpine +++ b/Dockerfile.django-alpine @@ -76,6 +76,7 @@ COPY \ docker/entrypoint-unit-tests-devDocker.sh \ docker/wait-for-it.sh \ docker/secret-file-loader.sh \ + docker/reach_database.sh \ docker/certs/* \ / COPY wsgi.py manage.py docker/unit-tests.sh ./ diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian index 9444da43cf5..8abba29b884 100644 --- a/Dockerfile.django-debian +++ b/Dockerfile.django-debian @@ -81,6 +81,7 @@ COPY \ docker/entrypoint-unit-tests-devDocker.sh \ docker/wait-for-it.sh \ docker/secret-file-loader.sh \ + docker/reach_database.sh \ docker/certs/* \ / COPY wsgi.py manage.py docker/unit-tests.sh ./ diff --git a/Dockerfile.integration-tests-debian b/Dockerfile.integration-tests-debian index c71848a0082..c7f38d4fd00 100644 --- a/Dockerfile.integration-tests-debian +++ b/Dockerfile.integration-tests-debian @@ -62,6 +62,7 @@ COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/ta COPY docker/wait-for-it.sh \ docker/secret-file-loader.sh \ + docker/reach_database.sh \ docker/entrypoint-integration-tests.sh \ / diff --git a/docker-compose.yml b/docker-compose.yml index 80e2d783f7c..d71da59c498 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,7 @@ services: DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" + DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" volumes: - type: bind source: ./docker/extra_settings @@ -75,6 +76,7 @@ services: DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" + DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" volumes: - type: bind source: ./docker/extra_settings @@ -95,6 +97,7 @@ services: DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL} DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" + DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" volumes: - type: bind source: ./docker/extra_settings @@ -119,6 +122,7 @@ services: DD_INITIALIZE: "${DD_INITIALIZE:-true}" DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}" DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}" + DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}" volumes: - type: bind source: ./docker/extra_settings diff --git a/docker/entrypoint-celery-beat.sh b/docker/entrypoint-celery-beat.sh index 5dd6d9d999f..2404ec473f5 100755 --- a/docker/entrypoint-celery-beat.sh +++ b/docker/entrypoint-celery-beat.sh @@ -1,4 +1,7 @@ #!/bin/bash + +. /reach_database.sh + umask 0002 id @@ -16,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then rm -f /app/dojo/settings/README.md fi -echo -n "Waiting for database to be reachable " -until echo "select 1;" | python3 manage.py dbshell > /dev/null -do - echo -n "." - sleep 1 -done +wait_for_database_to_be_reachable echo # do the check with Django stack diff --git a/docker/entrypoint-celery-worker.sh b/docker/entrypoint-celery-worker.sh index 0afd0f91493..e02ec177f5c 100755 --- a/docker/entrypoint-celery-worker.sh +++ b/docker/entrypoint-celery-worker.sh @@ -4,6 +4,7 @@ umask 0002 id . /secret-file-loader.sh +. /reach_database.sh # Allow for bind-mount multiple settings.py overrides FILES=$(ls /app/docker/extra_settings/* 2>/dev/null) @@ -18,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then rm -f /app/dojo/settings/README.md fi -echo -n "Waiting for database to be reachable " -until echo "select 1;" | python3 manage.py dbshell > /dev/null -do - echo -n "." - sleep 1 -done +wait_for_database_to_be_reachable echo if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then diff --git a/docker/entrypoint-initializer.sh b/docker/entrypoint-initializer.sh index 6e09353103e..c6f86970d89 100755 --- a/docker/entrypoint-initializer.sh +++ b/docker/entrypoint-initializer.sh @@ -1,6 +1,7 @@ #!/bin/bash . /secret-file-loader.sh +. /reach_database.sh initialize_data() { @@ -60,12 +61,7 @@ then fi echo "Initializing." -echo -n "Waiting for database to be reachable " -until echo "select 1;" | python3 manage.py dbshell > /dev/null -do - echo -n "." - sleep 1 -done +wait_for_database_to_be_reachable echo echo "Checking ENABLE_AUDITLOG" diff --git a/docker/entrypoint-unit-tests-devDocker.sh b/docker/entrypoint-unit-tests-devDocker.sh index 28529d3960a..c590974b1b4 100755 --- a/docker/entrypoint-unit-tests-devDocker.sh +++ b/docker/entrypoint-unit-tests-devDocker.sh @@ -7,6 +7,7 @@ set -e set -v . /secret-file-loader.sh +. /reach_database.sh cd /app # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py) @@ -15,6 +16,8 @@ unset DD_DATABASE_URL # Unset the celery broker URL so that we can force the other DD_CELERY_BROKER settings unset DD_CELERY_BROKER_URL +wait_for_database_to_be_reachable + python3 manage.py makemigrations dojo python3 manage.py migrate diff --git a/docker/entrypoint-unit-tests.sh b/docker/entrypoint-unit-tests.sh index 0e0533f34ab..6c45ce489d6 100755 --- a/docker/entrypoint-unit-tests.sh +++ b/docker/entrypoint-unit-tests.sh @@ -8,6 +8,7 @@ . /secret-file-loader.sh +. /reach_database.sh cd /app || exit # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py) @@ -22,6 +23,8 @@ unset DD_CELERY_BROKER_URL # cp dojo/settings/settings.dist.py dojo/settings/settings.py # fi +wait_for_database_to_be_reachable + python3 manage.py spectacular --fail-on-warn > /dev/null || { cat <<-EOF diff --git a/docker/reach_database.sh b/docker/reach_database.sh new file mode 100644 index 00000000000..819f69282e3 --- /dev/null +++ b/docker/reach_database.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +wait_for_database_to_be_reachable() { + echo -n "Waiting for database to be reachable " + failure_count=0 + DD_DATABASE_READINESS_TIMEOUT=${DD_DATABASE_READINESS_TIMEOUT:-30} + until echo "select 1;" | python3 manage.py dbshell > /dev/null + do + echo -n "." + failure_count=$((failure_count + 1)) + sleep 1 + if [ $DD_DATABASE_READINESS_TIMEOUT = $failure_count ]; then + exit 1 + fi + done +} \ No newline at end of file diff --git a/dojo/settings/settings.py b/dojo/settings/settings.py index ce4b1be6200..20f13285a77 100644 --- a/dojo/settings/settings.py +++ b/dojo/settings/settings.py @@ -21,4 +21,4 @@ msg = "Change of 'settings.dist.py' file was detected. It is not allowed to edit this file. " \ "Any customization of variables need to be done via environmental variables or in 'local_settings.py'. " \ "For more information check https://documentation.defectdojo.com/getting_started/configuration/ " - raise ValueError(msg) + sys.exit(msg) From 1a4e78f881a3b1677e7b062e9c8482fae0bb8d1c Mon Sep 17 00:00:00 2001 From: DefectDojo release bot Date: Mon, 10 Jun 2024 22:22:10 +0000 Subject: [PATCH 5/6] Update versions in application files --- components/package.json | 2 +- dojo/__init__.py | 2 +- helm/defectdojo/Chart.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/package.json b/components/package.json index 688f4f5d52f..253123cce6e 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.36.0-dev", + "version": "2.35.2", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/dojo/__init__.py b/dojo/__init__.py index 423f4050e5c..f45061eab28 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = '2.36.0-dev' +__version__ = '2.35.2' __url__ = 'https://github.com/DefectDojo/django-DefectDojo' __docs__ = 'https://documentation.defectdojo.com' diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 56203e30611..1249159ed36 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.36.0-dev" +appVersion: "2.35.2" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.6.134-dev +version: 1.6.134 icon: https://www.defectdojo.org/img/favicon.ico maintainers: - name: madchap From 959353067b603be631d5cbea9d66a430a46c4675 Mon Sep 17 00:00:00 2001 From: DefectDojo release bot Date: Mon, 10 Jun 2024 22:58:51 +0000 Subject: [PATCH 6/6] Update versions in application files --- components/package.json | 2 +- dojo/__init__.py | 2 +- helm/defectdojo/Chart.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/package.json b/components/package.json index 253123cce6e..688f4f5d52f 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.35.2", + "version": "2.36.0-dev", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/dojo/__init__.py b/dojo/__init__.py index f45061eab28..423f4050e5c 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = '2.35.2' +__version__ = '2.36.0-dev' __url__ = 'https://github.com/DefectDojo/django-DefectDojo' __docs__ = 'https://documentation.defectdojo.com' diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 1249159ed36..400ff62de63 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.35.2" +appVersion: "2.36.0-dev" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.6.134 +version: 1.6.135-dev icon: https://www.defectdojo.org/img/favicon.ico maintainers: - name: madchap