diff --git a/docs/content/en/getting_started/upgrading/2.38.md b/docs/content/en/getting_started/upgrading/2.38.md index a234c6f0dea..6f717bb810e 100644 --- a/docs/content/en/getting_started/upgrading/2.38.md +++ b/docs/content/en/getting_started/upgrading/2.38.md @@ -2,6 +2,27 @@ title: 'Upgrading to DefectDojo Version 2.38.x' toc_hide: true weight: -20240805 -description: No special instructions. +description: Breaking Change for HELM deployments --- -There are no special instructions for upgrading to 2.38.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.38.0) for the contents of the release. + +**Breaking Change** + +Previous HELM `values.yaml` file was not following the official HELM best practicies on key naming - [https://helm.sh/docs/chart_best_practices/values/#naming-conventions](https://helm.sh/docs/chart_best_practices/values/#naming-conventions) + +The following `snake_case` keys are replaced with `camelCase` keys in the `values.yaml`: + +- `site_url` is replaced with `siteUrl` +- `celery.worker.app_settings` block is replaced with `celery.worker.appSettings`. In this block: + + - `pool_type` is replaced with `poolType` + - `autoscale_min` is replaced with `autoscaleMin` + - `autoscale_max` is replaced with `autoscaleMax` + - `prefetch_multiplier` is replaced with `prefetchMultiplier` + +- `django.uwsgi.app_settings` block is changed to `django.uwsgi.appSettings`. In this block: + + - `max_fd` is changed to `maxFd` + +- `django.uwsgi.enable_debug` is changed to `django.uwsgi.enableDebug` + +There are no other special instructions for upgrading to 2.38.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.38.0) for the contents of the release. diff --git a/helm/defectdojo/templates/_helpers.tpl b/helm/defectdojo/templates/_helpers.tpl index 52e44b5a5d3..5fb32aff5b5 100644 --- a/helm/defectdojo/templates/_helpers.tpl +++ b/helm/defectdojo/templates/_helpers.tpl @@ -151,7 +151,7 @@ Create chart name and version as used by the chart label. name: {{ .fullName }}-extrasecrets optional: true env: - {{- if .Values.django.uwsgi.enable_debug }} + {{- if .Values.django.uwsgi.enableDebug }} - name: DD_DEBUG value: 'True' {{- end }} diff --git a/helm/defectdojo/templates/configmap.yaml b/helm/defectdojo/templates/configmap.yaml index 71d28cc0cb4..b5248a0a9ba 100644 --- a/helm/defectdojo/templates/configmap.yaml +++ b/helm/defectdojo/templates/configmap.yaml @@ -14,7 +14,7 @@ data: DD_ADMIN_FIRST_NAME: {{ .Values.admin.FirstName | default "Admin" }} DD_ADMIN_LAST_NAME: {{ .Values.admin.LastName | default "User" }} DD_ALLOWED_HOSTS: {{ include "django.allowed_hosts" . }} - DD_SITE_URL: {{ .Values.site_url | default "http://localhost:8080" }} + DD_SITE_URL: {{ .Values.siteUrl | default "http://localhost:8080" }} DD_CELERY_BROKER_SCHEME: {{ if eq .Values.celery.broker "redis" }}{{ template "redis.scheme" . }}{{ end }} DD_CELERY_BROKER_USER: '' DD_CELERY_BROKER_HOST: {{ if eq .Values.celery.broker "redis" }}{{ template "redis.hostname" . }}{{ end }} @@ -22,11 +22,11 @@ data: DD_CELERY_BROKER_PARAMS: '{{ if eq .Values.celery.broker "redis" }}{{- if .Values.redis.transportEncryption.enabled -}}{{ .Values.redis.transportEncryption.params | default "ssl_cert_reqs=optional" }}{{ end }}{{ end }}' DD_CELERY_BROKER_PATH: '{{ .Values.celery.path | default "//" }}' DD_CELERY_LOG_LEVEL: {{ .Values.celery.logLevel }} - DD_CELERY_WORKER_POOL_TYPE: {{ .Values.celery.worker.app_settings.pool_type | default "solo" }} - DD_CELERY_WORKER_AUTOSCALE_MIN: '{{ if eq .Values.celery.worker.app_settings.pool_type "prefork" }}{{ .Values.celery.worker.app_settings.autoscale_min | default "2" }}{{ end }}' - DD_CELERY_WORKER_AUTOSCALE_MAX: '{{ if eq .Values.celery.worker.app_settings.pool_type "prefork" }}{{ .Values.celery.worker.app_settings.autoscale_max | default "8" }}{{ end }}' - DD_CELERY_WORKER_CONCURRENCY: '{{ if eq .Values.celery.worker.app_settings.pool_type "prefork" }}{{ .Values.celery.worker.app_settings.concurrency | default "8" }}{{ end }}' - DD_CELERY_WORKER_PREFETCH_MULTIPLIER: '{{ if eq .Values.celery.worker.app_settings.pool_type "prefork" }}{{ .Values.celery.worker.app_settings.prefetch_multiplier | default "128" }}{{ end }}' + DD_CELERY_WORKER_POOL_TYPE: {{ .Values.celery.worker.appSettings.poolType | default "solo" }} + DD_CELERY_WORKER_AUTOSCALE_MIN: '{{ if eq .Values.celery.worker.appSettings.poolType "prefork" }}{{ .Values.celery.worker.appSettings.autoscaleMin | default "2" }}{{ end }}' + DD_CELERY_WORKER_AUTOSCALE_MAX: '{{ if eq .Values.celery.worker.appSettings.poolType "prefork" }}{{ .Values.celery.worker.appSettings.autoscaleMax | default "8" }}{{ end }}' + DD_CELERY_WORKER_CONCURRENCY: '{{ if eq .Values.celery.worker.appSettings.poolType "prefork" }}{{ .Values.celery.worker.appSettings.concurrency | default "8" }}{{ end }}' + DD_CELERY_WORKER_PREFETCH_MULTIPLIER: '{{ if eq .Values.celery.worker.appSettings.poolType "prefork" }}{{ .Values.celery.worker.appSettings.prefetchMultiplier | default "128" }}{{ end }}' DD_DATABASE_ENGINE: django.db.backends.{{ if eq .Values.database "postgresql" }}postgresql{{ end }}{{ if eq .Values.database "postgresqlha" }}postgresql{{ end }} DD_DATABASE_HOST: {{ if eq .Values.database "postgresql" }}{{ template "postgresql.hostname" . }}{{ end }}{{ if eq .Values.database "postgresqlha" }}{{ template "postgresqlha.hostname" . }}{{ end }} DD_DATABASE_PORT: '{{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.primary.service.ports.postgresql }}{{ end }}{{ if eq .Values.database "postgresqlha" }}{{ .Values.postgresqlha.service.ports.postgresql }}{{ end }}' @@ -36,9 +36,9 @@ data: DD_UWSGI_ENDPOINT: /run/defectdojo/uwsgi.sock DD_UWSGI_HOST: localhost DD_UWSGI_PASS: unix:///run/defectdojo/uwsgi.sock - DD_UWSGI_NUM_OF_PROCESSES: '{{ .Values.django.uwsgi.app_settings.processes | default 2 }}' - DD_UWSGI_NUM_OF_THREADS: '{{ .Values.django.uwsgi.app_settings.threads | default 2 }}' - DD_UWSGI_MAX_FD: '{{ .Values.django.uwsgi.app_settings.max_fd }}' + DD_UWSGI_NUM_OF_PROCESSES: '{{ .Values.django.uwsgi.appSettings.processes | default 2 }}' + DD_UWSGI_NUM_OF_THREADS: '{{ .Values.django.uwsgi.appSettings.threads | default 2 }}' + DD_UWSGI_MAX_FD: '{{ .Values.django.uwsgi.appSettings.maxFd }}' DD_DJANGO_METRICS_ENABLED: '{{ .Values.monitoring.enabled }}' NGINX_METRICS_ENABLED: '{{ .Values.monitoring.enabled }}' METRICS_HTTP_AUTH_USER: {{ .Values.monitoring.user | default "monitoring" }} diff --git a/helm/defectdojo/templates/django-deployment.yaml b/helm/defectdojo/templates/django-deployment.yaml index 6e0a0353561..ce126f1cc28 100644 --- a/helm/defectdojo/templates/django-deployment.yaml +++ b/helm/defectdojo/templates/django-deployment.yaml @@ -155,7 +155,7 @@ spec: - name: http-uwsgi protocol: TCP containerPort: 8081 - {{- if .Values.django.uwsgi.enable_debug }} + {{- if .Values.django.uwsgi.enableDebug }} - name: debug protocol: TCP containerPort: 3000 @@ -174,7 +174,7 @@ spec: name: {{ .Values.redis.auth.existingSecret | default "defectdojo-redis-specific" }} key: {{ .Values.redis.auth.existingSecretPasswordKey | default "redis-password" }} {{- end }} - {{- if .Values.django.uwsgi.enable_debug }} + {{- if .Values.django.uwsgi.enableDebug }} - name: DD_DEBUG value: 'True' {{- end }} diff --git a/helm/defectdojo/values.yaml b/helm/defectdojo/values.yaml index 495f363d3ec..c53e7077683 100644 --- a/helm/defectdojo/values.yaml +++ b/helm/defectdojo/values.yaml @@ -42,7 +42,7 @@ database: postgresql host: defectdojo.default.minikube.local # The full URL to your defectdojo instance, depends on the domain where DD is deployed, it also affects links in Jira -# site_url: 'https://' +# siteUrl: 'https://' # optional list of alternative hostnames to use that gets appended to # DD_ALLOWED_HOSTS. This is necessary when your local hostname does not match @@ -139,15 +139,15 @@ celery: cpu: 2000m memory: 512Mi tolerations: [] - app_settings: - pool_type: solo + appSettings: + poolType: solo # Performance improved celery worker config when needing to deal with a lot of findings (e.g deduplication ops) # Comment out the "solo" line, and uncomment the following lines. - # pool_type: prefork - # autoscale_min: 2 - # autoscale_max: 8 + # poolType: prefork + # autoscaleMin: 2 + # autoscaleMax: 8 # concurrency: 8 - # prefetch_multiplier: 128 + # prefetchMultiplier: 128 # A list of extra volumes to mount. This # is useful for bringing in extra data that can be referenced by other configurations @@ -228,11 +228,11 @@ django: limits: cpu: 2000m memory: 512Mi - app_settings: + appSettings: processes: 2 threads: 2 - # max_fd: 102400 # Uncomment to set the maximum number of file descriptors. If not set will be detected by uwsgi - enable_debug: false # this also requires DD_DEBUG to be set to True + # maxFd: 102400 # Uncomment to set the maximum number of file descriptors. If not set will be detected by uwsgi + enableDebug: false # this also requires DD_DEBUG to be set to True certificates: # includes additional CA certificate as volume, it refrences REQUESTS_CA_BUNDLE env varible # to create configMap `kubectl create cm defectdojo-ca-certs --from-file=ca.crt` diff --git a/readme-docs/KUBERNETES.md b/readme-docs/KUBERNETES.md index 85bdd11b22a..f8854228025 100644 --- a/readme-docs/KUBERNETES.md +++ b/readme-docs/KUBERNETES.md @@ -10,9 +10,11 @@ and [Helm](https://helm.sh/) can be installed locally by following this [guide](https://helm.sh/docs/using_helm/#installing-helm). ## Supported Kubernetes Versions + The tests cover the deployment on the lastest [kubernetes version](https://kubernetes.io/releases/) and the oldest supported [version from AWS](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#available-versions). The assumption is that version in between do not have significant differences. Current tested versions can looks up in the [github k8s workflow](https://github.com/DefectDojo/django-DefectDojo/blob/master/.github/workflows/k8s-tests.yml). ## Helm chart + Starting with version 1.14.0, a helm chart will be pushed onto the `helm-charts` branch during the release process. Don't look for a chart museum, we're leveraging the "raw" capabilities of GitHub at this time. To use it, you can add our repo. @@ -49,11 +51,14 @@ minikube addons enable ingress ``` Helm >= v3 + ```zsh helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update ``` + Then pull the dependent charts: + ```zsh helm dependency update ./helm/defectdojo ``` @@ -61,19 +66,25 @@ helm dependency update ./helm/defectdojo Now, install the helm chart into minikube. If you have setup an ingress controller: + ```zsh DJANGO_INGRESS_ENABLED=true ``` + else: + ```zsh DJANGO_INGRESS_ENABLED=false ``` If you have configured TLS: + ```zsh DJANGO_INGRESS_ACTIVATE_TLS=true ``` + else: + ```zsh DJANGO_INGRESS_ACTIVATE_TLS=false ``` @@ -94,7 +105,7 @@ helm install \ ``` It usually takes up to a minute for the services to startup and the -status of the containers can be viewed by starting up ```minikube dashboard```. +status of the containers can be viewed by starting up `minikube dashboard`. Note: If the containers are not cached locally the services will start once the containers have been pulled locally. @@ -134,15 +145,18 @@ If testing containers locally, then set the imagePullPolicy to Never, which ensures containers are not pulled from Docker hub. Use the same commands as before but add: + ```zsh --set imagePullPolicy=Never ``` ### Installing from a private registry + If you have stored your images in a private registry, you can install defectdojo chart with (helm 3). - First create a secret named "defectdojoregistrykey" based on the credentials that can pull from the registry: see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - Then install the chart with the same commands as before but adding: + ```zsh --set repositoryPrefix= \ --set imagePullSecrets=defectdojoregistrykey @@ -168,12 +182,14 @@ You can set breakpoints in code that is handled by uWSGI. The feature is meant t The port is currently hard-coded to 3000. -* In `values.yaml`, ensure the value for `enable_ptvsd` is set to `true` (the default is `false`). Make sure the change is taken into account in your deployment. -* Have `DD_DEBUG` set to `True`. -* Port forward port 3000 to the pod, such as `kubectl port-forward defectdojo-django-7886f49466-7cwm7 3000`. +- In `values.yaml`, ensure the value for `enable_ptvsd` is set to `true` (the default is `false`). Make sure the change is taken into account in your deployment. +- Have `DD_DEBUG` set to `True`. +- Port forward port 3000 to the pod, such as `kubectl port-forward defectdojo-django-7886f49466-7cwm7 3000`. ### Upgrade the chart + If you want to change kubernetes configuration of use an updated docker image (evolution of defectDojo code), upgrade the application: + ``` kubectl delete job defectdojo-initializer helm upgrade defectdojo ./helm/defectdojo/ \ @@ -181,8 +197,8 @@ helm upgrade defectdojo ./helm/defectdojo/ \ --set django.ingress.activateTLS=${DJANGO_INGRESS_ACTIVATE_TLS} ``` - ### Re-install the chart + In case of issue or in any other situation where you need to re-install the chart, you can do it and re-use the same secrets. **Note: With postgresql you'll keep the same database (more information below)** @@ -229,7 +245,6 @@ If you want to encrypt the traffic to the nginx server you can use the option `- Be aware that the traffic to the database and celery broker are unencrypted at the moment. - ### Media persistent volume By default, DefectDojo helm installation doesn't support persistent storage for storing images (dynamically uploaded by users). By default, it uses emptyDir, which is ephemeral by its nature and doesn't support multiple replicas of django pods, so should not be in use for production. @@ -245,7 +260,7 @@ mediaPersistentVolume: type: pvc # there are two options to create pvc 1) when you want the chart to create pvc for you, set django.mediaPersistentVolume.persistentVolumeClaim.create to true and do not specify anything for django.mediaPersistentVolume.PersistentVolumeClaim.name 2) when you want to create pvc outside the chart, pass the pvc name via django.mediaPersistentVolume.PersistentVolumeClaim.name and ensure django.mediaPersistentVolume.PersistentVolumeClaim.create is set to false persistentVolumeClaim: - create: true + create: true name: size: 5Gi accessModes: @@ -327,10 +342,12 @@ It's possible to enable Nginx prometheus exporter by setting `--set monitoring.e ## Useful stuff ### Setting your own domain -The `site_url` in values.yaml controls what domain is configured in Django, and also what the celery workers will put as links in Jira tickets for example. + +The `siteUrl` in values.yaml controls what domain is configured in Django, and also what the celery workers will put as links in Jira tickets for example. Set this to your `https://` in values.yaml ### Multiple Hostnames + Django requires a list of all hostnames that are valid for requests. You can add additional hostnames via helm or values file as an array. This helps if you have a local service submitting reports to defectDojo using @@ -347,6 +364,7 @@ This will also work with shell inserted variables: You will still need to set a host value as well. ### Using an existing redis setup with redis-sentinel + If you want to use a redis-sentinel setup as the Celery broker, you will need to set the following. 1. Set redis.scheme to "sentinel" in values.yaml @@ -354,23 +372,21 @@ If you want to use a redis-sentinel setup as the Celery broker, you will need to ```yaml celery: - broker: "redis" + broker: 'redis' redis: - redisServer: "PutYourRedisSentinelAddress" - scheme: "sentinel" + redisServer: 'PutYourRedisSentinelAddress' + scheme: 'sentinel' extraEnv: - name: DD_CELERY_BROKER_TRANSPORT_OPTIONS value: '{"master_name": "mymaster"}' - name: 'DD_CELERY_BROKER_PORT' - value: "26379" + value: '26379' ``` - - - ### kubectl commands + ```zsh # View logs of a specific pod kubectl logs $(kubectl get pod --selector=defectdojo.org/component=${POD} \ @@ -388,12 +404,15 @@ kubectl exec -it $(kubectl get pod --selector=defectdojo.org/component=${POD} \ ``` ### Clean up Kubernetes + Helm >= v3 + ``` helm uninstall defectdojo ``` To remove persistent objects not removed by uninstall (this will remove any database): + ``` kubectl delete secrets defectdojo defectdojo-redis-specific defectdojo-postgresql-specific kubectl delete serviceAccount defectdojo