From 3ca5f534e1bd71a0519bb5cfdda3910e62dff5f5 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 09:36:01 -0500 Subject: [PATCH 1/6] Support arbitrary S3 bucket URL. Allow S3 bucket config from Secret. --- helm-chart/templates/_helpers.tpl | 29 +++++++++++++++++++++++++++-- helm-chart/values.yaml | 5 ++++- mop/settings.py | 1 + 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index f9becf05..3a4ed8d6 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -107,13 +107,38 @@ build it here and use it everywhere. - name: IRSA_USERNAME value: {{ .Values.irsaUsername | quote }} - name: IRSA_PASSWORD - value: {{ .Values.irsaPassword | quote }} + value: {{ .Values.irsaPassword | quote }} +{{- if .Values.awsExistingSecret }} +- name: AWS_S3_ENDPOINT_URL + valueFrom: + secretKeyRef: + name: {{ .Values.awsExistingSecret | quote }} + key: "awsEndpointUrl" +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.awsExistingSecret | quote }} + key: "awsAccessKeyId" +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.awsExistingSecret | quote }} + key: "awsSecretAccessKey" +- name: AWS_S3_BUCKET + valueFrom: + secretKeyRef: + name: {{ .Values.awsExistingSecret | quote }} + key: "awsS3Bucket" +{{- else }} +- name: AWS_S3_ENDPOINT_URL + value: {{ .Values.awsEndpointUrl | quote }} - name: AWS_ACCESS_KEY_ID value: {{ .Values.awsAccessKeyId | quote }} - name: AWS_SECRET_ACCESS_KEY value: {{ .Values.awsSecretAccessKey | quote }} - name: AWS_S3_BUCKET - value: {{ .Values.awsS3Bucket | quote }} + value: {{ .Values.awsS3Bucket | quote }} +{{- end }} - name: GEMINI_USERNAME value: {{ .Values.geminiUsername | quote }} - name: GEMINI_N_API_KEY diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 55559682..85a2bf41 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -205,7 +205,10 @@ lcoProposalId: "" lcoUsername: "" # AWS Credentials -# These MUST be overriden in secret configuration file +# If `awsExistingSecret` is specified, all `aws*` params must be included in the specified secret +awsExistingSecret: "" +# These MUST be overriden in secret configuration file if `awsExistingSecret` is not specified +awsEndpointUrl: "" awsAccessKeyId: "" awsSecretAccessKey: "" awsS3Bucket: "" diff --git a/mop/settings.py b/mop/settings.py index 9239dec9..93660ad7 100644 --- a/mop/settings.py +++ b/mop/settings.py @@ -189,6 +189,7 @@ DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +AWS_S3_ENDPOINT_URL = os.getenv('AWS_S3_ENDPOINT_URL', '') AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', '') AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY', '') AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_S3_BUCKET', '') From 56d1741ea5c3bb36d97a5904a1e637d4c9f7901c Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 09:48:22 -0500 Subject: [PATCH 2/6] Support non-root URL base path --- helm-chart/templates/_helpers.tpl | 2 ++ helm-chart/templates/deployment.yaml | 4 ++-- helm-chart/values.yaml | 1 + mop/settings.py | 12 ++++++++---- mop/urls.py | 13 +++++++++++-- templates/tom_common/navbar_content.html | 2 +- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index 3a4ed8d6..e66de913 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -76,6 +76,8 @@ build it here and use it everywhere. {{- define "mop.backendEnv" -}} - name: PYTHONUNBUFFERED value: "1" +- name: URL_BASE_PATH + value: {{ .Values.ingress.basePath | quote }} - name: DB_HOST value: {{ include "mop.dbhost" . | quote }} - name: DB_NAME diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index 5052eb27..bc49e440 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -130,11 +130,11 @@ spec: protocol: TCP livenessProbe: httpGet: - path: / + path: /{{ .Values.ingress.basePath }} port: gunicorn readinessProbe: httpGet: - path: / + path: /{{ .Values.ingress.basePath }} port: gunicorn resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 85a2bf41..6b479faa 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -142,6 +142,7 @@ service: ingress: enabled: false + basePath: "" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" diff --git a/mop/settings.py b/mop/settings.py index 93660ad7..4157c39b 100644 --- a/mop/settings.py +++ b/mop/settings.py @@ -22,7 +22,11 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - +# Get URL base path +base_path = os.environ.get('URL_BASE_PATH', '').strip('/') +base_path_trailing_slash = '' +if base_path: + base_path_trailing_slash = '/' # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ @@ -149,9 +153,9 @@ }, ] -LOGIN_URL = '/accounts/login/' -LOGIN_REDIRECT_URL = '/' -LOGOUT_REDIRECT_URL = '/' +LOGIN_URL = f'/{base_path}{base_path_trailing_slash}accounts/login/' +LOGIN_REDIRECT_URL = f'/{base_path}' +LOGOUT_REDIRECT_URL = f'/{base_path}' AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', diff --git a/mop/urls.py b/mop/urls.py index 253fe72c..7a6fdd0c 100644 --- a/mop/urls.py +++ b/mop/urls.py @@ -16,8 +16,17 @@ from django.urls import path, include from mop.views import MOPTargetDetailView +from django.views.generic import TemplateView +import os +base_path = os.environ.get('URL_BASE_PATH', '').strip('/') +trailing_slash = '' +if base_path: + trailing_slash = '/' urlpatterns = [ - path('targets//', MOPTargetDetailView.as_view(), name='detail'), - path('', include('tom_common.urls')), + path(f'''{base_path}{trailing_slash}targets//''', MOPTargetDetailView.as_view(), name='detail'), + path(f'''{base_path}{trailing_slash}''', TemplateView.as_view( + template_name='tom_common/index.html', + extra_context={"base_path": base_path}), name='home'), + path(f'''{base_path}{trailing_slash}''', include('tom_common.urls')), ] diff --git a/templates/tom_common/navbar_content.html b/templates/tom_common/navbar_content.html index 5a256b8c..edda9f74 100644 --- a/templates/tom_common/navbar_content.html +++ b/templates/tom_common/navbar_content.html @@ -5,7 +5,7 @@ {% endcomment %}
  • - Home (current) + Home (current)
  • From 3607e3a2348ea60f2e037336fe4df21ea1730c87 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 09:48:47 -0500 Subject: [PATCH 3/6] Fix mountPath typo in deployment.yaml --- helm-chart/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index bc49e440..ab266a3b 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -77,7 +77,7 @@ spec: readOnly: false - name: astropy mountPath: /.astropy - readonly: false + readOnly: false {{- end }} - name: django-collectstatic image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" @@ -102,7 +102,7 @@ spec: readOnly: false - name: astropy mountPath: /.astropy - readonly: false + readOnly: false - name: static mountPath: /static readOnly: false From 6cd636337a347240c6f1cc8b44e196bc5007e14f Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 09:49:07 -0500 Subject: [PATCH 4/6] Update Helm chart to apiVersion: v2 --- helm-chart/Chart.yaml | 10 ++++++++-- helm-chart/requirements.yaml | 5 ----- 2 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 helm-chart/requirements.yaml diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 4a062acf..df89a221 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -1,4 +1,4 @@ -apiVersion: v1 +apiVersion: v2 name: mop description: A Helm chart for Kubernetes for the MOP TOM @@ -13,8 +13,14 @@ description: A Helm chart for Kubernetes for the MOP TOM type: application # This is the chart version. Don't change it. The helmPipeline() appends the git hash to it as needed. -version: 0.2.0 +version: 0.2.1 # This is the version number of the application being deployed. Don't change it. This value # is edited by helmPipeline() to be consistent with the latest git-tag on the master branch of the repository. appVersion: 0.0.0 + +dependencies: +- name: postgresql + version: 8.6.4 + repository: https://charts.helm.sh/stable + condition: useDockerizedDatabase diff --git a/helm-chart/requirements.yaml b/helm-chart/requirements.yaml deleted file mode 100644 index 4269bb85..00000000 --- a/helm-chart/requirements.yaml +++ /dev/null @@ -1,5 +0,0 @@ -dependencies: -- name: postgresql - version: 8.6.4 - repository: https://charts.helm.sh/stable - condition: useDockerizedDatabase From 91152950725780ada3ce00f0eb18ad55642656f8 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 09:53:42 -0500 Subject: [PATCH 5/6] Remove requirements.lock --- helm-chart/requirements.lock | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 helm-chart/requirements.lock diff --git a/helm-chart/requirements.lock b/helm-chart/requirements.lock deleted file mode 100644 index 02ff0a59..00000000 --- a/helm-chart/requirements.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://kubernetes-charts.storage.googleapis.com - version: 8.6.4 -digest: sha256:f2b56726d815fac0b860406a670a7cbb87544689a4f4ce259d998fb7858d6797 -generated: "2020-09-10T13:05:56.384525436-07:00" From 2ca850d01c83726ba1e3dc13997eede937c0d465 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Mon, 11 Apr 2022 16:30:02 -0500 Subject: [PATCH 6/6] Implement support for OIDC using mozilla_django_oidc --- mop/auth_backends.py | 48 ++++++++++++++++++++++++++++++++++ mop/settings.py | 37 +++++++++++++++++++++++--- mop/urls.py | 1 + requirements.txt | 1 + templates/tom_common/base.html | 8 ++++-- 5 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 mop/auth_backends.py diff --git a/mop/auth_backends.py b/mop/auth_backends.py new file mode 100644 index 00000000..7f79a0e5 --- /dev/null +++ b/mop/auth_backends.py @@ -0,0 +1,48 @@ +# This auth backend subclass fixes a bug related to the error: +# OIDC callback state not found in session `oidc_states`! +# ref: https://github.com/mozilla/mozilla-django-oidc/issues/435 + + +from mozilla_django_oidc.auth import OIDCAuthenticationBackend + +class KeycloakOIDCAuthenticationBackend(OIDCAuthenticationBackend): + + def create_user(self, claims): + """ Overrides Authentication Backend so that Django users are + created with the keycloak preferred_username. + If nothing found matching the email, then try the username. + """ + user = super(KeycloakOIDCAuthenticationBackend, self).create_user(claims) + user.first_name = claims.get('given_name', '') + user.last_name = claims.get('family_name', '') + user.email = claims.get('email') + user.is_staff = True #Here fix that error + user.username = claims.get('preferred_username') + user.save() + return user + + def filter_users_by_claims(self, claims): + """ Return all users matching the specified email. + If nothing found matching the email, then try the username + """ + email = claims.get('email') + + if not email: + return self.UserModel.objects.none() + users = self.UserModel.objects.filter(email__iexact=email) + return users + + def update_user(self, user, claims): + user.first_name = claims.get('given_name', '') + user.last_name = claims.get('family_name', '') + user.email = claims.get('email') + user.save() + return user + +import unicodedata + +def generate_username(email): + # Using Python 3 and Django 1.11+, usernames can contain alphanumeric + # (ascii and unicode), _, @, +, . and - characters. So we normalize + # it and slice at 150 characters. + return unicodedata.normalize('NFKC', email)[:150] diff --git a/mop/settings.py b/mop/settings.py index 4157c39b..bff1826e 100644 --- a/mop/settings.py +++ b/mop/settings.py @@ -44,6 +44,7 @@ 'whitenoise.runserver_nostatic', 'django.contrib.admin', 'django.contrib.auth', + 'mozilla_django_oidc', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', @@ -153,15 +154,45 @@ }, ] -LOGIN_URL = f'/{base_path}{base_path_trailing_slash}accounts/login/' -LOGIN_REDIRECT_URL = f'/{base_path}' -LOGOUT_REDIRECT_URL = f'/{base_path}' +# LOGIN_REDIRECT_URL = f'/{base_path}' +# LOGOUT_REDIRECT_URL = f'/{base_path}' AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', + 'mop.auth_backends.KeycloakOIDCAuthenticationBackend', 'guardian.backends.ObjectPermissionBackend', ) +LOGIN_URL = f'/{base_path}{base_path_trailing_slash}accounts/login/' +oidc_login_url = os.environ.get('OIDC_LOGIN_URL_PATH', '') +if oidc_login_url: + LOGIN_URL = f'/{base_path}{base_path_trailing_slash}{oidc_login_url}/' + +LOGIN_REDIRECT_URL = os.environ.get('OIDC_REDIRECT_URL', f'/{base_path}') +LOGOUT_REDIRECT_URL = os.environ.get('OIDC_REDIRECT_URL_POST_LOGOUT', f'/{base_path}') + +OIDC_RP_CLIENT_ID = os.environ.get('OIDC_CLIENT_ID', '') +OIDC_RP_CLIENT_SECRET = os.environ.get('OIDC_CLIENT_SECRET', '') +OIDC_RP_SCOPES = os.environ.get('OIDC_SCOPES', '') +if not OIDC_RP_SCOPES: + OIDC_RP_SCOPES = "openid profile email" +OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get('OIDC_OP_AUTHORIZATION_ENDPOINT', '') +OIDC_OP_TOKEN_ENDPOINT = os.environ.get('OIDC_OP_TOKEN_ENDPOINT', '') +OIDC_OP_USER_ENDPOINT = os.environ.get('OIDC_OP_USER_ENDPOINT', '') + +## Required for Keycloak +OIDC_RP_SIGN_ALGO = os.environ.get('OIDC_RP_SIGN_ALGO', 'RS256') +OIDC_OP_JWKS_ENDPOINT = os.environ.get('OIDC_OP_JWKS_ENDPOINT', '') + +# OIDC_USERNAME_ALGO +# ref: https://mozilla-django-oidc.readthedocs.io/en/stable/installation.html#generating-usernames +OIDC_USERNAME_ALGO = 'mop.auth_backends.generate_username' + +# SESSION_ENGINE +# ref: https://github.com/mozilla/mozilla-django-oidc/issues/435#issuecomment-1036372844 +# ref: https://docs.djangoproject.com/en/4.0/topics/http/sessions/ +SESSION_ENGINE = "django.contrib.sessions.backends.cache" + # Internationalization # https://docs.djangoproject.com/en/2.1/topics/i18n/ diff --git a/mop/urls.py b/mop/urls.py index 7a6fdd0c..488b3bc6 100644 --- a/mop/urls.py +++ b/mop/urls.py @@ -24,6 +24,7 @@ if base_path: trailing_slash = '/' urlpatterns = [ + path(f'''{base_path}{trailing_slash}oidc/''', include('mozilla_django_oidc.urls')), path(f'''{base_path}{trailing_slash}targets//''', MOPTargetDetailView.as_view(), name='detail'), path(f'''{base_path}{trailing_slash}''', TemplateView.as_view( template_name='tom_common/index.html', diff --git a/requirements.txt b/requirements.txt index d4e0b824..e09bfdba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ boto3==1.16.61 lxml==4.6.3 pandas #numpy +mozilla-django-oidc diff --git a/templates/tom_common/base.html b/templates/tom_common/base.html index 1b5b1ec6..12211c72 100644 --- a/templates/tom_common/base.html +++ b/templates/tom_common/base.html @@ -54,11 +54,15 @@ {% endif %}
  • - Logout + +
    + {% csrf_token %} + +
    • {% else %}
  • - Login + Login
    • {% endif %}