From afbc0b744ac09731aab85339b40a1e324a450e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 28 Jun 2023 15:49:21 +0200 Subject: [PATCH 1/2] feat: compatibility with k8s OK so running JupyterHub on Kubernetes is super hard. The zero-to-jupyterhub ("z2jh") project runs the KubeSpawner, which is otherwise undocumented. We deployed z2jh to observe how it works. In particular, we added a "jupyterhub" service account to this plugin. This allows the KubeSpawner to monitor the k8s cluster for pods creation/deletion. --- README.rst | 2 +- changelog.d/20230629_153446_regis_k8s.md | 1 + tutorjupyter/patches/k8s-deployments | 72 +++++++++++++++++++ tutorjupyter/patches/k8s-jobs | 24 +++++++ tutorjupyter/patches/k8s-services | 16 +++++ .../patches/kustomization-configmapgenerator | 6 ++ .../patches/local-docker-compose-services | 1 + ...penedx-dockerfile-post-python-requirements | 4 +- .../jupyter/apps/jupyterhub_config.py | 46 +++++++----- .../templates/jupyter/build/hub/Dockerfile | 7 +- 10 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 changelog.d/20230629_153446_regis_k8s.md create mode 100644 tutorjupyter/patches/k8s-deployments create mode 100644 tutorjupyter/patches/k8s-jobs create mode 100644 tutorjupyter/patches/k8s-services create mode 100644 tutorjupyter/patches/kustomization-configmapgenerator diff --git a/README.rst b/README.rst index 369448a..c21eb8f 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ This is a plugin for Tutor that makes it easy to integrate `Jupyter `__ project. +⚠️ Compatibility with Kubernetes was not battle-tested. Please report any issue you face. For a more production-ready Kubernetes environment, you are encouraged to check the documentation of the `Zero to JupyterHub with Kubernetes `__ project. Installation ------------ diff --git a/changelog.d/20230629_153446_regis_k8s.md b/changelog.d/20230629_153446_regis_k8s.md new file mode 100644 index 0000000..a7b0f19 --- /dev/null +++ b/changelog.d/20230629_153446_regis_k8s.md @@ -0,0 +1 @@ +- [Feature] Run JupyterHub on Kubernetes. This is an alpha feature. Feedback is welcome! (by @regisb) diff --git a/tutorjupyter/patches/k8s-deployments b/tutorjupyter/patches/k8s-deployments new file mode 100644 index 0000000..c464623 --- /dev/null +++ b/tutorjupyter/patches/k8s-deployments @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jupyterhub + labels: + app.kubernetes.io/name: jupyterhub +spec: + selector: + matchLabels: + app.kubernetes.io/name: jupyterhub + template: + metadata: + labels: + app.kubernetes.io/name: jupyterhub + spec: + containers: + - name: jupyterhub + image: {{ JUPYTER_DOCKER_IMAGE_HUB }} + env: + - name: JUPYTERHUB_CRYPT_KEY + value: "{{ JUPYTER_HUB_CRYPT_KEY }}" + - name: SPAWNER + value: kubernetes + ports: + - containerPort: 9045 + - containerPort: 8081 + volumeMounts: + - mountPath: /srv/jupyterhub/ + name: config + serviceAccountName: jupyterhub + volumes: + - name: config + configMap: + name: jupyterhub-config +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: jupyterhub + labels: + app.kubernetes.io/name: jupyterhub +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: jupyterhub + labels: + app.kubernetes.io/name: jupyterhub +rules: + - apiGroups: [""] + resources: ["pods", "persistentvolumeclaims", "secrets", "services"] + verbs: ["get", "watch", "list", "create", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "watch", "list"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: jupyterhub + labels: + app.kubernetes.io/name: jupyterhub +subjects: + - kind: ServiceAccount + name: jupyterhub + namespace: "{{ K8S_NAMESPACE }}" +roleRef: + kind: Role + name: jupyterhub + apiGroup: rbac.authorization.k8s.io + diff --git a/tutorjupyter/patches/k8s-jobs b/tutorjupyter/patches/k8s-jobs new file mode 100644 index 0000000..abd084b --- /dev/null +++ b/tutorjupyter/patches/k8s-jobs @@ -0,0 +1,24 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: jupyterhub-job + labels: + app.kubernetes.io/component: job +spec: + template: + spec: + restartPolicy: Never + containers: + - name: jupyterhub + image: {{ JUPYTER_DOCKER_IMAGE_HUB }} + env: + - name: JUPYTERHUB_CRYPT_KEY + value: "{{ JUPYTER_HUB_CRYPT_KEY }}" + volumeMounts: + - mountPath: /srv/jupyterhub/ + name: config + volumes: + - name: config + configMap: + name: jupyterhub-config diff --git a/tutorjupyter/patches/k8s-services b/tutorjupyter/patches/k8s-services new file mode 100644 index 0000000..6b0798b --- /dev/null +++ b/tutorjupyter/patches/k8s-services @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: jupyterhub +spec: + type: NodePort + ports: + - port: 9045 + protocol: TCP + name: hub + - port: 8081 + protocol: TCP + name: proxy + selector: + app.kubernetes.io/name: jupyterhub diff --git a/tutorjupyter/patches/kustomization-configmapgenerator b/tutorjupyter/patches/kustomization-configmapgenerator new file mode 100644 index 0000000..c23ea78 --- /dev/null +++ b/tutorjupyter/patches/kustomization-configmapgenerator @@ -0,0 +1,6 @@ +- name: jupyterhub-config + files: + - plugins/jupyter/apps/jupyterhub_config.py + options: + labels: + app.kubernetes.io/name: jupyterhub diff --git a/tutorjupyter/patches/local-docker-compose-services b/tutorjupyter/patches/local-docker-compose-services index 8824885..65c24c8 100644 --- a/tutorjupyter/patches/local-docker-compose-services +++ b/tutorjupyter/patches/local-docker-compose-services @@ -3,6 +3,7 @@ jupyterhub: environment: JUPYTERHUB_CRYPT_KEY: "{{ JUPYTER_HUB_CRYPT_KEY }}" NETWORK_NAME: "{{ LOCAL_PROJECT_NAME }}_default" + SPAWNER: "docker" volumes: - ../plugins/jupyter/apps/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro # Spawn Docker containers diff --git a/tutorjupyter/patches/openedx-dockerfile-post-python-requirements b/tutorjupyter/patches/openedx-dockerfile-post-python-requirements index 572ddc1..8cf748d 100644 --- a/tutorjupyter/patches/openedx-dockerfile-post-python-requirements +++ b/tutorjupyter/patches/openedx-dockerfile-post-python-requirements @@ -1,4 +1,4 @@ # Install Jupyter XBlock -# Remember to bump the version when we upgrade from Olive. +# Remember to bump the version when we upgrade from Palm. # https://pypi.org/project/jupyter-xblock/ -RUN pip install "jupyter-xblock>=15.0.3,<16.0.0" +RUN pip install "jupyter-xblock>=16.0.0,<17.0.0" diff --git a/tutorjupyter/templates/jupyter/apps/jupyterhub_config.py b/tutorjupyter/templates/jupyter/apps/jupyterhub_config.py index ca64e7b..52e9549 100644 --- a/tutorjupyter/templates/jupyter/apps/jupyterhub_config.py +++ b/tutorjupyter/templates/jupyter/apps/jupyterhub_config.py @@ -17,6 +17,8 @@ # Database c.JupyterHub.db_url = "mysql+pymysql://{{ JUPYTER_HUB_MYSQL_USERNAME }}:{{ JUPYTER_HUB_MYSQL_PASSWORD }}@{{ MYSQL_HOST }}:{{ MYSQL_PORT }}/{{ JUPYTER_HUB_MYSQL_DATABASE }}" c.JupyterHub.cookie_secret = "{{ JUPYTER_HUB_COOKIE_SECRET }}" +# Don't write pid file to current folder, where we may not have write access +c.ConfigurableHTTPProxy.pid_file = "/tmp/jupyter-proxy.pid" # Authorise embedding in some iframes. # Add "*" to allow embedding in all iframes (though it's dangerous and you probably @@ -69,26 +71,38 @@ c.ServerApp.nbserver_extensions = {"nbgitpuller": True} # Spawner - -# Run as Docker containers -# https://jupyterhub-dockerspawner.readthedocs.io/en/latest/api/index.html -c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" -c.DockerSpawner.image = "{{ JUPYTER_DOCKER_IMAGE_LAB }}" -c.DockerSpawner.debug = True -c.DockerSpawner.remove = True -c.DockerSpawner.use_internal_ip = True -c.DockerSpawner.network_name = os.environ.get("NETWORK_NAME") -c.DockerSpawner.environment = {"CONTENT_SECURITY_POLICY": content_security_policy} -# Persist user data: this will create a new "jupyterhub-user-{username}" named volume on -# the host for every Docker container. -# https://jupyterhub-dockerspawner.readthedocs.io/en/latest/data-persistence.html -notebook_dir = "/home/jovyan/work" -c.DockerSpawner.notebook_dir = notebook_dir -c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} # Limit spawner cpu and memory {% if JUPYTER_LAB_CPU_LIMIT %} c.Spawner.cpu_limit = {{ JUPYTER_LAB_CPU_LIMIT }} {% endif %} c.Spawner.mem_limit = "{{ JUPYTER_LAB_MEMORY_LIMIT }}" +if os.environ.get("SPAWNER") == "docker": + # Run as Docker containers + # https://jupyterhub-dockerspawner.readthedocs.io/en/latest/api/index.html + c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" + c.DockerSpawner.image = "{{ JUPYTER_DOCKER_IMAGE_LAB }}" + c.DockerSpawner.debug = True + c.DockerSpawner.remove = True + c.DockerSpawner.use_internal_ip = True + c.DockerSpawner.network_name = os.environ.get("NETWORK_NAME") + c.DockerSpawner.environment = {"CONTENT_SECURITY_POLICY": content_security_policy} + # Persist user data: this will create a new "jupyterhub-user-{username}" named volume on + # the host for every Docker container. + # https://jupyterhub-dockerspawner.readthedocs.io/en/latest/data-persistence.html + c.DockerSpawner.notebook_dir = "/home/jovyan/work" + c.DockerSpawner.volumes = {"jupyterhub-user-{username}": c.DockerSpawner.notebook_dir} +elif os.environ.get("SPAWNER") == "kubernetes": + # Run as kubernetes pods + # https://jupyterhub-kubespawner.readthedocs.io/en/latest/spawner.html + # https://z2jh.jupyter.org/en/stable/resources/reference.html#helm-chart-configuration-reference + # spoiler: you're in for one hell of a ride... + c.JupyterHub.spawner_class = "kubespawner.KubeSpawner" + c.KubeSpawner.debug = True + c.KubeSpawner.hub_connect_url = "http://jupyterhub:8081" + # c.KubeSpawner.port = 8081 + c.KubeSpawner.service_account = "jupyterhub" + c.KubeSpawner.image = "{{ JUPYTER_DOCKER_IMAGE_LAB }}" + c.KubeSpawner.environment = {"CONTENT_SECURITY_POLICY": content_security_policy} + {{ patch("jupyterhub-config") }} diff --git a/tutorjupyter/templates/jupyter/build/hub/Dockerfile b/tutorjupyter/templates/jupyter/build/hub/Dockerfile index 596278c..f0cdd5f 100644 --- a/tutorjupyter/templates/jupyter/build/hub/Dockerfile +++ b/tutorjupyter/templates/jupyter/build/hub/Dockerfile @@ -3,8 +3,13 @@ FROM docker.io/jupyterhub/jupyterhub:4.0.0 # https://pypi.org/project/dockerspawner/ +# https://pypi.org/project/jupyterhub-kubespawner/ # https://pypi.org/project/jupyterhub-ltiauthenticator/ # https://pypi.org/project/pymysql/ -RUN pip install dockerspawner==12.1.0 jupyterhub-ltiauthenticator==1.5.1 pymysql==1.0.3 +RUN pip install \ + dockerspawner==12.1.0 \ + jupyterhub-kubespawner==6.0.0 \ + jupyterhub-ltiauthenticator==1.5.1 \ + pymysql==1.0.3 CMD ["jupyterhub"] From 4021115a96914bc056957c7a5d436fe2aa279fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 29 Jun 2023 15:41:38 +0200 Subject: [PATCH 2/2] v16.0.1 --- CHANGELOG.md | 5 +++++ changelog.d/20230629_153446_regis_k8s.md | 1 - tutorjupyter/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230629_153446_regis_k8s.md diff --git a/CHANGELOG.md b/CHANGELOG.md index dfe91a5..8c48991 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ instructions, because git commits are used to generate release notes: + +## v16.0.1 (2023-06-29) + +- [Feature] Run JupyterHub on Kubernetes. This is an alpha feature. Feedback is welcome! (by @regisb) + ## v16.0.0 (2023-06-15) diff --git a/changelog.d/20230629_153446_regis_k8s.md b/changelog.d/20230629_153446_regis_k8s.md deleted file mode 100644 index a7b0f19..0000000 --- a/changelog.d/20230629_153446_regis_k8s.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] Run JupyterHub on Kubernetes. This is an alpha feature. Feedback is welcome! (by @regisb) diff --git a/tutorjupyter/__about__.py b/tutorjupyter/__about__.py index 1b1318c..c1d7120 100644 --- a/tutorjupyter/__about__.py +++ b/tutorjupyter/__about__.py @@ -1,4 +1,4 @@ -__version__ = "16.0.0" +__version__ = "16.0.1" # Handle version suffix for nightly, just like tutor core. __version_suffix__ = ""