diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index 0d4411c..b711cc2 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Rollin Thomas " # Base Ubuntu packages @@ -29,29 +29,37 @@ RUN \ # Python 3 Miniconda and dependencies for JupyterHub we can get via conda RUN \ - curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + curl -s -o /tmp/miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash /tmp/miniconda3.sh -b -p /opt/anaconda3 && \ rm -rf /tmp/miniconda3.sh && \ echo "python 3.7.3" >> /opt/anaconda3/conda-meta/pinned && \ /opt/anaconda3/bin/conda update --yes conda && \ /opt/anaconda3/bin/conda install --yes \ - alembic \ - cryptography \ - decorator \ - entrypoints \ - jinja2 \ - mako \ - markupsafe \ - nodejs \ - oauthlib \ - pamela \ - psycopg2 \ - pyopenssl \ - python-dateutil \ - python-editor \ - sqlalchemy \ - tornado \ - traitlets + alembic \ + attrs \ + certipy \ + cryptography \ + decorator \ + entrypoints \ + jinja2 \ + jsonschema \ + mako \ + markupsafe \ + more-itertools \ + nodejs \ + oauthlib \ + pamela \ + psycopg2 \ + pyopenssl \ + pyrsistent \ + python-dateutil \ + python-editor \ + ruamel.yaml \ + ruamel.yaml.clib \ + sqlalchemy \ + tornado \ + traitlets \ + zipp # Install JupyterHub @@ -59,11 +67,9 @@ ENV PATH=/opt/anaconda3/bin:$PATH WORKDIR /tmp RUN \ npm install -g configurable-http-proxy && \ -# git clone https://github.com/jupyterhub/jupyterhub.git && \ - git clone https://github.com/rcthomas/jupyterhub.git && \ + git clone https://github.com/jupyterhub/jupyterhub.git && \ cd jupyterhub && \ -# git checkout tags/1.0.0 && \ - git checkout auth-state-to-spawner && \ + git checkout tags/1.1.0 && \ /opt/anaconda3/bin/python setup.py js && \ /opt/anaconda3/bin/pip --no-cache-dir install . && \ cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \ diff --git a/jupyter-base/build.sh b/jupyter-base/build.sh index e6265ec..6b34fda 100644 --- a/jupyter-base/build.sh +++ b/jupyter-base/build.sh @@ -1,7 +1,21 @@ #!/bin/bash +imcmd="" +for command in docker podman; do + if [ $(command -v $command) ]; then + imcmd=$command + break + fi +done +if [ -n "$imcmd" ]; then + echo "Using $imcmd" +else + echo "No image command defined" + exit 1 +fi + branch=$(git symbolic-ref --short HEAD) -docker build \ - --no-cache \ +$imcmd build \ + "$@" \ --tag registry.spin.nersc.gov/das/jupyter-base-$branch:latest . diff --git a/jupyter-nersc/app-monitoring/Dockerfile b/jupyter-nersc/app-monitoring/Dockerfile index 7d6e874..98811ab 100644 --- a/jupyter-nersc/app-monitoring/Dockerfile +++ b/jupyter-nersc/app-monitoring/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Rollin Thomas " # Base Ubuntu packages @@ -25,10 +25,10 @@ RUN \ # Python 3 Miniconda RUN \ - curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + curl -s -o /tmp/miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \ rm -rf /tmp/miniconda3.sh && \ -# /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda update --yes conda && \ /opt/anaconda3/bin/pip install --no-cache-dir \ pika diff --git a/jupyter-nersc/app-notebooks/Dockerfile b/jupyter-nersc/app-notebooks/Dockerfile index a18427d..86fc70f 100644 --- a/jupyter-nersc/app-notebooks/Dockerfile +++ b/jupyter-nersc/app-notebooks/Dockerfile @@ -23,6 +23,7 @@ RUN \ ldap-utils \ libnss-ldapd \ libpam-ldap \ + libxrender-dev \ nscd \ openssh-server \ supervisor \ diff --git a/jupyter-nersc/web-announcement/Dockerfile b/jupyter-nersc/web-announcement/Dockerfile index 1d05094..b50af16 100644 --- a/jupyter-nersc/web-announcement/Dockerfile +++ b/jupyter-nersc/web-announcement/Dockerfile @@ -4,7 +4,7 @@ FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest LABEL maintainer="Rollin Thomas " RUN \ - pip install git+https://github.com/rcthomas/jupyterhub-announcement.git@0.3.1 + pip install git+https://github.com/rcthomas/jupyterhub-announcement.git@0.4.1 WORKDIR /srv diff --git a/jupyter-nersc/web-announcement/announcement_config.py b/jupyter-nersc/web-announcement/announcement_config.py index ba3fc72..09eb9ef 100644 --- a/jupyter-nersc/web-announcement/announcement_config.py +++ b/jupyter-nersc/web-announcement/announcement_config.py @@ -50,6 +50,12 @@ # AnnouncementQueue(LoggingConfigurable) configuration #------------------------------------------------------------------------------ +## Number of days to retain announcements. +# +# Announcements that have been in the queue for this many days are purged from +# the queue. +#c.AnnouncementQueue.lifetime_days = 7.0 + ## File path where announcements persist as JSON. # # For a persistent announcement queue, this parameter must be set to a non-empty @@ -70,3 +76,15 @@ #c.AnnouncementQueue.persist_path = '' c.AnnouncementQueue.persist_path = 'announcements.json' +#------------------------------------------------------------------------------ +# SSLContext(Configurable) configuration +#------------------------------------------------------------------------------ + +## SSL CA, use with keyfile and certfile +#c.SSLContext.cafile = '' + +## SSL cert, use with keyfile +#c.SSLContext.certfile = '' + +## SSL key, use with certfile +#c.SSLContext.keyfile = '' diff --git a/jupyter-nersc/web-jupyterhub/Dockerfile b/jupyter-nersc/web-jupyterhub/Dockerfile index a1ca275..bbda83c 100644 --- a/jupyter-nersc/web-jupyterhub/Dockerfile +++ b/jupyter-nersc/web-jupyterhub/Dockerfile @@ -7,9 +7,9 @@ WORKDIR /srv # Authenticator and spawner RUN \ - pip install git+https://github.com/nersc/sshapiauthenticator.git && \ - pip install git+https://github.com/jupyterhub/batchspawner.git@4747946 && \ - pip install git+https://github.com/jupyterhub/wrapspawner.git && \ + pip install git+https://github.com/nersc/sshapiauthenticator.git && \ + pip install git+https://github.com/jupyterhub/batchspawner.git@v1.0.0-rc0 && \ + pip install git+https://github.com/jupyterhub/wrapspawner.git && \ pip install git+https://github.com/nersc/sshspawner.git # Customized templates @@ -22,6 +22,7 @@ ENV PYTHONPATH=/srv ADD nerscspawner.py . ADD nerscslurmspawner.py . ADD iris.py . +ADD spinproxy.py . # Hub scripts diff --git a/jupyter-nersc/web-jupyterhub/build.sh b/jupyter-nersc/web-jupyterhub/build.sh index b0340fc..86f8551 100644 --- a/jupyter-nersc/web-jupyterhub/build.sh +++ b/jupyter-nersc/web-jupyterhub/build.sh @@ -1,8 +1,22 @@ #!/bin/bash +imcmd="" +for command in docker podman; do + if [ $(command -v $command) ]; then + imcmd=$command + break + fi +done +if [ -n "$imcmd" ]; then + echo "Using $imcmd" +else + echo "No image command defined" + exit 1 +fi + branch=$(git symbolic-ref --short HEAD) -docker build \ +$imcmd build \ --build-arg branch=$branch \ "$@" \ --tag registry.spin.nersc.gov/das/web-jupyterhub.jupyter-nersc-$branch:latest . diff --git a/jupyter-nersc/web-jupyterhub/iris.py b/jupyter-nersc/web-jupyterhub/iris.py index 21ab9e0..8a7a857 100644 --- a/jupyter-nersc/web-jupyterhub/iris.py +++ b/jupyter-nersc/web-jupyterhub/iris.py @@ -13,6 +13,7 @@ async def query_user(self, name): query {{ systemInfo {{ users(name: "{}") {{ + uid baseRepos {{ computeAllocation {{ repoName diff --git a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py index a2e2517..ffac7b4 100644 --- a/jupyter-nersc/web-jupyterhub/jupyterhub_config.py +++ b/jupyter-nersc/web-jupyterhub/jupyterhub_config.py @@ -2,12 +2,13 @@ import os import sys +from uuid import uuid4 import asyncssh -import requests from tornado import web from jupyterhub.utils import url_path_join +nersc_jupyterhub_subdomain = os.environ.get("NERSC_JUPYTERHUB_SUBDOMAIN", "jupyter") def comma_split(string): """Handle env variables that may be None, empty string, or have spaces""" @@ -20,8 +21,6 @@ def comma_split(string): else: return list() -ip = requests.get('https://v4.ifconfig.co/json').json()['ip'] - #------------------------------------------------------------------------------ # Application(SingletonConfigurable) configuration #------------------------------------------------------------------------------ @@ -233,6 +232,7 @@ def comma_split(string): # # .. versionadded:: 0.9 #c.JupyterHub.hub_bind_url = '' +c.JupyterHub.hub_bind_url = 'http://web-jupyterhub:8081' ## The ip or hostname for proxies and spawners to use for connecting to the Hub. # @@ -248,7 +248,6 @@ def comma_split(string): # # .. versionadded:: 0.8 #c.JupyterHub.hub_connect_ip = '' -c.JupyterHub.hub_connect_ip = ip ## DEPRECATED # @@ -272,6 +271,7 @@ def comma_split(string): # # .. versionadded:: 0.9 #c.JupyterHub.hub_connect_url = '' +c.JupyterHub.hub_connect_url = f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub" ## The ip address for the Hub process to *bind* to. # @@ -615,6 +615,9 @@ def comma_split(string): # across upgrades, so if you are using the callable take care to verify it # continues to work after upgrades! #c.Spawner.environment = {} +c.Spawner.environment = { + "JUPYTER_RUNTIME_DIR": lambda spawner: f"/tmp/jupyter-runtime-{uuid4()}" +} ## Timeout (in seconds) before giving up on a spawned HTTP server # @@ -753,7 +756,7 @@ def comma_split(string): # takes longer than this. start should return when the server process is started # and its location is known. #c.Spawner.start_timeout = 60 -c.Spawner.start_timeout = 120 +c.Spawner.start_timeout = 180 #------------------------------------------------------------------------------ # LocalProcessSpawner(Spawner) configuration @@ -996,6 +999,9 @@ def comma_split(string): # Additional ConfigurableHTTPProxy configuration #------------------------------------------------------------------------------ +from spinproxy import ConfigurableHTTPProxySpin +c.JupyterHub.proxy_class = ConfigurableHTTPProxySpin + c.ConfigurableHTTPProxy.should_start = False c.ConfigurableHTTPProxy.api_url = 'http://web-proxy:8001' @@ -1008,8 +1014,10 @@ def comma_split(string): { "name": "gerty-shared-node-cpu" }, { "name": "gerty-exclusive-node-cpu" }, { "name": "cori-shared-node-cpu" }, + { "name": "cori2-shared-node-cpu" }, + { "name": "cori-shared-node-gpu" }, { "name": "cori-exclusive-node-cpu" }, - { "name": "cori-exclusive-node-gpu" }, + { "name": "cori-configurable-gpu" }, { "name": "spin-shared-node-cpu" }, ] @@ -1021,6 +1029,11 @@ def comma_split(string): "name": "cpu", "description": "Shared CPU Node", "roles": [], + }, + { + "name": "gpu", + "description": "Shared GPU Node", + "roles": ["gpu"], } ], "resources": "Use a node shared with other users' notebooks but outside the batch queues.", @@ -1033,16 +1046,23 @@ def comma_split(string): "name": "cpu", "description": "Exclusive CPU Node", "roles": ["cori-exclusive-node-cpu"], - }, - { - "name": "gpu", - "description": "Exclusive GPU Node", - "roles": ["gpu"], - } + } ], "resources": "Use your own node within a job allocation using defaults.", "use_cases": "Visualization, analytics, machine learning that is compute or memory intensive but can be done on a single node." }, +# { +# "name": "configurable", +# "architectures": [ +# { +# "name": "gpu", +# "description": "Configurable GPU", +# "roles": ["gpu"], +# } +# ], +# "resources": "Use multiple compute nodes with specialized settings.", +# "use_cases": "Multi-node analytics jobs, jobs in reservations, custom project charging, and more." +# }, ] c.NERSCSpawner.systems = [ @@ -1054,6 +1074,10 @@ def comma_split(string): "name": "cori", "roles": [] }, + { + "name": "cori2", + "roles": [] + }, { "name": "spin", "roles": [] @@ -1064,20 +1088,19 @@ def comma_split(string): "gerty-shared-node-cpu": ( "sshspawner.sshspawner.SSHSpawner", { "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", - "/global/common/cori_cle7/software/jupyter/19-11/bin/jupyter-labhub"], + "/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"], "args": ["--transport=ipc"], "environment": {"OMP_NUM_THREADS" : "2"}, "remote_hosts": ["gerty.nersc.gov"], "remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", - "hub_api_url": "http://{}:8081/hub/api".format(ip), - "path": "/global/common/cori_cle7/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", + "path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", "ssh_keyfile": '/certs/{username}.key' } ), "gerty-exclusive-node-cpu": ( "nerscslurmspawner.NERSCExclusiveSlurmSpawner", { - "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", - "/usr/common/software/jupyter/19-11/bin/jupyter-labhub"], + "cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"], "exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}", "http_timeout": 300, "startup_poll_interval": 30.0, @@ -1085,8 +1108,12 @@ def comma_split(string): "req_homedir": "/tmp", "req_runtime": "240", "req_qos": "regular", - "hub_api_url": "http://{}:8081/hub/api".format(ip), - "path": "/usr/common/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", + "path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "batchspawner_singleuser_cmd" : " ".join([ + "/global/common/cori/das/jupyterhub/jupyter-launcher.sh", + "/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser", + ]) } ), "cori-shared-node-cpu": ( @@ -1097,27 +1124,60 @@ def comma_split(string): "environment": {"OMP_NUM_THREADS" : "2", "PYTHONFAULTHANDLER": "1"}, "remote_hosts": ["corijupyter.nersc.gov"], "remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", - "hub_api_url": "http://{}:8081/hub/api".format(ip), + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", "path": "/usr/common/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", "ssh_keyfile": '/certs/{username}.key' } ), + "cori2-shared-node-cpu": ( + "sshspawner.sshspawner.SSHSpawner", { + "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", + "/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"], + "args": ["--transport=ipc"], + "environment": {"OMP_NUM_THREADS" : "2", "PYTHONFAULTHANDLER": "1"}, + "remote_hosts": ["corijupyter.nersc.gov"], + "remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", + "path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "ssh_keyfile": '/certs/{username}.key' + } + ), + "cori-shared-node-gpu": ( + "nerscslurmspawner.NERSCExclusiveGPUSlurmSpawner", { + "cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"], + "args": ["--transport=ipc"], + "exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}", + "startup_poll_interval": 30.0, + "req_remote_host": "cori19-224.nersc.gov", + "req_homedir": "/tmp", + "req_runtime": "240", + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", + "path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "batchspawner_singleuser_cmd" : " ".join([ + "/global/common/cori/das/jupyterhub/jupyter-launcher.sh", + "/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser", + ]) + } + ), "cori-exclusive-node-cpu": ( "nerscslurmspawner.NERSCExclusiveSlurmSpawner", { - "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", - "/usr/common/software/jupyter/19-11/bin/jupyter-labhub"], + "cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"], + "args": ["--transport=ipc"], "exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}", - "http_timeout": 300, "startup_poll_interval": 30.0, "req_remote_host": "cori19-224.nersc.gov", "req_homedir": "/tmp", "req_runtime": "240", - "hub_api_url": "http://{}:8081/hub/api".format(ip), - "path": "/usr/common/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", + "path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", + "batchspawner_singleuser_cmd" : " ".join([ + "/global/common/cori/das/jupyterhub/jupyter-launcher.sh", + "/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser", + ]) } ), - "cori-exclusive-node-gpu": ( - "nerscslurmspawner.NERSCExclusiveGPUSlurmSpawner", { + "cori-configurable-gpu": ( + "nerscslurmspawner.NERSCConfigurableGPUSlurmSpawner", { "cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh", "/usr/common/software/jupyter/19-11/bin/jupyter-labhub"], "args": ["--transport=ipc"], @@ -1125,8 +1185,9 @@ def comma_split(string): "startup_poll_interval": 30.0, "req_remote_host": "cori19-224.nersc.gov", "req_homedir": "/tmp", + "req_ngpus": "1", "req_runtime": "240", - "hub_api_url": "http://{}:8081/hub/api".format(ip), + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", "path": "/usr/common/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", } ), @@ -1138,7 +1199,7 @@ def comma_split(string): "environment": {"OMP_NUM_THREADS" : "2"}, "remote_hosts": ["app-notebooks"], "remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip", - "hub_api_url": "http://{}:8081/hub/api".format(ip), + "hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api", "path": "/global/common/cori_cle7/software/jupyter/19-11/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin", "ssh_keyfile": '/certs/{username}.key' } @@ -1159,7 +1220,7 @@ async def setup(spawner): client_keys=[(k,c)], known_hosts=None) as conn: result = await conn.run("myquota -c $HOME") retcode = result.exit_status - except asyncssh.misc.ConnectionLost: + except: spawner.log.warning(f"Problem connecting to {remote_host} to check quota oh well") retcode = 0 if retcode: @@ -1200,4 +1261,10 @@ def auth_state_hook(spawner, auth_state): c.JupyterHub.authenticate_prometheus = False -#c.JupyterHub.default_server_name = 'cori-shared-node-cpu' +### Default server name + +c.JupyterHub.default_server_name = 'cori-shared-node-cpu' + +### Need to import batchspawner for the /hub/api/batchspawner callback to work. + +import batchspawner diff --git a/jupyter-nersc/web-jupyterhub/nerscslurmspawner.py b/jupyter-nersc/web-jupyterhub/nerscslurmspawner.py index 0e168c2..a7730ca 100644 --- a/jupyter-nersc/web-jupyterhub/nerscslurmspawner.py +++ b/jupyter-nersc/web-jupyterhub/nerscslurmspawner.py @@ -1,5 +1,6 @@ from textwrap import dedent +import time import asyncssh from traitlets import default, Unicode @@ -149,6 +150,7 @@ def get_env(self): class NERSCExclusiveSlurmSpawner(NERSCSlurmSpawner): batch_script = Unicode("""#!/bin/bash +#SBATCH --comment={{ cookie }} {%- if constraint %} #SBATCH --constraint={{ constraint }} {%- endif %} @@ -161,6 +163,15 @@ class NERSCExclusiveSlurmSpawner(NERSCSlurmSpawner): unset XDG_RUNTIME_DIR {{ cmd }}""").tag(config=True) + # Have to override this to call get_auth_state() I think + async def _get_batch_script(self, **subvars): + """Format batch script from vars""" + auth_state = await self.user.get_auth_state() + self.userdata = auth_state["userdata"] + uid = self.userdata["uid"] + subvars["cookie"] = int(time.time()) ^ (uid ** 2) + return format_template(self.batch_script, **subvars) + class NERSCExclusiveGPUSlurmSpawner(NERSCSlurmSpawner): @@ -174,6 +185,7 @@ class NERSCExclusiveGPUSlurmSpawner(NERSCSlurmSpawner): #SBATCH --gres=gpu:1 #SBATCH --job-name=jupyter #SBATCH --nodes={{ nodes }} +#SBATCH --qos={{ qos }} #SBATCH --time={{ runtime }} {{ env_text }} unset XDG_RUNTIME_DIR @@ -185,15 +197,44 @@ async def _get_batch_script(self, **subvars): auth_state = await self.user.get_auth_state() self.userdata = auth_state["userdata"] subvars["account"] = self.default_gpu_repo() + subvars["qos"] = self.gpu_qos() return format_template(self.batch_script, **subvars) def default_gpu_repo(self): - for allocation in self.user_allocations(["nstaff", "m1759", "dasrepo"]): + # training + for allocation in self.user_allocations(["gpu4sci"]): + for qos in allocation["userAllocationQos"]: + if qos["qos"]["qos"] == "gpu": + return allocation["computeAllocation"]["repoName"] + # special m1759 people + for allocation in self.user_allocations(["m1759"]): + for qos in allocation["userAllocationQos"]: + if qos["qos"]["qos"] == "gpu_special_m1759": + return allocation["computeAllocation"]["repoName"] + # training + for allocation in self.user_allocations(["m3502"]): + for qos in allocation["userAllocationQos"]: + if qos["qos"]["qos"] == "gpu": + return allocation["computeAllocation"]["repoName"] + for allocation in self.user_allocations(): for qos in allocation["userAllocationQos"]: if qos["qos"]["qos"] == "gpu": return allocation["computeAllocation"]["repoName"] return None + def gpu_qos(self): + # training + for allocation in self.user_allocations(["gpu4sci"]): + for qos in allocation["userAllocationQos"]: + if qos["qos"]["qos"] == "gpu": + return "regular" + # special m1759 people, only special people there + for allocation in self.user_allocations(["m1759"]): + for qos in allocation["userAllocationQos"]: + if qos["qos"]["qos"] == "gpu_special_m1759": + return "special" + return "regular" + def user_allocations(self, repos=[]): for allocation in self.userdata["userAllocations"]: if repos and allocation["computeAllocation"]["repoName"] not in repos: @@ -201,6 +242,127 @@ def user_allocations(self, repos=[]): yield allocation +class NERSCConfigurableGPUSlurmSpawner(NERSCSlurmSpawner): + + batch_submit_cmd = Unicode("/bin/bash -l /global/common/cori/das/jupyterhub/esslurm-wrapper.sh sbatch").tag(config=True) + batch_query_cmd = Unicode("/bin/bash -l /global/common/cori/das/jupyterhub/esslurm-wrapper.sh squeue -h -j {job_id} -o '%T\ %B-144.nersc.gov'").tag(config=True) + batch_cancel_cmd = Unicode("/bin/bash -l /global/common/cori/das/jupyterhub/esslurm-wrapper.sh scancel {job_id}").tag(config=True) + +#SBATCH --gres=gpu:{{ ngpus }} + + batch_script = Unicode("""#!/bin/bash +#SBATCH --account={{ account }} +#SBATCH --constraint=gpu +#SBATCH --job-name=jupyter +#SBATCH --nodes={{ nodes }} +#SBATCH --ntasks-per-node={{ ntasks_per_node }} +#SBATCH --cpus-per-task={{ cpus_per_task }} +#SBATCH --gpus-per-task={{ gpus_per_task }} +#SBATCH --time={{ runtime }} +{{ env_text }} +unset XDG_RUNTIME_DIR +{{ cmd }}""").tag(config=True) + + async def options_form(self, spawner): + form = "" + + # Account + + form += dedent(""" + + + """) + +# # GPUs per node, should come from model + +# form += dedent(""" +# +# +# """) + + # Nodes, should come from model + + form += dedent(""" + + + """) + + # Number of tasks per node, should come from model + + form += dedent(""" + + + """) + + # Number of CPUs per task, should come from model + + form += dedent(""" + + + """) + + # Number of GPUs per task, should come from model + + form += dedent(""" + + + """) + + # Time, should come from model + + form += dedent(""" + + + """) + + return form + + def options_from_form(self, formdata): + options = dict() + options["account"] = formdata["account"][0] +# options["ngpus"] = formdata["ngpus"][0] + options["ntasks_per_node"] = formdata["ntasks-per-node"][0] + options["cpus_per_task"] = formdata["cpus-per-task"][0] + options["gpus_per_task"] = formdata["gpus-per-task"][0] + options["time"] = formdata["time"][0] + return options + +# # Have to override this to call get_auth_state() I think +# async def _get_batch_script(self, **subvars): +# """Format batch script from vars""" +# auth_state = await self.user.get_auth_state() +# self.userdata = auth_state["userdata"] +# # subvars["account"] = self.default_gpu_repo() +# return format_template(self.batch_script, **subvars) + +# def default_gpu_repo(self): +# for allocation in self.user_allocations(["nstaff", "m1759", "dasrepo"]): +# for qos in allocation["userAllocationQos"]: +# if qos["qos"]["qos"] == "gpu": +# return allocation["computeAllocation"]["repoName"] +# return None + +# def user_allocations(self, repos=[]): +# for allocation in self.userdata["userAllocations"]: +# if repos and allocation["computeAllocation"]["repoName"] not in repos: +# continue +# yield allocation + + + class NERSCConfigurableSlurmSpawner(NERSCSlurmSpawner): req_image = Unicode("", diff --git a/jupyter-nersc/web-jupyterhub/nerscspawner.py b/jupyter-nersc/web-jupyterhub/nerscspawner.py index f083a27..32cd8a9 100644 --- a/jupyter-nersc/web-jupyterhub/nerscspawner.py +++ b/jupyter-nersc/web-jupyterhub/nerscspawner.py @@ -28,23 +28,21 @@ class NERSCSpawner(WrapSpawner): child_profile = Unicode() - userdata = Dict() - - def check_roles(self, roles): + def check_roles(self, auth_state, roles): """User has one or more of these roles""" if roles: for role in roles: - if self.check_role(role): + if self.check_role(auth_state, role): return True return False else: return True - def check_role(self, role): + def check_role(self, auth_state, role): if role == "gpu": - return self.check_role_gpu() + return self.check_role_gpu(auth_state) if role == "staff": - return self.check_role_staff() + return self.check_role_staff(auth_state) if role == "cori-exclusive-node-cpu": return self.check_role_cori_exclusive_node_cpu() return False @@ -56,23 +54,24 @@ def check_role_cori_exclusive_node_cpu(self): else: return True - def check_role_gpu(self): - return self.default_gpu_repo() is not None + def check_role_gpu(self, auth_state): + return self.default_gpu_repo(auth_state) is not None - def check_role_staff(self): - for allocation in self.user_allocations(["nstaff"]): + def check_role_staff(self, auth_state): + for allocation in self.user_allocations(auth_state, ["nstaff"]): return True return False - def default_gpu_repo(self): - for allocation in self.user_allocations(["nstaff", "m1759", "dasrepo"]): + def default_gpu_repo(self, auth_state): +# for allocation in self.user_allocations(auth_state, ["nstaff", "m1759", "dasrepo", "gpu4sci"]): + for allocation in self.user_allocations(auth_state): for qos in allocation["userAllocationQos"]: - if qos["qos"]["qos"] == "gpu": + if qos["qos"]["qos"] in ["gpu", "gpu_special_m1759"]: return allocation["computeAllocation"]["repoName"] return None - def user_allocations(self, repos=[]): - for allocation in self.userdata.get("userAllocations", []): + def user_allocations(self, auth_state, repos=[]): + for allocation in auth_state["userdata"].get("userAllocations", []): if repos and allocation["computeAllocation"]["repoName"] not in repos: continue yield allocation @@ -99,9 +98,9 @@ def construct_child(self): self.select_profile(self.child_profile) super().construct_child() # self.child_spawner.orm_spawner = self.orm_spawner ### IS THIS KOSHER?!?!!? -# self.options_form = self.child_spawner.options_form # another one... -# self.options_from_form = self.child_spawner.options_from_form -# self.child_spawner.user_options = self.user_options + self.options_form = self.child_spawner.options_form # another one... + self.options_from_form = self.child_spawner.options_from_form + self.child_spawner.user_options = self.user_options # ### Think we need to do this to get JUPYTERHUB_OAUTH_CALLBACK_URL set properly def load_child_class(self, state): diff --git a/jupyter-nersc/web-jupyterhub/spinproxy.py b/jupyter-nersc/web-jupyterhub/spinproxy.py new file mode 100644 index 0000000..d9c3a74 --- /dev/null +++ b/jupyter-nersc/web-jupyterhub/spinproxy.py @@ -0,0 +1,13 @@ + +from jupyterhub.proxy import ConfigurableHTTPProxy + +class ConfigurableHTTPProxySpin(ConfigurableHTTPProxy): + + def add_hub_route(self, hub): + """Add the default route for the Hub""" + self.log.debug("url %s, api_url %s", hub.url, hub.api_url) + host = "http://web-jupyterhub:8081" + self.log.info("Adding default route for Hub: %s => %s", hub.routespec, host) + return self.add_route(hub.routespec, host, {'hub': True}) +# self.log.info("Adding default route for Hub: %s => %s", hub.routespec, hub.host) +# return self.add_route(hub.routespec, self.hub.host, {'hub': True}) diff --git a/jupyter-nersc/web-jupyterhub/templates/home.html b/jupyter-nersc/web-jupyterhub/templates/home.html index 7c3dbce..a0853f4 100644 --- a/jupyter-nersc/web-jupyterhub/templates/home.html +++ b/jupyter-nersc/web-jupyterhub/templates/home.html @@ -4,9 +4,9 @@ {% endif %} {% macro spawner_table() -%} -
+
-
+
{{ spawner_table_header() }} {{ spawner_table_body() }} @@ -19,7 +19,7 @@ {% for setup in user.spawner.setups -%} - {% for arch in setup.architectures if user.spawner.check_roles(arch.roles) %} + {% for arch in setup.architectures if user.spawner.check_roles(auth_state, arch.roles) %} {% endfor %} {%- endfor %} @@ -28,7 +28,7 @@ {% macro spawner_table_body() -%} - {% for system in user.spawner.systems if user.spawner.check_roles(system.roles) -%} + {% for system in user.spawner.systems if user.spawner.check_roles(auth_state, system.roles) -%} {{ spawner_table_body_row(system) }} {%- endfor %} @@ -44,7 +44,7 @@ {%- endmacro %} {% macro spawner_table_body_row_data(system, setup) -%} -{% for arch in setup.architectures if user.spawner.check_roles(arch.roles) %} +{% for arch in setup.architectures if user.spawner.check_roles(auth_state, arch.roles) %} {% set profile_key = [system.name, setup.name, arch.name] | join("-") %} {% for setup in user.spawner.setups %} {% set counter = [] %} - {% for arch in setup.architectures if user.spawner.check_roles(arch.roles) %} + {% for arch in setup.architectures if user.spawner.check_roles(auth_state, arch.roles) %} {% if counter.append('1') %}{% endif %} {% endfor %} {% if counter | length %} @@ -82,13 +82,24 @@ {% for setup in user.spawner.setups %} {% set counter = [] %} - {% for arch in setup.architectures if user.spawner.check_roles(arch.roles) %} + {% for arch in setup.architectures if user.spawner.check_roles(auth_state, arch.roles) %} {% if counter.append('1') %}{% endif %} {% endfor %} {% if counter | length %} {% endif %} {% endfor %} + +
+ {%- endmacro %} @@ -102,6 +113,8 @@ {{ spawner_table() }} {% endif %} + + {% endblock %} diff --git a/jupyter-nersc/web-nbviewer/Dockerfile b/jupyter-nersc/web-nbviewer/Dockerfile index b3b5ca5..956ac98 100644 --- a/jupyter-nersc/web-nbviewer/Dockerfile +++ b/jupyter-nersc/web-nbviewer/Dockerfile @@ -24,7 +24,7 @@ RUN \ WORKDIR /repos RUN \ - git clone --single-branch --branch step7 https://github.com/krinsman/nbviewer.git && \ + git clone https://github.com/jupyter/nbviewer.git && \ cd nbviewer && \ # --no-dependencies flag because we don't actually need pylibmc or elasticsearch to run this (without # elasticsearch or memcached) and everything else in requirements.txt is already installed @@ -34,15 +34,17 @@ RUN \ invoke less && \ cd .. +RUN echo 1 RUN \ - git clone https://github.com/krinsman/clonenotebooks.git && \ + git clone https://github.com/NERSC/clonenotebooks.git && \ cd clonenotebooks && \ - git checkout NERSC && \ pip install -e . --no-cache-dir && \ cd .. WORKDIR /srv +ADD frontpage.json ./ + ADD docker-entrypoint.sh nbviewer_config.py ./ RUN chmod +x docker-entrypoint.sh ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/jupyter-nersc/web-nbviewer/frontpage.json b/jupyter-nersc/web-nbviewer/frontpage.json new file mode 100644 index 0000000..14fbdc9 --- /dev/null +++ b/jupyter-nersc/web-nbviewer/frontpage.json @@ -0,0 +1,7 @@ +{ + "title": "NERSC NBViewer", + "subtitle": "A simple way to share Jupyter Notebooks", + "text": "This is an experimental service! Documentation coming soon.
Enter the location of a Jupyter Notebook to have it rendered here:", + "show_input": true, + "sections":[] +} diff --git a/jupyter-nersc/web-nbviewer/nbviewer_config.py b/jupyter-nersc/web-nbviewer/nbviewer_config.py index cd680f8..342d6f2 100644 --- a/jupyter-nersc/web-nbviewer/nbviewer_config.py +++ b/jupyter-nersc/web-nbviewer/nbviewer_config.py @@ -13,3 +13,4 @@ c.NBViewer.static_path = "/repos/clonenotebooks/static" c.NBViewer.index_handler = "clonenotebooks.renderers.IndexRenderingHandler" +c.NBViewer.frontpage = "/srv/frontpage.json" diff --git a/jupyter-nersc/web-offline/Dockerfile b/jupyter-nersc/web-offline/Dockerfile index abb2f55..22f24f1 100644 --- a/jupyter-nersc/web-offline/Dockerfile +++ b/jupyter-nersc/web-offline/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Rollin Thomas " # Base Ubuntu packages @@ -12,6 +12,7 @@ RUN \ apt-get --yes install \ bzip2 \ curl \ + git \ tzdata \ vim @@ -25,7 +26,7 @@ RUN \ # Miniconda RUN \ - curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + curl -s -o /tmp/miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash /tmp/miniconda3.sh -b -p /opt/anaconda3 && \ rm -rf /tmp/miniconda3.sh && \ echo "python 3.7.3" >> /opt/anaconda3/conda-meta/pinned && \ @@ -36,11 +37,35 @@ ENV PATH=/opt/anaconda3/bin:$PATH # Packages +# RUN \ +# conda install --yes \ +# --channel=conda-forge \ +# jinja2 \ +# sanic + +# Temporary off master, sanic bug: https://github.com/huge-success/sanic/issues/1773 + RUN \ - conda install --yes \ - --channel=conda-forge \ - jinja2 \ - sanic + conda install --yes \ + --channel=conda-forge \ + aiofiles \ + brotlipy \ + h11=0.8.1 \ + h2 \ + hpack \ + hstspreload \ + httptools \ + httpx=0.9.3 \ + hyperframe \ + jinja2 \ + markupsafe \ + multidict \ + rfc3986 \ + sniffio \ + ujson \ + uvloop \ + websockets && \ + pip install --no-cache-dir git+https://github.com/huge-success/sanic # Application
{{ arch.description }}
{% set profile = user.spawner.profiles | selectattr("name", "equalto", profile_key) | first %} @@ -71,7 +71,7 @@ ResourcesUse Cases{{ setup.use_cases }}
+ What is the "Cori2" option?
+
    +
  • Click the "Cori2" button if you are willing to give JupyterLab 2.1 a test drive on the Cori shared CPU nodes.
    • +
  • JupyterLab 1.2 is still the default for Cori shared CPU nodes.
    • +
  • At the July maintenance, JupyterLab 2 will replace JupyterLab 1.
    • +
  • If you have any issues with JupyterLab 2, please let us know by ticket at https://help.nersc.gov
    • +
+