Skip to content
This repository has been archived by the owner on Apr 29, 2021. It is now read-only.

Commit

Permalink
Merge pull request #60 from NERSC/19-05
Browse files Browse the repository at this point in the history
19 05
  • Loading branch information
rcthomas authored May 22, 2019
2 parents 88d17cf + 1c3b1d7 commit 0a581b7
Show file tree
Hide file tree
Showing 11 changed files with 733 additions and 19 deletions.
8 changes: 6 additions & 2 deletions jupyter-base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ RUN \
curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \
rm -rf /tmp/miniconda3.sh && \
echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \
echo "python 3.7.*" >> /opt/anaconda3/conda-meta/pinned && \
/opt/anaconda3/bin/conda update --yes conda && \
/opt/anaconda3/bin/conda install --yes \
alembic \
cryptography \
decorator \
entrypoints \
jinja2 \
Expand All @@ -58,8 +59,11 @@ ENV PATH=/opt/anaconda3/bin:$PATH
WORKDIR /tmp
RUN \
npm install -g configurable-http-proxy && \
git clone https://github.com/jupyterhub/jupyterhub.git && \
# git clone https://github.com/jupyterhub/jupyterhub.git && \
git clone https://github.com/rcthomas/jupyterhub.git && \
cd jupyterhub && \
# git checkout tags/1.0.0 && \
git checkout auth-state-to-spawner && \
/opt/anaconda3/bin/python setup.py js && \
/opt/anaconda3/bin/pip --no-cache-dir install . && \
cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \
Expand Down
1 change: 0 additions & 1 deletion jupyter-nersc/app-monitoring/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ RUN \
curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \
rm -rf /tmp/miniconda3.sh && \
echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \
/opt/anaconda3/bin/conda update --yes conda && \
/opt/anaconda3/bin/pip install --no-cache-dir \
pika==0.13.1
Expand Down
10 changes: 8 additions & 2 deletions jupyter-nersc/web-jupyterhub/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,25 @@ WORKDIR /srv

# Authenticator and spawner

# pip install git+https://github.com/nersc/nerscspawner.git && \

RUN \
pip install git+https://github.com/nersc/sshapiauthenticator.git && \
pip install git+https://github.com/jupyterhub/batchspawner.git@4747946 && \
pip install git+https://github.com/jupyterhub/wrapspawner.git && \
pip install git+https://github.com/nersc/nerscspawner.git && \
pip install git+https://github.com/nersc/sshspawner.git

# Customized templates

ADD templates templates

# Announcement service
# FIXME Install this stuff

ENV PYTHONPATH=/srv
ADD nerscspawner.py .
ADD nerscslurmspawner.py .
ADD announcement.py .
ADD iris.py .

# Hub scripts

Expand Down
20 changes: 20 additions & 0 deletions jupyter-nersc/web-jupyterhub/hub-scripts/kill-cori.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

# Shut down all but the newest jupyterhub server running.

hostname=$1
username=$2
cert=/certs/$username.key
echo $username $cert
if [ ! -f $cert ]; then
echo " ... no cert for $username"
exit 1
fi
/usr/bin/ssh \
-i $cert \
-l $username \
-o PreferredAuthentications=publickey \
-o StrictHostKeyChecking=no \
-p 22 \
$hostname \
/global/common/shared/das/jupyterhub/kill-my-old-jupyters.sh
25 changes: 25 additions & 0 deletions jupyter-nersc/web-jupyterhub/hub-scripts/scram-user.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash

# Useful when JupyterHub thinks a user has no server running
# but actually they do, and they still have a cert.

hostname=$1
username=$2
cert=/certs/$username.key
echo $username $cert
if [ ! -f $cert ]; then
echo " ... SKIPPED no cert for $username"
continue
fi
for i in 1 2 3
do
/usr/bin/ssh \
-i $cert \
-l $username \
-o PreferredAuthentications=publickey \
-o StrictHostKeyChecking=no \
-p 22 \
$hostname \
killall -u $username
sleep 1
done
47 changes: 47 additions & 0 deletions jupyter-nersc/web-jupyterhub/iris.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@

from textwrap import dedent

from tornado import escape, httpclient

class Iris:

def __init__(self, iris_url="https://iris.nersc.gov/graphql"):
self.iris_url = iris_url

async def query_user(self, name):
query = dedent("""
query {{
systemInfo {{
users(name: "{}") {{
baseRepos {{
computeAllocation {{
repoName
}}
}}
userAllocations {{
computeAllocation {{
repoName
}}
userAllocationQos {{
qos {{
qos
}}
}}
}}
}}
}}
}}""".format(name)).strip()
data = await self.query(query)
return data["data"]["systemInfo"]["users"][0]

async def query(self, query):
client = httpclient.AsyncHTTPClient()
request = self.format_request(query)
response = await client.fetch(request)
return escape.json_decode(response.body)

def format_request(self, query):
return httpclient.HTTPRequest(self.iris_url,
method="POST",
headers={"Content-Type": "application/json"},
body=escape.json_encode({"query": query}))
116 changes: 109 additions & 7 deletions jupyter-nersc/web-jupyterhub/jupyterhub_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,8 @@ def comma_split(string):
#
# Should be a subclass of Spawner.
#c.JupyterHub.spawner_class = 'jupyterhub.spawner.LocalProcessSpawner'
c.JupyterHub.spawner_class = 'nerscspawner.nerscspawner.NERSCSpawner'
#c.JupyterHub.spawner_class = 'nerscspawner.nerscspawner.NERSCSpawner'
c.JupyterHub.spawner_class = 'nerscspawner.NERSCSpawner'

## Path to SSL certificate file for the public facing interface of the proxy
#
Expand Down Expand Up @@ -588,6 +589,7 @@ def comma_split(string):
# process's environment (such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the
# single-user server's process.
#c.Spawner.env_keep = ['PATH', 'PYTHONPATH', 'CONDA_ROOT', 'CONDA_DEFAULT_ENV', 'VIRTUAL_ENV', 'LANG', 'LC_ALL']
c.Spawner.env_keep = ['PATH', 'CONDA_ROOT', 'CONDA_DEFAULT_ENV', 'VIRTUAL_ENV', 'LANG', 'LC_ALL']

## Extra environment variables to set for the single-user server's process.
#
Expand All @@ -607,13 +609,13 @@ def comma_split(string):
# across upgrades, so if you are using the callable take care to verify it
# continues to work after upgrades!
#c.Spawner.environment = {}
c.Spawner.environment = {"OMP_NUM_THREADS" : "2"}

## Timeout (in seconds) before giving up on a spawned HTTP server
#
# Once a server has successfully been spawned, this is the amount of time we
# wait before assuming that the server is unable to accept connections.
#c.Spawner.http_timeout = 30
c.Spawner.http_timeout = 300

## The IP address (or hostname) the single-user server should listen on.
#
Expand Down Expand Up @@ -745,6 +747,7 @@ def comma_split(string):
# takes longer than this. start should return when the server process is started
# and its location is known.
#c.Spawner.start_timeout = 60
c.Spawner.start_timeout = 900

#------------------------------------------------------------------------------
# LocalProcessSpawner(Spawner) configuration
Expand Down Expand Up @@ -865,6 +868,7 @@ def comma_split(string):
#
# New in JupyterHub 0.8
#c.Authenticator.enable_auth_state = False
c.Authenticator.enable_auth_state = True

## Dictionary mapping authenticator usernames to JupyterHub users.
#
Expand Down Expand Up @@ -987,8 +991,11 @@ def comma_split(string):
#------------------------------------------------------------------------------

c.NERSCSpawner.profiles = [
{ "name": "cori-shared-node-cpu" },
{ "name": "spin-shared-node-cpu" },
{ "name": "gerty-shared-node-cpu" },
{ "name": "cori-shared-node-cpu" },
{ "name": "cori-exclusive-node-cpu" },
{ "name": "cori-exclusive-node-gpu" },
{ "name": "spin-shared-node-cpu" },
]

c.NERSCSpawner.setups = [
Expand All @@ -998,34 +1005,103 @@ def comma_split(string):
{
"name": "cpu",
"description": "Shared CPU Node",
"roles": [],
}
],
"resources": "Use a node shared with other users' notebooks but outside the batch queues.",
"use_cases": "Visualization and analytics that are not memory intensive and can run on just a few cores."
}
},
{
"name": "exclusive-node",
"architectures": [
{
"name": "cpu",
"description": "Exclusive CPU Node",
"roles": ["cori-exclusive-node-cpu"],
},
{
"name": "gpu",
"description": "Exclusive GPU Node",
"roles": ["gpu"],
}
],
"resources": "Use your own node within a job allocation using defaults.",
"use_cases": "Visualization, analytics, machine learning that is compute or memory intensive but can be done on a single node."
},
]

c.NERSCSpawner.systems = [
{ "name": "cori" },
{ "name": "spin" }
{
"name": "gerty",
"roles": ["staff"]
},
{
"name": "cori",
"roles": []
},
{
"name": "spin",
"roles": []
}
]

c.NERSCSpawner.spawners = {
"gerty-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori/software/python/3.6-anaconda-5.2/bin/jupyter-labhub"],
"environment": {"OMP_NUM_THREADS" : "2"},
"remote_hosts": ["gert01.nersc.gov"],
"remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": "http://{}:8081/hub/api".format(ip),
"path": "/global/common/cori/software/python/3.6-anaconda-5.2/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"ssh_keyfile": '/certs/{username}.key'
}
),
"cori-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori/software/python/3.6-anaconda-5.2/bin/jupyter-labhub"],
"environment": {"OMP_NUM_THREADS" : "2"},
"remote_hosts": ["corijupyter.nersc.gov"],
"remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": "http://{}:8081/hub/api".format(ip),
"path": "/global/common/cori/software/python/3.6-anaconda-5.2/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"ssh_keyfile": '/certs/{username}.key'
}
),
"cori-exclusive-node-cpu": (
"nerscslurmspawner.NERSCExclusiveSlurmSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori/software/python/3.6-anaconda-5.2/bin/jupyter-labhub"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"http_timeout": 300,
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "240",
"hub_api_url": "http://{}:8081/hub/api".format(ip),
"path": "/global/common/cori/software/python/3.6-anaconda-5.2/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
}
),
"cori-exclusive-node-gpu": (
"nerscslurmspawner.NERSCExclusiveGPUSlurmSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori/software/python/3.6-anaconda-5.2/bin/jupyter-labhub"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "240",
"hub_api_url": "http://{}:8081/hub/api".format(ip),
"path": "/global/common/cori/software/python/3.6-anaconda-5.2/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
}
),
"spin-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/opt/anaconda3/bin/jupyter-labhub"],
"environment": {"OMP_NUM_THREADS" : "2"},
"remote_hosts": ["app-notebooks"],
"remote_port_command": "/opt/anaconda3/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": "http://{}:8081/hub/api".format(ip),
Expand Down Expand Up @@ -1069,6 +1145,32 @@ async def setup(spawner):

# c.Spawner.pre_spawn_hook = setup

###

from iris import Iris

async def post_auth_hook(authenticator, handler, authentication):
iris = Iris()
userdata = await iris.query_user(authentication["name"])
if authentication["auth_state"] is None:
authentication["auth_state"] = {}
authentication["auth_state"]["userdata"] = userdata
return authentication

c.Authenticator.post_auth_hook = post_auth_hook

###

def auth_state_hook(spawner, auth_state):
spawner.userdata = auth_state["userdata"]

c.Spawner.auth_state_hook = auth_state_hook

### Prometheus

c.JupyterHub.authenticate_prometheus = False


## c.NERSCSpawner.spawners = [
## ("spin", "sshspawner.sshspawner.SSHSpawner", {
## "remote_hosts" : ["jupyter"],
Expand Down
Loading

0 comments on commit 0a581b7

Please sign in to comment.