Skip to content
This repository has been archived by the owner on Apr 29, 2021. It is now read-only.

Commit

Permalink
Merge pull request #80 from NERSC/20-09
Browse files Browse the repository at this point in the history
20 09
  • Loading branch information
rcthomas authored Sep 24, 2020
2 parents 6b7edfa + 7792969 commit 021ccbe
Show file tree
Hide file tree
Showing 9 changed files with 241 additions and 30 deletions.
7 changes: 3 additions & 4 deletions jupyter-base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ RUN \
oauthlib \
pamela \
psycopg2 \
pycurl \
pyopenssl \
pyrsistent \
python-dateutil \
Expand All @@ -58,7 +59,7 @@ RUN \
ruamel.yaml.clib \
sqlalchemy \
tornado \
traitlets \
traitlets=4.3.3 \
zipp

# Install JupyterHub
Expand All @@ -69,9 +70,7 @@ RUN \
npm install -g configurable-http-proxy && \
git clone https://github.com/jupyterhub/jupyterhub.git && \
cd jupyterhub && \
git checkout tags/1.1.0 && \
# git checkout tags/1.1.0 && \
/opt/anaconda3/bin/python setup.py js && \
/opt/anaconda3/bin/pip --no-cache-dir install . && \
cp examples/cull-idle/cull_idle_servers.py /opt/anaconda3/bin/. && \
chmod u+x /opt/anaconda3/bin/cull_idle_servers.py && \
rm -rf ~/.cache ~/.npm
14 changes: 14 additions & 0 deletions jupyter-nersc/app-idle-culler/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
ARG branch=unknown

FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
LABEL maintainer="Rollin Thomas <[email protected]>"

RUN \
pip install --no-cache-dir jupyterhub-idle-culler

WORKDIR /srv

ADD docker-entrypoint.sh ./
RUN chmod +x docker-entrypoint.sh
ENTRYPOINT ["./docker-entrypoint.sh"]
CMD ["python3", "-m", "jupyterhub_idle_culler", "--timeout=57600", "--cull-every=3600", "--url=http://web-jupyterhub:8081/hub/api"]
28 changes: 28 additions & 0 deletions jupyter-nersc/app-idle-culler/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/bash

imcmd=""
for command in docker podman; do
if [ $(command -v $command) ]; then
imcmd=$command
break
fi
done
if [ -n "$imcmd" ]; then
echo "Using $imcmd"
else
echo "No image command defined"
exit 1
fi

format=""
if [ "$imcmd" == "podman" ]; then
format="--format docker"
fi

branch=$(git symbolic-ref --short HEAD)

$imcmd build \
--build-arg branch=$branch \
$format \
"$@" \
--tag registry.spin.nersc.gov/das/app-idle-culler.jupyter-nersc-$branch:latest .
28 changes: 28 additions & 0 deletions jupyter-nersc/app-idle-culler/docker-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/bash

# file_env VAR [DEFAULT]
# ----------------------
# Treat the value of VAR_FILE as the path to a secrets file and initialize VAR
# with the contents of that file. From postgres docker-entrypoint.sh.

file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}

file_env 'JUPYTERHUB_API_TOKEN'

exec "$@"
1 change: 1 addition & 0 deletions jupyter-nersc/web-jupyterhub/docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,6 @@ file_env 'CONFIGPROXY_AUTH_TOKEN'
file_env 'JUPYTERHUB_CRYPT_KEY'
file_env 'ANNOUNCEMENT_JUPYTERHUB_API_TOKEN'
file_env 'NBVIEWER_JUPYTERHUB_API_TOKEN'
file_env 'IDLE_CULLER_JUPYTERHUB_API_TOKEN'

exec "$@"
76 changes: 50 additions & 26 deletions jupyter-nersc/web-jupyterhub/jupyterhub_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ def comma_split(string):
{
'name': 'cull-idle',
'admin': True,
'command': 'cull_idle_servers.py --timeout=57600'.split(),
'api_token': os.environ["IDLE_CULLER_JUPYTERHUB_API_TOKEN"]
},
{
'name': 'announcement',
Expand Down Expand Up @@ -860,7 +860,7 @@ def comma_split(string):
#
# .. versionadded: 0.9
#c.Authenticator.blacklist = set()
c.Authenticator.blacklist = set(comma_split(os.environ.get("BLACKLIST")))
c.Authenticator.blocked_users = set(comma_split(os.environ.get("BLOCKED_USERS")))

## Enable persisting auth_state (if available).
#
Expand Down Expand Up @@ -901,7 +901,7 @@ def comma_split(string):
#
# If empty, does not perform any additional restriction.
#c.Authenticator.whitelist = set()
c.Authenticator.whitelist = set(comma_split(os.environ.get("WHITELIST")))
c.Authenticator.allowed_users = set(comma_split(os.environ.get("ALLOWED_USERS")))

#------------------------------------------------------------------------------
# LocalAuthenticator(Authenticator) configuration
Expand Down Expand Up @@ -1019,6 +1019,7 @@ def comma_split(string):
{ "name": "cori-exclusive-node-cpu" },
{ "name": "cori-exclusive-node-largemem" },
{ "name": "cori-configurable-gpu" },
{ "name": "cori-configurable-dgx" },
{ "name": "spin-shared-node-cpu" },
]

Expand Down Expand Up @@ -1064,6 +1065,11 @@ def comma_split(string):
"name": "gpu",
"description": "Configurable GPU",
"roles": ["gpu"],
},
{
"name": "dgx",
"description": "Configurable DGX",
"roles": ["dgx"],
}
],
"resources": "Use multiple compute nodes with specialized settings.",
Expand All @@ -1090,101 +1096,119 @@ def comma_split(string):
"gerty-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"environment": {"OMP_NUM_THREADS" : "2"},
"remote_hosts": ["gerty.nersc.gov"],
"remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"ssh_keyfile": '/certs/{username}.key'
}
),
"gerty-exclusive-node-cpu": (
"nerscslurmspawner.NERSCExclusiveSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"cmd": ["/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"http_timeout": 300,
"startup_poll_interval": 30.0,
"req_remote_host": "gerty.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "240",
"req_qos": "regular",
"req_qos": "jupyter",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser",
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/batchspawner-singleuser",
])
}
),
"cori-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"environment": {"OMP_NUM_THREADS" : "2", "PYTHONFAULTHANDLER": "1"},
"remote_hosts": ["corijupyter.nersc.gov"],
"remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"ssh_keyfile": '/certs/{username}.key'
}
),
"cori-shared-node-gpu": (
"nerscslurmspawner.NERSCExclusiveGPUSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"cmd": ["/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "240",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser",
"/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/batchspawner-singleuser",
])
}
),
"cori-exclusive-node-cpu": (
"nerscslurmspawner.NERSCExclusiveSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"cmd": ["/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "240",
"req_runtime": "480",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser",
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/batchspawner-singleuser",
])
}
),
"cori-exclusive-node-largemem": (
"nerscslurmspawner.NERSCExclusiveLargeMemSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"cmd": ["/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_runtime": "480",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser",
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/batchspawner-singleuser",
])
}
),
"cori-configurable-gpu": (
"nerscslurmspawner.NERSCConfigurableGPUSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"cmd": ["/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
"req_remote_host": "cori19-224.nersc.gov",
"req_homedir": "/tmp",
"req_ngpus": "1",
"req_runtime": "240",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/batchspawner-singleuser",
])
}
),
"cori-configurable-dgx": (
"nerscslurmspawner.NERSCConfigurableDGXSlurmSpawner", {
"cmd": ["/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"exec_prefix": "/usr/bin/ssh -q -o StrictHostKeyChecking=no -o preferredauthentications=publickey -l {username} -i /certs/{username}.key {remote_host}",
"startup_poll_interval": 30.0,
Expand All @@ -1193,23 +1217,23 @@ def comma_split(string):
"req_ngpus": "1",
"req_runtime": "240",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/usr/common/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"batchspawner_singleuser_cmd" : " ".join([
"/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/batchspawner-singleuser",
"/global/common/cori_cle7/software/jupyter/cgpu/20-09/bin/batchspawner-singleuser",
])
}
),
"spin-shared-node-cpu": (
"sshspawner.sshspawner.SSHSpawner", {
"cmd": ["/global/common/cori/das/jupyterhub/jupyter-launcher.sh",
"/global/common/cori_cle7/software/jupyter/20-06/bin/jupyterhub-singleuser"],
"/global/common/cori_cle7/software/jupyter/cori/20-09/bin/jupyterhub-singleuser"],
"args": ["--transport=ipc"],
"environment": {"OMP_NUM_THREADS" : "2"},
"remote_hosts": ["app-notebooks"],
"remote_port_command": "/usr/bin/python /global/common/cori/das/jupyterhub/new-get-port.py --ip",
"hub_api_url": f"https://{nersc_jupyterhub_subdomain}.nersc.gov/hub/api",
"path": "/global/common/cori_cle7/software/jupyter/20-06/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"path": "/global/common/cori_cle7/software/jupyter/cori/20-09/bin:/global/common/cori/das/jupyterhub:/usr/common/usg/bin:/usr/bin:/bin",
"ssh_keyfile": '/certs/{username}.key'
}
)
Expand Down
Loading

0 comments on commit 021ccbe

Please sign in to comment.