Merge branch 'master' of github.com:UBC-DSCI/rudaux

.github/workflows/publish_docker_images.yml

@@ -0,0 +1,50 @@
+name: Publish JupyterHub Docker images on DockerHub
+on: 
+  push: 
+    paths: 
+      - 'dockerfiles/dsci-student/Dockerfile'
+      - 'dockerfiles/dsci-grading/Dockerfile'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - uses: actions/checkout@master
+      with:
+        fetch-depth: '0'
+    - name: Bump version and push tag
+      uses: anothrNick/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        WITH_V: true
+      id: bump
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+      with:
+        tag_name: ${{ steps.bump.outputs.new_tag }}
+        release_name: ${{ steps.bump.outputs.new_tag }}
+        body: |
+          Changes in this Release
+          - Rebuilt r-dsci-100 Docker image and published to DockerHub with new tag
+          - Rebuilt r-dsci-grading Docker image and published to DockerHub with new tag
+        draft: false
+        prerelease: false
+    - name: Publish to Registry
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: ubcstat/dsci-student
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        workdir: dockerfiles/dsci-student
+        tags: "${{ steps.bump.outputs.new_tag }}"
+    - name: Publish to Registry
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: ubcdsci/dsci-grading
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        workdir: dockerfiles/dsci-grading
+        tags: "${{ steps.bump.outputs.new_tag }}"

ansible/.gitignore

@@ -0,0 +1,3 @@
+.hostfiles
+inventory
+bin

ansible/Makefile

@@ -0,0 +1,63 @@
+UNAME := $(shell uname | tr '[:upper:]' '[:lower:]')
+ARCH := $(shell uname -m)
+TOP := $(shell pwd)
+TF_PATH := $(TOP)/../terraform
+
+export PATH := $(TF_PATH)/bin/$(UNAME)_$(ARCH):$(PATH)
+
+#ANSIBLE_CMD  = TF_STATE=$(TF_PATH)/$(ENV) ansible --private-key=../keys/id_syzygy_stat -i ./inventory
+#PLAYBOOK_CMD = TF_STATE=$(TF_PATH)/$(ENV) ansible-playbook --private-key=../keys/id_syzygy_stat -i ./inventory
+ANSIBLE_CMD  = TF_STATE=$(TF_PATH)/$(ENV) ansible --private-key=~/.ssh/id_rsa -i ./inventory
+PLAYBOOK_CMD = TF_STATE=$(TF_PATH)/$(ENV) ansible-playbook --private-key=~/.ssh/id_rsa -i ./inventory
+
+help: tasks
+
+tasks: 
+	@grep -A1 ^HELP Makefile | sed -e ':begin;$$!N;s/HELP: \(.*\)\n\(.*:\).*/\2 \1/;tbegin;P;D' | grep -v \\\-\\\- | sort | awk -F: '{printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+
+check-playbook:
+ifndef PLAYBOOK
+	$(error PLAYBOOK is not defined [PLAYBOOK=])
+else
+export _PLAYBOOK = plays/$(PLAYBOOK).yml
+endif
+
+check-env:
+ifndef ENV
+	$(error environment is not defined [ENV=])
+endif
+
+check-limit:
+ifndef LIMIT
+export _LIMIT = --limit all
+else
+export _LIMIT = --limit $(LIMIT)
+endif
+
+HELP: Lists plays
+list-playbooks:
+	@grep -RH ^## plays/ | sed -e 's/\(plays\/\)\(.*\)\(.yml\)/\2/' | sort | awk 'BEGIN {FS = ":## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+
+HELP: Lists the tasks in a $PLAYBOOK in $ENV
+list-tasks: check-playbook check-limit
+	$(PLAYBOOK_CMD) --list-tasks $(_LIMIT) $(_PLAYBOOK)
+
+HELP: Lists all hosts in $ENV
+hosts: check-env check-limit
+	$(ANSIBLE_CMD) --list-hosts $(_LIMIT) all
+
+HELP: Run $PLAYBOOK ON $ENV in check-mode
+playbook/check: check-env check-playbook check-limit
+	$(PLAYBOOK_CMD) --check --diff $(_LIMIT) $(_PLAYBOOK)
+
+HELP: Run $PLAYBOOK ON $ENV
+playbook: check-env check-playbook check-limit
+	$(PLAYBOOK_CMD) $(_LIMIT) $(_PLAYBOOK)
+
+HELP: Setup ansible and refresh external roles
+ansible/setup: check-env check-limit
+	$(ANSIBLE_CMD) -m setup all
+	/bin/bash scripts/role_module_update.sh
+

ansible/ansible.cfg

@@ -0,0 +1,12 @@
+[defaults]
+remote_user = centos
+roles_path = ./roles/internal:./roles/external
+private_key_file = ~/.ssh/id_rsa
+# vault_password_file=~/.ansible_vault
+library = ./library/internal:./library/external
+hash_behaviour = merge
+host_key_checking=false
+
+[ssh_connection]
+pipelining = True
+

ansible/group_vars/all/docker.yml

@@ -0,0 +1,3 @@
+---
+docker_interface: 'docker0'
+docker_install_compose: false

ansible/group_vars/all/firewalld.yml

@@ -0,0 +1,3 @@
+firewalld_allow_services: 
+    - { service: 'http' }
+    - { service: 'https' }

ansible/group_vars/all/httpd.yml

@@ -0,0 +1 @@
+apache_create_vhosts: false

ansible/group_vars/all/local_vars.yml

@@ -0,0 +1,19 @@
+admin_email: [email protected]
+support_email: [email protected]
+domain: "dsc.stat.ubc.ca"
+jupyterhub_html_family: syzygy-html
+syzygy_html_longname: "UBC"
+
+# syzygyauthenticator_version: shib-ubc
+# jupyterhub_spawner: dockerspawner
+# jupyterhub_authenticator: shib
+
+# Devices should really be reported by the terraform provider and kept passed
+# back as variables inside the inventory, but I haven't figured that out yet.
+# For the moment, we need to set the zfs and docker device letters manually.
+# Thankfully the order they're specified in the terraform config is preserved,
+# so zfs is the second device and docker is the third.
+zfs_vdev_config: /dev/nvme1n1
+openstack_ephemeral_docker_disk: /dev/nvme2n1
+
+dehydrate_test_cert: False

ansible/group_vars/all/secret.yml

@@ -0,0 +1,9 @@
+# Stash of information we want to retain on the ansible client (e.g. certs)
+secret: "{{ inventory_dir | realpath }}/.hostfiles/secret"
+secret_host: "{{ secret }}/{{ inventory_hostname }}"
+secret_host_directories:
+  - name: 'acme_secrets'
+    path: 'acme'
+  - name: 'shib_secrets'
+    path: 'shib'
+

ansible/group_vars/all/ssh-public-keys.yml

@@ -0,0 +1,5 @@
+---
+ssh_public_keys:
+  - user: 'centos'
+    state: 'present'
+    public_key: 'ssh-rsa FAKEKEYAAAAB3NzaC1yc2EAAAADAQABAAACAQDsqCs8irideWHhAXD7UdjLyrNzSldJcz2d2IChszaB+Z1xThYAPITguMTe4w2kA9c4n9OcoKgn49D8JnNoXjM7Pc3Iy9qJOEe0Qyf+rnPuONAK5ar+BIcp/5JWW4y1aEpE4+MBQvdUd/6hJ86dOEFi9ngvhsMqR8usT2h5KRWHRgnlL0hL/TF/O4Ekm6eHgAsQbYiu0OMPYHmf8li8tpyXl4dMYTOJeLMQC5ZGh882oYarxmP5N0WdTKGQ5pNZ2RUYDws0IipBdpmDanfaXu8asaURTRzQwIM0OOIoEfQzgC/nZVJzvzWu/aY3PN95B0AFmgFKtcA8jKzPyXyNSHNs3IreFJ1pBLmjyXr+kQx/NOZTfGletUqU9+50UciVaEnkeGwT5UjDrP4dvLi66TjY2rrVJ2IE1UQPRFWurOjMkgQaSzPoiR0ocnDCFVIqHFIRnZYOKpHVqXqxfQg8e57NQsl2eaJ1FohcMgr5hfvU+NfIsBu1cbakmazqK6iy3Os5/Y2NxJI10Qgl9xVifSI3aS5ntqtS7dHEmvTWPR1cUHgTVNeylDcR+ULy2bM9Y8w== [email protected]'

ansible/group_vars/all/ssh.yml

@@ -0,0 +1,3 @@
+sshd_PasswordAuthenticaton: no
+sshd_PermitRootLogin: no
+sshd_AllowUsers: "{{ ansible_ssh_user }} jupyter"

ansible/group_vars/all/sudo.yml

@@ -0,0 +1,2 @@
+sudo_defaults:
+  - defaults: '!requiretty'

ansible/group_vars/hubs/.gitignore

@@ -0,0 +1 @@
+secrets.yml

ansible/group_vars/hubs/jupyterhub.yml

@@ -0,0 +1,23 @@
+jupyterhub_authenticator: 'ltiauthenticator'
+jupyterhub_course_name: 'dsci100'
+jupyterhub_docker_image: 'ubcdsci/r-dsci-100'
+jupyterhub_docker_container: 'ubcdsci/r-dsci-100:latest'
+
+jupyterhub_spawners:
+  dockerspawner:
+    name: 'dockerspawner.DockerSpawner'
+    options:
+      - conf_object: 'DockerSpawner.extra_host_config'
+        value: "{'mem_limit': '2g', 'memswap_limit': '2g', 'cpu_period': 100000, 'cpu_quota': 100000, 'ulimits': [{'name': 'core', 'hard': 0}] }"
+      - conf_object: 'DockerSpawner.remove_containers'
+        value: True
+      - conf_object: 'DockerSpawner.use_internal_ip'
+        value: "True"
+      - conf_object: 'DockerSpawner.host_ip'
+        value: "'0.0.0.0'"
+      - conf_object: 'DockerSpawner.image'
+        value: "'{{ jupyterhub_docker_container }}'"
+      - conf_object: 'DockerSpawner.volumes'
+        value: "{'/tank/home/dsci100/{username}': '/home/jupyter'}"
+
+jupyterhub_zfs_user_quota: '2G'

ansible/group_vars/hubs/rrsync.yml

@@ -0,0 +1,10 @@
+rrsync_enable: true
+rrsync_local_user: 'jupyter'
+rrsync_source_directory: '/tank/home/'
+rrsync_dest_directory: '/tank/backup/syzygy/{{ inventory_hostname }}/'
+rrsync_remote_host: '37zfs2.stat.ubc.ca'
+rrsync_remote_user: 'syzygy'
+rrsync_private_key_path: '.ssh/id_rrsync_backup_rsa'
+rrsync_public_key_path: '{{ rrsync_private_key_path }}.pub'
+
+rrsync_ssh_options: ',no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding'

ansible/group_vars/hubs/zfs-snapshot.yml

@@ -0,0 +1,10 @@
+---
+zfs_snapshot_tasks:
+  - name: 'wed'
+    minute: 10
+    hour: 5
+    weekday: 4
+  - name: 'sat'
+    minute: 10
+    hour: 1
+    weekday: 0

ansible/group_vars/markers/jupyterhub.yml

@@ -0,0 +1,24 @@
+jupyterhub_spawner: 'dockerspawner'
+jupyterhub_authenticator: 'shib'
+jupyterhub_course_name: 'dsci100'
+jupyterhub_docker_image: 'ubcdsci/r-dsci-grading'
+jupyterhub_docker_container: 'ubcdsci/r-dsci-grading:latest'
+
+jupyterhub_authenticators:
+  shib:
+    name: 'syzygyauthenticator.shib.RemoteUserAuthenticator'
+    options:
+      - conf_object: 'SyzygyAuthenticator.shibIDAttribute'
+        value: "'REMOTE_USER'"
+      - conf_object: 'SyzygyAuthenticator.shibUserEntitlements'
+        value: "'eduPersonEntitlement'"
+      - conf_object: 'SyzygyAuthenticator.shibValidEntitlements'
+        value: "set(['https://entitlements.ubc.ca/student/current', 'https://entitlements.ubc.ca/employee', 'https://entitlements.ubc.ca/faculty', 'https://entitlements.ubc.ca/student'])"
+      - conf_object: 'SyzygyAuthenticator.shibLogoutURL'
+        value: "'/Shibboleth.sso/Logout?return=https://authentication.ubc.ca/idp/profile/Logout'"
+      - conf_object: 'DockerSpawner.volumes'
+        value: "{'/tank/home/dsci100/{username}': '/home/jupyter'}"
+      - conf_object: 'Authenticator.whitelist'
+        value: "{'timberst', 'ianatr', 'hgill100', 'leem44', 'tdjc', 'petalvit', 'chowwj', 'dalimohd', 'jnbk', 'granseto'}"
+
+jupyterhub_zfs_user_quota: '10G'

ansible/group_vars/markers/python36.yml

@@ -0,0 +1,3 @@
+ius_extra_packages:
+  - pandas
+  - paramiko

ansible/group_vars/markers/shibboleth.yml

@@ -0,0 +1,13 @@
+---
+shib_idp_version: 3
+shib_id_attribute: 'eduPersonPrincipalName'
+shib_auth_attribute: 'eduPersonEntitlement'
+shib_auth_values: set([
+    'https://entitlements.ubc.ca/employee',
+    'https://entitlements.ubc.ca/faculty',
+    'https://entitlements.ubc.ca/student/current',
+    'https://entitlements.ubc.ca/student'])
+shibboleth_sso_entity_id: https://authentication.ubc.ca
+shibboleth_sso_discovery_url: https://authentication.ubc.ca/idp/shibboleth
+shib_idp_metadata_uri: https://authentication.ubc.ca/idp/shibboleth
+shib_auth_logout_url: '/Shibboleth.sso/Logout?return=https://authentication.ubc.ca/idp/profile/Logout'

ansible/group_vars/markers/syzygyauthenticator.yml

@@ -0,0 +1 @@
+syzygyauthenticator_version: shib-ubc

ansible/inventory

@@ -0,0 +1,7 @@
+course101.rudaux.com ansible_host=3.96.232.57
+
+[hubs]
+course101.rudaux.com
+
+[production:children]
+hubs

ansible/plays/.gitignore

@@ -0,0 +1 @@
+*.retry