diff --git a/README.md b/README.md index ffcf52a76..854dc4187 100644 --- a/README.md +++ b/README.md @@ -52,34 +52,51 @@ To run the full stack including the database and API, we use docker-compose 1. Define the `.env` file to change the default configuration (user credential to enable/disable GPU on the cluster, Slack config): - - ```bash - DB_PASSWORD=password - API_PASSWORD=password - CLUSTER_API_KEY=token - SLACK_BOT_TOKEN=xoxb-0000000000-0000000000-0000000000 - SLACK_CHANNEL=C03B48CQ3QW + SLACK_CHANNEL=C00000000QW + CLUSTER_API_KEY="" + SMTP_FROM=email@email.nl + SMTP_HOST= + SMTP_PORT= + SAML_SP_ENTITY_ID= + SAML_SALT= + SAML_ADMIN_PW= + SAML_TRUSTED_DOMAINS= + SAML_BASEURLPATH= + FOLDER_BOOKING_APP=/foo + FOLDER_SAML_CONFIG=/foo + FOLDER_SAML_METADATA=/foo + FOLDER_SAML_CERT=/foo ``` +### + 2. Run the stack: * API on http://localhost:8000, automatically reloaded on change to the code, with CRON job enabled * The GPU calendar on http://localhost:8001 * Database accessible through phpMyAdmin on http://localhost:8002 - ```bash - docker-compose up + ```bash + # Don't forget to switch to the right volumes for development! + + # Build images for dsri-api, cron, gpu-calender and gpu-reservation + docker compose build --no-cache + + # Start the containers (detached), you can also choose to up specific services e.g. append: nginx_proxy dsri-api + docker compose up -d --force-recreate ``` > ⚠️ The first time you start the stack you might need to stop and restart the stack once the SQL database has been initialized for the API to properly connect to the database +### + 3. In another terminal, run the website on http://localhost:3000, it will use the local API to display stats: ```bash @@ -100,6 +117,8 @@ yarn upgrade Alternatively you can also change the packages versions requirements in the `package.json` and run `yarn` +### + ## 🚀 Deploy in production ### Deploy the frontend to GitHub pages @@ -113,20 +132,40 @@ The GitHub Action will automatically compile the website to HTML in the `gh-page Define the `.env` file to change the default configuration (user credential to enable/disable GPU on the cluster, Slack config): ```bash +DB_NAME=dsri-db +DB_USER=dsri-user DB_PASSWORD=password +DB_HOST=mysql API_PASSWORD=password -CLUSTER_USER=dsri.username -CLUSTER_PASSWORD=password SLACK_BOT_TOKEN=xoxb-0000000000-0000000000-0000000000 -SLACK_CHANNEL=C03B48CQ3QW +SLACK_CHANNEL=C00000000QW +CLUSTER_API_KEY="" +SMTP_FROM=email@email.nl +SMTP_HOST= +SMTP_PORT= +SAML_SP_ENTITY_ID= +SAML_SALT= +SAML_ADMIN_PW= +SAML_TRUSTED_DOMAINS= +SAML_BASEURLPATH= +FOLDER_BOOKING_APP=/foo +FOLDER_SAML_CONFIG=/foo +FOLDER_SAML_METADATA=/foo +FOLDER_SAML_CERT=/foo ``` Start the docker-compose in production using [nginx-proxy](https://github.com/nginx-proxy/nginx-proxy). ```bash -docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d +# Build images for dsri-api, cron, gpu-calender and gpu-reservation +docker compose build --no-cache + +# Start the containers (detached), you can also choose to up specific services e.g. append: nginx_proxy dsri-api +docker compose up -d --force-recreate ``` +### + ### Database setup #### Import CSV @@ -168,6 +207,8 @@ To change the password of your database user, click on the **SQL** tab, and exec SET PASSWORD FOR 'username'@'%' = PASSWORD('newpassword'); ``` +### + ### Backup #### Export as CSV diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index e89b58624..000000000 --- a/deploy.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -## Deploy the stack to a server through SSH - -## Use cache: -ssh ids1 'cd /data/deploy-services/dsri-documentation ; git pull ; docker compose -f docker-compose.yml -f docker-compose.prod.yml up --force-recreate --build -d' - -## Build without cache: -# ssh ids1 'cd /data/deploy-services/dsri-documentation ; git pull ; docker-compose -f docker-compose.yml -f docker-compose.prod.yml build --no-cache ; docker-compose down ; docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d' diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100755 index 80eec48a1..000000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: "3" - -services: - - # In dev we can use 1 container for the API and CRON job - api: - ports: - - 8000:80 - environment: - - ENABLE_CRON=true - volumes: - - ./server/api:/app/api - - ./backup:/backup - command: uvicorn api.main:app --host 0.0.0.0 --port 80 --reload - - - gpu-calendar: - ports: - - 8001:80 - - gpu-booking: - ports: - - 8001:80 - - mysql: - volumes: - - ./data:/var/lib/mysql - - ./backup:/backup - # - ./server/cron/backup_database.sh:/app/backup_database.sh - # - ./server/cron/backup_database.sh:/app/restore_backup.sh - # ports: - # - 3306:3306 - - phpmyadmin: - ports: - - 8002:80 - - - # cronjob: - # volumes: - # - ./server/api:/app/api - # - ./server/cron:/app/cron - # - ./server/cron/gpu-notification-cron:/etc/cron.d/gpu-notification-cron - # - ./backup:/backup diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml deleted file mode 100755 index d15e3f1cb..000000000 --- a/docker-compose.prod.yml +++ /dev/null @@ -1,123 +0,0 @@ -version: "3" - -# To deploy in prod we use a nginx proxy for docker containers with a companion to enable HTTPS with LetsEncrypt -# https://github.com/nginx-proxy/nginx-proxy -# https://github.com/nginx-proxy/acme-companion - -services: - - api: - volumes: - - /data/database/backup:/backup - environment: - - VIRTUAL_HOST=api.dsri.maastrichtuniversity.nl - - LETSENCRYPT_HOST=api.dsri.maastrichtuniversity.nl - - VIRTUAL_PORT=80 - - - # Run the API on a single thread just for the CRON job in prod (enabled with env variable) - # Because gunicorn would run the process on every workers - cron: - build: ./server - restart: unless-stopped - depends_on: - - mysql - - phpmyadmin - volumes: - - /data/database/backup:/backup - environment: - - SQL_URL=mysql://${DB_USER}:${DB_PASSWORD-password}@${DB_HOST-mysql}:3306/${DB_NAME} - - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN-xoxb} - - SLACK_CHANNEL=${SLACK_CHANNEL-UQL6BCQJH} - # - CLUSTER_USER=${CLUSTER_USER-Vincent.Emonet} - # - CLUSTER_PASSWORD=${CLUSTER_PASSWORD-password} - - CLUSTER_API_KEY=${CLUSTER_API_KEY-token} - - API_PASSWORD=${API_PASSWORD-password} - - ENABLE_CRON=true - command: uvicorn api.main:app --host 0.0.0.0 --port 80 - - - gpu-calendar: - environment: - - VIRTUAL_HOST=calendar.dsri.maastrichtuniversity.nl - - LETSENCRYPT_HOST=calendar.dsri.maastrichtuniversity.nl - - VIRTUAL_PORT=80 - - gpu-booking: - environment: - - VIRTUAL_HOST=booking.dsri.maastrichtuniversity.nl - - LETSENCRYPT_HOST=booking.dsri.maastrichtuniversity.nl - - VIRTUAL_PORT=80 - - DB_PASSWORD=${DB_PASSWORD} - - DB_USER=${DB_USER} - - DB_NAME=${DB_NAME} - - DB_HOST=${DB_HOST-mysql} - - SMTP_HOST=${SMTP_HOST-localhost} - - SMTP_PORT=${SMTP_PORT-25} - - SMTP_FROM=${SMTP_FROM} - - SAML_SP_ENTITY_ID=${SAML_SP_ENTITY_ID} - - SAML_SALT=${SAML_SALT} - - SAML_ADMIN_PW=${SAML_ADMIN_PW} - - SAML_TRUSTED_DOMAINS=${SAML_TRUSTED_DOMAINS} - - SAML_BASEURLPATH=${SAML_BASEURLPATH} - - mysql: - volumes: - - /data/database/data:/var/lib/mysql - - /data/database/backup:/backup - - phpmyadmin: - environment: - - PMA_ABSOLUTE_URI=https://admin.dsri.maastrichtuniversity.nl - - VIRTUAL_HOST=admin.dsri.maastrichtuniversity.nl - - LETSENCRYPT_HOST=admin.dsri.maastrichtuniversity.nl - - VIRTUAL_PORT=80 - - - # https://github.com/nginx-proxy/nginx-proxy - nginx-proxy: - image: nginxproxy/nginx-proxy:latest - container_name: nginx-proxy - ports: - - 80:80 - - 443:443 - volumes: - - /data/certs:/etc/nginx/certs:ro - - /var/run/docker.sock:/tmp/docker.sock:ro - - vhost:/etc/nginx/vhost.d - - html:/usr/share/nginx/html - restart: unless-stopped - - # Use letsencrypt to serve HTTPS https://github.com/nginx-proxy/acme-companion - # Docs: https://github.com/nginx-proxy/acme-companion/wiki/Container-configuration - nginx-encrypt-https: - image: nginxproxy/acme-companion:latest - container_name: acme-companion - depends_on: - - nginx-proxy - volumes: - - /data/certs:/etc/nginx/certs:rw - - /var/run/docker.sock:/var/run/docker.sock:ro - - acme:/etc/acme.sh - volumes_from: - - nginx-proxy:rw - environment: - - NGINX_PROXY_CONTAINER=nginx-proxy - - REUSE_KEY=false - - DEBUG=true - - DEFAULT_EMAIL=DSRI-SUPPORT-L@maastrichtuniversity.nl - restart: unless-stopped - - - ## Website could be also served with nginx from the same server as the API - # website: - # build: website - # environment: - # - VIRTUAL_HOST=dsri.maastrichtuniversity.nl - # - LETSENCRYPT_HOST=dsri.maastrichtuniversity.nl - # - VIRTUAL_PORT=80 - -volumes: - vhost: - html: - acme: diff --git a/docker-compose.yml b/docker-compose.yml index 1456ea7f1..574b3936e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,86 +1,228 @@ -version: "3" +# +# +# services: # Frontend deployed on GitHub Pages by a GitHub Action + ## Website could be also served with nginx from the same server as the API + # website: + # build: website + # environment: + # - VIRTUAL_HOST=dsri.maastrichtuniversity.nl + # - LETSENCRYPT_HOST=dsri.maastrichtuniversity.nl + # - VIRTUAL_PORT=80 + + # To deploy in prod we use a nginx proxy for docker containers with a companion to enable HTTPS with LetsEncrypt + # https://github.com/nginx-proxy/nginx-proxy + # https://github.com/nginx-proxy/acme-companion + + nginx-proxy: + image: nginxproxy/nginx-proxy:latest + container_name: nginx-proxy + ports: + - 80:80 + - 443:443 + volumes: + # - ./data/certs:/etc/nginx/certs:ro # use this in dev + - /data/certs:/etc/nginx/certs:ro + - /var/run/docker.sock:/tmp/docker.sock:ro + - vhost:/etc/nginx/vhost.d + - html:/usr/share/nginx/html + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.4 + + # Use letsencrypt to serve HTTPS https://github.com/nginx-proxy/acme-companion + # Docs: https://github.com/nginx-proxy/acme-companion/wiki/Container-configuration + nginx-encrypt-https: + image: nginxproxy/acme-companion:latest + container_name: acme-companion + depends_on: + - nginx-proxy + environment: + - NGINX_PROXY_CONTAINER=nginx-proxy + - REUSE_KEY=false + - DEBUG=true + - DEFAULT_EMAIL=DSRI-SUPPORT-L@maastrichtuniversity.nl + volumes_from: + - nginx-proxy:rw + volumes: + # - ./data/certs:/etc/nginx/certs:rw # use this in dev + - /data/certs:/etc/nginx/certs:rw + - /var/run/docker.sock:/var/run/docker.sock:ro + - acme:/etc/acme.sh + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.5 + mysql: restart: unless-stopped image: mariadb:10.3 + container_name: mysql-db environment: MYSQL_DATABASE: dsri-db - MYSQL_ROOT_PASSWORD: ${DB_PASSWORD-password} + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD} MARIADB_USER: dsri-user - MARIADB_PASSWORD: ${DB_PASSWORD-password} - MARIADB_ROOT_PASSWORD: ${DB_PASSWORD-password} + MARIADB_PASSWORD: ${DB_PASSWORD} + MARIADB_ROOT_PASSWORD: ${DB_PASSWORD} # volumes: - # - ./data:/var/lib/mysql - # - ./backup:/backup - # - ./server/cron/backup_database.sh:/app/backup_database.sh - # - ./server/cron/backup_database.sh:/app/restore_backup.sh - # docker-compose exec mysql chown mysql: /backup - # - ./server/schemas:/docker-entrypoint-initdb.d + # - ./server/cron/backup_database.sh:/app/backup_database.sh + # - ./server/cron/backup_database.sh:/app/restore_backup.sh + # docker-compose exec mysql chown mysql: /backup + # - ./server/schemas:/docker-entrypoint-initdb.d command: --max-allowed-packet=16M + volumes: + # - ./data/database/data:/var/lib/mysql # use this in dev + # - ./data/database/backup:/backup # use this in dev + - /data/database/data:/var/lib/mysql + - /data/database/backup:/backup + networks: + default: + ipv4_address: 172.19.0.6 phpmyadmin: image: phpmyadmin/phpmyadmin:latest - restart: unless-stopped + container_name: phpmyadmin depends_on: - mysql + # ports: + # - 8002:80 environment: - - PMA_PASSWORD=${DB_PASSWORD-password} - - MYSQL_ROOT_PASSWORD=${DB_PASSWORD-password} - - MYSQL_PASSWORD=${DB_PASSWORD-password} + - PMA_PASSWORD=${DB_PASSWORD} + - MYSQL_ROOT_PASSWORD=${DB_PASSWORD} + - MYSQL_PASSWORD=${DB_PASSWORD} - PMA_HOST=mysql:3306 + - PMA_ABSOLUTE_URI=https://admin.dsri.maastrichtuniversity.nl + - VIRTUAL_HOST=admin.dsri.maastrichtuniversity.nl + - LETSENCRYPT_HOST=admin.dsri.maastrichtuniversity.nl + - VIRTUAL_PORT=80 + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.7 - api: + dsri-api: build: ./server - restart: unless-stopped + container_name: dsri-api depends_on: - mysql - phpmyadmin + command: uvicorn api.main:app --host 0.0.0.0 --port 80 --reload + # ports: + # - 8000:80 environment: - - SQL_URL=mysql://dsri-user:${DB_PASSWORD-password}@mysql:3306/dsri-db - - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN-xoxb} - - SLACK_CHANNEL=${SLACK_CHANNEL-UQL6BCQJH} - # - CLUSTER_USER=${CLUSTER_USER-Vincent.Emonet} - # - CLUSTER_PASSWORD=${CLUSTER_PASSWORD-password} - - CLUSTER_API_KEY=${CLUSTER_API_KEY-token} - - API_PASSWORD=${API_PASSWORD-password} - + - VIRTUAL_HOST=api.dsri.maastrichtuniversity.nl + - LETSENCRYPT_HOST=api.dsri.maastrichtuniversity.nl + - VIRTUAL_PORT=80 + - SQL_URL=mysql://dsri-user:${DB_PASSWORD}@mysql:3306/dsri-db + - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} + - SLACK_CHANNEL=${SLACK_CHANNEL} + - CLUSTER_API_KEY=${CLUSTER_API_KEY} + - API_PASSWORD=${API_PASSWORD} + - ENABLE_CRON=true + volumes: + # - ./data/database/backup:/backup:ro # use this in dev + - /data/database/backup:/backup + - ./server/api:/app/api + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.8 gpu-calendar: build: ./gpu-calendar - restart: unless-stopped + container_name: gpu-calendar depends_on: - mysql - phpmyadmin + # ports: + # - 8001:80 environment: - - DB_PASSWORD=${DB_PASSWORD-password} + - DB_PASSWORD=${DB_PASSWORD} + - VIRTUAL_HOST=calendar.dsri.maastrichtuniversity.nl + - LETSENCRYPT_HOST=calendar.dsri.maastrichtuniversity.nl + - VIRTUAL_PORT=80 + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.9 gpu-booking: build: ./gpu-reservation/docker/php - restart: unless-stopped + container_name: gpu-booking depends_on: - mysql - phpmyadmin + # ports: + # - 8001:80 environment: - - DB_PASSWORD=${DB_PASSWORD-password} + - VIRTUAL_HOST=booking.dsri.maastrichtuniversity.nl + - LETSENCRYPT_HOST=booking.dsri.maastrichtuniversity.nl + - VIRTUAL_PORT=80 + - DB_PASSWORD=${DB_PASSWORD} + - DB_USER=${DB_USER} + - DB_NAME=${DB_NAME} + - DB_HOST=${DB_HOST-mysql} + - SMTP_HOST=${SMTP_HOST} + - SMTP_PORT=${SMTP_PORT} + - SMTP_FROM=${SMTP_FROM} + - SAML_SP_ENTITY_ID=${SAML_SP_ENTITY_ID} + - SAML_SALT=${SAML_SALT} + - SAML_ADMIN_PW=${SAML_ADMIN_PW} + - SAML_TRUSTED_DOMAINS=${SAML_TRUSTED_DOMAINS} + - SAML_BASEURLPATH=${SAML_BASEURLPATH} # Bind the container folder to the host folder env setting volumes: - ${FOLDER_BOOKING_APP}:/var/www/html - ${FOLDER_SAML_CONFIG}:/var/www/simplesamlphp/config - ${FOLDER_SAML_METADATA}:/var/www/simplesamlphp/metadata - ${FOLDER_SAML_CERT}:/var/www/simplesamlphp/cert + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.10 - # cron: - # # Same image as the API - # build: ./server - # restart: unless-stopped - # depends_on: - # - mysql - # - phpmyadmin - # environment: - # - SQL_URL=mysql://dsri-user:${DB_PASSWORD-password}@mysql:3306/dsri-db - # - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN-xoxb} - # - SLACK_CHANNEL=${SLACK_CHANNEL-UQL6BCQJH} - # command: /app/cron/start_cron.sh + # Run the API on a single thread just for the CRON job in prod (enabled with env variable) + # Because gunicorn would run the process on every workers + cron: + build: ./server + container_name: cron + depends_on: + - mysql + - phpmyadmin + command: uvicorn api.main:app --host 0.0.0.0 --port 80 + environment: + - SQL_URL=mysql://${DB_USER}:${DB_PASSWORD}@${DB_HOST-mysql}:3306/${DB_NAME} + - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} + - SLACK_CHANNEL=${SLACK_CHANNEL} + # - CLUSTER_USER=${CLUSTER_USER-Vincent.Emonet} + # - CLUSTER_PASSWORD=${CLUSTER_PASSWORD-password} + - CLUSTER_API_KEY=${CLUSTER_API_KEY} + - API_PASSWORD=${API_PASSWORD} + - ENABLE_CRON=true + volumes: + # - ./data/database/backup:/backup # use this in dev + - /data/database/backup:/backup + # - ./server/api:/app/api + # - ./server/cron:/app/cron + # - ./server/cron/gpu-notification-cron:/etc/cron.d/gpu-notification-cron + restart: unless-stopped + networks: + default: + ipv4_address: 172.19.0.11 + +networks: + default: + driver: bridge + ipam: + config: + - subnet: "172.19.0.0/24" + gateway: 172.19.0.1 + +volumes: + vhost: + html: + acme: diff --git a/restart.sh b/restart.sh deleted file mode 100755 index b6191bda4..000000000 --- a/restart.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Script to restart the stack directly from the server where it is deployed - -git pull - -docker compose -f docker-compose.yml -f docker-compose.prod.yml up --force-recreate --build -d -