Merge pull request #459 from OpenConext/feature/mariadb_docker

Add small MariaDB docker role, for running on a single node

roles/attribute-aggregation/defaults/main.yml

@@ -9,3 +9,5 @@ aa_manage_provision_oidcrp_description_en: "OAuth client to access VOOT for grou
 aa_manage_provision_oidcrp_grants: "client_credentials"
 aa_manage_provision_oidcrp_allowed_resource_servers: '{"name": "{{ voot.oidcng_checkToken_clientId }}"}'
 aa_spring_flyway_enabled: true
+aa_docker_networks:
+  - name: loadbalancer

roles/attribute-aggregation/tasks/main.yml

@@ -21,15 +21,21 @@
     - apachelink.conf
   notify: restart attribute-aggregationserver
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    aa_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create and start the server container
   community.docker.docker_container:
     name: aaserver
     image: ghcr.io/openconext/openconext-attribute-aggregation/aa-server:{{ attribute_aggregation_server_version }}
     pull: true
     restart_policy: "always"
     state: started
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ aa_docker_networks }}"
     mounts:
       - source: /opt/openconext/attribute-aggregation/serverapplication.yml
         target: /application.yml

roles/engineblock/defaults/main.yml

@@ -107,3 +107,5 @@ engine_site_notice_show: false
 engineblock_log_attributes: []
 
 engine_php_memory: 256M
+engine_docker_networks:
+  - name: loadbalancer

roles/engineblock/tasks/main.yml

@@ -176,14 +176,20 @@
     name: engineblock_sessions
     state: present
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    engine_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create the container
   community.docker.docker_container:
     name: "engineblock"
     image: ghcr.io/openconext/openconext-engineblock/openconext-engineblock:{{ engine_version }}
     pull: true
     restart_policy: "always"
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ engine_docker_networks}}"
     labels:
       traefik.http.routers.engine.rule: "Host(`engine.{{ base_domain }}`)"
       traefik.http.routers.engine.service: "engineblock"

roles/invite/defaults/main.yml

@@ -15,3 +15,5 @@ invite_manage_provision_oauth_rs_scopes: "openid"
 invite_mock_install: false
 # Override is in the dockerX.env host_var files
 invite_cronjobmaster: true
+invite_docker_networks:
+  - name: loadbalancer

roles/invite/tasks/main.yml

@@ -48,6 +48,13 @@
   when: invite_mock_install
   notify: restart inviteprovisioningmock
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    invite_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create and start the server container
   community.docker.docker_container:
     name: inviteserver
@@ -57,8 +64,7 @@
     pull: true
     restart_policy: "always"
     state: started
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ invite_docker_networks }}"
     mounts:
       - source: /opt/openconext/invite/serverapplication.yml
         target: /application.yml
@@ -80,7 +86,6 @@
       start_period: 10s
   register: inviteservercontainer
 
-
 - name: Create the client container
   community.docker.docker_container:
     name: inviteclient
@@ -133,6 +138,7 @@
     env:
       HTTPD_CSP: "{{ httpd_csp.strict_with_static_img }}"
 
+
 - name: Create and start the mock provisioning container
   community.docker.docker_container:
     name: inviteprovisioningmock
@@ -148,8 +154,7 @@
       - source: /etc/localtime
         target: /etc/localtime
         type: bind
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ invite_docker_networks }}"
     labels:
       traefik.http.routers.invitemock.rule: "Host(`mock.{{ base_domain }}`)"
       traefik.http.routers.invitemock.tls: "true"

roles/lifecycle/defaults/main.yml

@@ -11,3 +11,5 @@ lifecycle_api_enabled: true
 lifecycle_api_password: secret
 lifecycle_api_username: lifecycle
 current_release_config_dir_name: /opt/openconext/{{ appname }}
+lifecycle_docker_networks:
+  - name: loadbalancer

roles/lifecycle/tasks/main.yml

@@ -33,6 +33,13 @@
   notify:
     - restart {{ appname }}
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    lifecycle_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create the container
   community.docker.docker_container:
     name: "{{ appname }}"
@@ -41,8 +48,7 @@
       host.docker.internal: host-gateway
     pull: true
     restart_policy: "always"
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ lifecycle_docker_networks }}"
     labels:
       traefik.http.routers.lifecycle.rule: "Host(`lifecycle.{{ base_domain }}`)"
       traefik.http.routers.lifecycle.tls: "true"

roles/manage/defaults/main.yml

@@ -30,3 +30,5 @@ manage_tabs_enabled:
   - single_tenant_template
   - provisioning
   - sram
+manage_docker_networks:
+  - name: loadbalancer

roles/manage/tasks/main.yml

@@ -72,6 +72,13 @@
   notify:
     - "restart manageserver"
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    manage_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create and start the server container
   community.docker.docker_container:
     name: manageserver
@@ -80,8 +87,7 @@
     pull: true
     restart_policy: "always"
     state: started
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ manage_docker_networks}}"
     mounts:
       - source: /opt/openconext/manage/
         target: /config/

roles/mariadbdocker/defaults/main.yml

@@ -0,0 +1,3 @@
+docker_mariadb_network_range: "172.21.21.0/24"
+mysql_backup_user: backup_user
+backup_node: True

roles/mariadbdocker/files/settings.cnf

@@ -0,0 +1,2 @@
+[mariadb]
+sql_mode=NO_ENGINE_SUBSTITUTION

roles/mariadbdocker/tasks/main.yml

@@ -0,0 +1,114 @@
+---
+- name: Create MariaDB volume
+  community.docker.docker_volume:
+    name: openconext_mariadb
+    state: present
+- name: Create MariaDB config dir
+  ansible.builtin.file:
+    path: /opt/openconext/mariadb/
+    owner: root
+    group: root
+    mode: "0755"
+    state: directory
+
+- name: Copy mariadb config file
+  ansible.builtin.copy:
+    src: settings.cnf
+    dest: /opt/openconext/mariadb/settings.cnf
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Create MariaDB network
+  community.docker.docker_network:
+    name: openconext_mariadb
+    state: present
+    internal: false
+    ipam_config:
+      - subnet: "{{ docker_mariadb_network_range }}"
+
+- name: Create the MariaDB container
+  community.docker.docker_container:
+    name: openconext_mariadb
+    image: mariadb:10.6
+    state: started
+    pull: true
+    restart_policy: "always"
+    ports: "127.0.0.1:3306:3306"
+    networks:
+      - name: "openconext_mariadb"
+    mounts:
+      - type: volume
+        source: openconext_mariadb
+        target: /var/lib/mysql
+      - type: bind
+        source: /opt/openconext/mariadb/settings.cnf
+        target: /etc/mysql/conf.d/settings.cnf
+    env:
+      MARIADB_ROOT_PASSWORD: "{{ mariadb_root_password }}"
+
+- name: Create database
+  community.mysql.mysql_db:
+    name: "{{ item }}"
+    state: present
+    login_user: root
+    login_host: localhost
+    login_password: "{{ mariadb_root_password }}"
+  with_items:
+    - "{{ databases.names }}"
+
+- name: Create database user
+  community.mysql.mysql_user:
+    name: "{{ item[0].name }}"
+    host: "{{ item[1] }}"
+    password: "{{ item[0].password }}"
+    priv: "{{ item[0].db_name }}.*:{{ item[0].privilege }}"
+    state: present
+    append_privs: true
+    login_user: root
+    login_host: localhost
+    login_password: "{{ mariadb_root_password }}"
+  #  no_log: true
+  with_nested:
+    - "{{ databases.users }}"
+    - "{{ database_clients }}"
+
+- name: Add mariadb backup user
+  community.mysql.mysql_user:
+    name: "{{ mysql_backup_user }}"
+    password: "{{ mysql_backup_password }}"
+    login_user: root
+    login_password: "{{ mariadb_root_password }}"
+    login_host: localhost
+    priv: "*.*:SELECT,RELOAD,PROCESS,LOCK TABLES,BINLOG MONITOR,CONNECTION ADMIN,SHOW VIEW"
+    state: present
+  #  no_log: true
+
+- name: Create the backup directory
+  ansible.builtin.file:
+    path: /home/backup
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when:
+    - backup_node | bool
+
+- name: Put mariadb_backup script
+  ansible.builtin.template:
+    src: "mariadb_backup.sh.j2"
+    dest: "/usr/local/sbin/mariadb_backup.sh"
+    mode: "0700"
+    owner: root
+  when:
+    - backup_node | bool
+
+- name: Create cron symlink for backup script
+  file:
+    src: /usr/local/sbin/mariadb_backup.sh
+    dest: /etc/cron.daily/db_backup
+    state: link
+    mode: 0700
+    owner: root
+  when:
+    - backup_node | bool

roles/mariadbdocker/templates/mariadb_backup.sh.j2

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+umask 0077
+
+declare -x PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin
+
+MYSQL_USER="{{ mysql_backup_user }}"
+MYSQL_PASS="{{ mysql_backup_password }}"
+FOLDER="/home/backup"
+
+DAY=$(/bin/date +'%a')
+
+echo "-- Remove old backups --"
+find /home/backup/ -type f -ctime +2 -delete
+
+echo "-- START new backups --"
+
+echo "SET autocommit=0;SET unique_checks=0;SET foreign_key_checks=0;" > tmp_sqlhead.sql
+echo "SET autocommit=1;SET unique_checks=1;SET foreign_key_checks=1;" > tmp_sqlend.sql
+
+if [ -z "$1" ]
+  then
+    echo "-- Dumping all DB ..."
+    for I in $(docker exec openconext_mariadb mariadb -u $MYSQL_USER --password=$MYSQL_PASS -e 'show databases' -s --skip-column-names);
+    do
+      if [ "$I" = information_schema ] || [ "$I" =  mysql ] || [ "$I" =  sys ] || [ "$I" =  performance_schema ]  # exclude this DB
+      then
+         echo "-- Skip $I ..."
+       continue
+      fi
+      echo "-- Dumping $I ..."
+      # Pipe compress and concat the head/end with the stoutput of mysqlump ( '-' cat argument)
+      docker exec openconext_mariadb mysqldump -u $MYSQL_USER --password=$MYSQL_PASS $I | cat tmp_sqlhead.sql - tmp_sqlend.sql | gzip -fc > "$FOLDER/$DAY-$I.sql.gz"
+    done
+
+else
+      I=$1;
+      echo "-- Dumping $I ..."
+      # Pipe compress and concat the head/end with the stoutput of mysqlump ( '-' cat argument)
+      docker exec openconext_mariadb mysqldump -u $MYSQL_USER --password=$MYSQL_PASS $I | cat tmp_sqlhead.sql - tmp_sqlend.sql | gzip -fc > "$FOLDER/$DAY-$I.sql.gz"
+fi
+
+# remove tmp files
+rm tmp_sqlhead.sql
+rm tmp_sqlend.sql
+
+echo "-- FINISH —"
+
+umask 0022

roles/pdp/defaults/main.yml

@@ -21,3 +21,5 @@ pdp_manage_provision_samlsp_trusted_proxy: false
 pdp_manage_provision_samlsp_sign: false
 pdp_spring_flyway_enabled: true
 pdp_manage_push_testmode: true
+pdp_docker_networks:
+  -name: loadbalancer

roles/pdp/tasks/main.yml

@@ -20,6 +20,13 @@
     - xacml.conext.properties
   notify: restart pdpserver
 
+- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker
+  ansible.builtin.set_fact:
+    pdp_docker_networks:
+      - name: loadbalancer
+      - name: openconext_mariadb
+  when: mariadb_in_docker | default(false) | bool
+
 - name: Create and start the server container
   community.docker.docker_container:
     name: pdpserver
@@ -29,8 +36,7 @@
     pull: true
     restart_policy: "always"
     state: started
-    networks:
-      - name: "loadbalancer"
+    networks: "{{ pdp_docker_networks }}"
     mounts:
       - source: /opt/openconext/pdp/serverapplication.properties
         target: /application.properties
@@ -59,7 +65,7 @@
       retries: 3
       start_period: 10s
   register: pdpservercontainer
-    
+
 - name: Create the gui container
   community.docker.docker_container:
     name: pdpgui

roles/spdashboard/defaults/main.yml

@@ -37,3 +37,5 @@ spdashboard_oidcng_playground_uri_test: https://oidc-playground.dev.support.surf
 spdashboard_oidcng_playground_uri_prod: https://oidc-playground.dev.support.surfconext.nl/redirect
 spdashboard_show_global_notice: False
 spdashboard_global_notice_date: "01-01-2020"
+spdashboard_docker_networks: 
+  - name: loadbalancer