From b93cbc0bbb0426cdfd9bb4fdee26c6b4d6f48ea6 Mon Sep 17 00:00:00 2001 From: Ebubekir Ates <109050136+Honigeintopf@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:56:20 +0200 Subject: [PATCH 1/4] Add role for partition management firewall setup (#336) --- partition/roles/mgmt-firewall/README.md | 120 ++++++++ .../roles/mgmt-firewall/defaults/main.yaml | 63 ++++ partition/roles/mgmt-firewall/tasks/main.yaml | 273 ++++++++++++++++++ 3 files changed, 456 insertions(+) create mode 100644 partition/roles/mgmt-firewall/README.md create mode 100644 partition/roles/mgmt-firewall/defaults/main.yaml create mode 100644 partition/roles/mgmt-firewall/tasks/main.yaml diff --git a/partition/roles/mgmt-firewall/README.md b/partition/roles/mgmt-firewall/README.md new file mode 100644 index 00000000..24ee482d --- /dev/null +++ b/partition/roles/mgmt-firewall/README.md @@ -0,0 +1,120 @@ +# Automated Firewall Setup with Ansible + +This role automates the configuration of management firewalls using Ansible. It is designed to streamline the process of setting up firewalls for consistent deployment before mounting devices in the data center. By utilizing default configurations and flexible variables, this role simplifies the setup across multiple devices. + +**Note**: This role is intended to be run on devices reset to factory defaults. + +## Supported Devices + +| Manufacturer | Model | +| ------------ | ------ | +| Teltonika | RUTXR1 | + +## Key Features + +- **Automated firewall setup** using default configurations +- **VLAN and BGP** configuration support +- **Dynamic port forwarding** setup +- **Pre-configured firewall rules** for LAN, WAN, and global settings +- **Device-specific customization** via `routers.yaml` + +## Prerequisites + +- The device must be **reset to factory defaults** before running this role. +- An initial login is required to change the root password using credentials defined in the `routers.yaml` file. + +## Configuration Details + +### Firewall Rules + +The firewall is configured with the following settings by default: + +1. **Global Settings:** + + - Drop invalid packets: **Enabled** + - Input: **Drop** + - Output: **Accept** + - Forward: **Drop** + - Offloading: **On** + +2. **LAN Configuration:** + + - Input, Output, Forward: **Accept** + - Masquerading: **On** + - MSS Clamping: **On** + +3. **WAN Configuration:** + - Input: **Drop** + - Output: **Accept** + - Forward: **Drop** + - Masquerading: **On** + - MSS Clamping: **On** + +### VLAN Configuration + +- **VLAN 1:** Tagged to port 4 +- **VLAN 2:** Tagged to port 5 (WAN) +- Other VLANs can be configured dynamically. + +### BGP Configuration + +- The BGP peer is **hardcoded** as `mgmtsrv`. +- The IP address and AS number can be configured dynamically. + +## Interfaces + +Both LAN and WAN interfaces share the following mandatory fields: + +| Field | Description | +| --------------------------------------------------------- | -------------- | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.name` | Interface name | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.ipaddr` | IP address | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.netmask` | Subnet mask | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.device` | Router port | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.dhcp_options` | (LAN Only) | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.metric` | (WAN Only) | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.gateway` | (WAN Only) | +| `mgmt_firewall_interfaces.mgmt_firewall_lan.dns` (List) | (WAN Only) | + +### Default WAN Interface + +To enable configuration of the default WAN interface, set `mgmt_firewall_default_wan_enabled` to `true`. + +| Field | Description | +| ------------------------------------------------------------ | --------------- | +| `mgmt_firewall_interfaces.mgmt_firewall_wan.default.name` | Interface name | +| `mgmt_firewall_interfaces.mgmt_firewall_wan.default.ipaddr` | IP address | +| `mgmt_firewall_interfaces.mgmt_firewall_wan.default.netmask` | Subnet mask | +| `mgmt_firewall_interfaces.mgmt_firewall_wan.default.device` | Router port | +| `mgmt_firewall_interfaces.mgmt_firewall_wan.default.gateway` | Default gateway | + +## Port Forwarding Configuration + +The following fields define port forwarding rules: + +| Field | Description | +| ------------------------------------------- | ------------------------------ | +| `mgmt_firewall_port_forwards.name` | Rule name | +| `mgmt_firewall_port_forwards.src_dport` | External port | +| `mgmt_firewall_port_forwards.dest_ip` | Internal IP address | +| `mgmt_firewall_port_forwards.dest_port` | Internal port | +| `mgmt_firewall_port_forwards.src` | Source zone | +| `mgmt_firewall_port_forwards.priority` | Rule priority (start with 1) | +| `mgmt_firewall_port_forwards.dest` | Destination zone | +| `mgmt_firewall_port_forwards.reflection` | NAT Loopback (0 = off, 1 = on) | +| `mgmt_firewall_port_forwards.src_ip` (List) | Source IP addresses | +| `mgmt_firewall_port_forwards.proto` (List) | Protocols (e.g., TCP, UDP) | +| `mgmt_firewall_port_forwards.src_dip` | External IP address | + +## Variables + +The following variables can be customized for each firewall: + +| Variable | Mandatory | Description | +| ------------------------------------- | --------- | ---------------------------------------------- | +| `mgmt_firewall_location_name` | yes | Location of the firewall | +| `mgmt_firewall_device_name` | yes | Device name | +| `mgmt_firewall_public_key` | yes | Public key for the firewall | +| `mgmt_firewall_default_wan_enabled` | | Default: false | +| `mgmt_firewall_wireless_disabled` | | Default: true | +| `mgmt_firewall_static_routes_enabled` | | Set up static routes, by specifying a gateway. | diff --git a/partition/roles/mgmt-firewall/defaults/main.yaml b/partition/roles/mgmt-firewall/defaults/main.yaml new file mode 100644 index 00000000..c573cf62 --- /dev/null +++ b/partition/roles/mgmt-firewall/defaults/main.yaml @@ -0,0 +1,63 @@ +mgmt_firewall_location_name: +mgmt_firewall_device_name: + +mgmt_firewall_default_wan_enabled: true +mgmt_firewall_static_routes_enabled: true + +mgmt_firewall_config: + location_name: '' + device_name: '' + bgp: + enabled: true + general_ip: '' + general_as: + mgmtsrv_ipaddr: '' + mgmtsrv_as: + +mgmt_firewall_interfaces: + mgmt_firewall_lan: + - name: '' + ipaddr: '' + netmask: '' + device: '' + dhcp_options: + - { option: '3', value: '' } + - { option: '6', value: '' } + - { option: '12', value: '' } + mgmt_firewall_wan: + default: + ip_adress: '' + gateway: '' + net_mask: '' + interfaces: + - name: wan_mgmtsrv + device: eth1 + metric: '5' + ipaddr: '' + netmask: '' + dns: + - '1.1.1.1' + - '1.0.0.1' + gateway: '' + +mgmt_firewall_port_forwards: + - name: 'ssh_mgmtsrv' + src_dport: '22' + dest_ip: '' + dest_port: '22' + src: 'wan' + priority: '1' + dest: 'lan' + reflection: '0' + src_ip: [''] + proto: ['tcp'] + src_dip: '' + +mgmt_firewall_vlans: + - vlan: '3' + vid: '3' + ports: '0t 1' + +mgmt_firewall_static_routes: + - gateway: '' + network: 1 diff --git a/partition/roles/mgmt-firewall/tasks/main.yaml b/partition/roles/mgmt-firewall/tasks/main.yaml new file mode 100644 index 00000000..f85aa42c --- /dev/null +++ b/partition/roles/mgmt-firewall/tasks/main.yaml @@ -0,0 +1,273 @@ +--- +- name: Check mandatory variables for this role are set + assert: + fail_msg: 'not all mandatory variables given, check role documentation' + quiet: yes + that: + - mgmt_firewall_location_name is not none + - mgmt_firewall_device_name is not none + - mgmt_firewall_public_key is not none + +- name: Setup BGP configuration + ansible.builtin.raw: | + uci set bgp.bgp.enabled='1' + uci set bgp.bgp.enabled_vty='1' + uci set bgp.general.enabled='1' + uci add_list bgp.general.redistribute='static' + uci add_list bgp.general.redistribute='connected' + uci add_list bgp.general.redistribute='kernel' + uci set bgp.general.id={{ mgmt_firewall_config.bgp.general_ip }} + uci set bgp.general.as='{{ mgmt_firewall_config.bgp.general_as }}' + uci set bgp.general.ebgp_requires_policy='1' + uci set bgp.general.deterministic_med='0' + when: mgmt_firewall_config.bgp.enabled + +- name: Setup BGP Peer + ansible.builtin.raw: | + uci set bgp.mgmtsrv=bgp_peer + uci set bgp.mgmtsrv.instance='general' + uci set bgp.mgmtsrv.default_originate='0' + uci set bgp.mgmtsrv.ipaddr='{{ mgmt_firewall_config.bgp.mgmtsrv_ipaddr }}' + uci set bgp.mgmtsrv.as='{{ mgmt_firewall_config.bgp.mgmtsrv_as }}' + uci set bgp.mgmtsrv.enabled='1' + uci commit bgp + /etc/init.d/frr restart + when: mgmt_firewall_config.bgp.enabled + +- name: Setup SSH + ansible.builtin.raw: | + uci set dropbear.@dropbear[0]._sshWanAccess='1' + uci set dropbear.@dropbear[0].enable_key_ssh='1' + uci set dropbear.@dropbear[0].RootPasswordAuth='0' + uci commit dropbear + /etc/init.d/dropbear restart + uci set firewall.15.enabled='1' # Enable SSH Wan + uci commit firewall + /etc/init.d/firewall + +- name: Setup firewall default settings + ansible.builtin.raw: | + uci set firewall.1.input='DROP' + uci set firewall.1.forward='DROP' + uci set firewall.1.drop_invalid='1' + uci set firewall.2.masq='1' + uci set firewall.2.mtu_fix='1' + uci set firewall.3.input='DROP' + uci set firewall.3.forward='DROP' + {% if mgmt_firewall_config.bgp.enabled is true %} + uci set firewall.A_BGP=rule + uci set firewall.A_BGP.enabled='1' + uci set firewall.A_BGP.src='wan' + uci set firewall.A_BGP.name='Allow-BGP-WAN-traffic' + uci set firewall.A_BGP.target='ACCEPT' + uci set firewall.A_BGP.dest_port='179' + uci add_list firewall.A_BGP.proto='tcp' + uci add_list firewall.A_BGP.proto='udp' + {% endif %} + uci commit firewall + /etc/init.d/firewall restart + +- name: Get the total number of sms_utils rules + ansible.builtin.raw: | + uci show sms_utils | grep -o '@rule\[[0-9]\+\]' | sort -u | wc -l + register: rule_count + +- name: Disable all sms_utils rules + ansible.builtin.raw: | + uci set sms_utils.@rule[{{ item }}].enabled='0' + loop: '{{ range(0, rule_count.stdout | int) }}' + register: disable_output + +- name: Commit and restart sms_utils after disabling rules + ansible.builtin.raw: | + uci commit sms_utils + /etc/init.d/sms_utils restart + +- name: Disable rms_connect + ansible.builtin.raw: | + uci set rms_mqtt.rms_connect_mqtt.enable='0' + uci commit rms_mqtt + /etc/init.d/rms_mqtt restart + +- name: Change location Name + ansible.builtin.raw: | + uci set snmpd.@system[0].sysName='{{ mgmt_firewall_location_name }}' + uci set system.system.devicename='{{ mgmt_firewall_device_name }}' + uci set system.system.hostname='{{ mgmt_firewall_location_name }}' + uci set system.system.zoneName='Europe/Berlin' + uci set system.system.timezone='CET-1CEST,M3.5.0,M10.5.0/3' + uci commit snmpd + uci commit system + /etc/init.d/system restart + +- name: Enable Remote Https Access + ansible.builtin.raw: | + uci set uhttpd.main._httpsWanAccess='1' + uci set uhttpd.main.redirect_https='0' + uci commit uhttpd + /etc/init.d/uhttpd restart + +- name: Disable wireless + ansible.builtin.raw: | + uci set wireless.default_radio1.disabled='1' + uci set wireless.default_radio0.disabled='1' + uci commit wireless + /etc/init.d/network restart + when: mgmt_firewall_wireless_disabled | default(true) + +- name: Create authorized keys file in /etc/dropbear + ansible.builtin.raw: | + echo '{{mgmt_firewall_public_key}}' > ../etc/dropbear/authorized_keys + +- name: Adjust Lan Default to not Bridge + ansible.builtin.raw: | + uci delete network.br_lan + uci delete network.br_lan.name + uci delete network.br_lan.type + uci delete network.br_lan.ports + uci delete network.lan.device + uci set network.lan.device='eth0' + uci commit network + /etc/init.d/network restart + +- name: Configure Default wan + ansible.builtin.raw: | + uci set network.wan.ipaddr='{{mgmt_firewall_interfaces.mgmt_firewall_wan.default.ip_adress}}' + uci set network.wan.netmask='{{mgmt_firewall_interfaces.mgmt_firewall_wan.default.net_mask}}' + uci set network.wan.gateway='{{mgmt_firewall_interfaces.mgmt_firewall_wan.default.gateway}}' + uci add_list network.wan.dns='1.1.1.1' + uci add_list network.wan.dns='1.0.0.1' # Hardcoded for now + uci set network.wan.peerdns='0' + uci set network.wan.proto='static' + uci commit network + /etc/init.d/network + uci set firewall.3.network='' + uci add_list firewall.3.network=wan + uci commit firewall + /etc/init.d/firewall + when: mgmt_firewall_default_wan_enabled | default(false) + +- name: Configure new LAN interfaces + ansible.builtin.raw: | + section_id=$(uci add network interface) + uci rename network.$section_id="{{ item.name }}" + uci set network.{{ item.name }}.proto='static' + uci set network.{{ item.name }}.ipaddr='{{ item.ipaddr }}' + uci set network.{{ item.name }}.netmask='{{ item.netmask }}' + uci set network.{{ item.name }}.device='{{ item.device }}' + uci set network.{{ item.name }}.delegate='1' + uci set network.{{ item.name }}.force_link='1' + uci set network.{{ item.name }}.area_type='lan' + uci commit network + /etc/init.d/network restart + uci add_list firewall.2.network="{{ item.name }}" + uci commit firewall + /etc/init.d/firewall restart + + loop: '{{ mgmt_firewall_interfaces.mgmt_firewall_lan }}' + +- name: Configure DHCP + ansible.builtin.raw: | + uci set dhcp.{{ item.name }}=dhcp + uci set dhcp.{{ item.name }}.ignore_ipv6='1' + uci set dhcp.{{ item.name }}.interface="{{ item.name }}" + uci set dhcp.{{ item.name }}.ra='server' + uci set dhcp.{{ item.name }}.dhcpv6='server' + uci set dhcp.{{ item.name }}.leasetime='12h' + uci set dhcp.{{ item.name }}.start='2' + uci set dhcp.{{ item.name }}.limit='1' + uci set dhcp.{{ item.name }}.netmask='255.255.255.252' + {% for option in item.dhcp_options %} + uci add_list dhcp.{{ item.name }}.dhcp_option_force="{{ option.option }},{{ option.value }}" + {% endfor %} + uci commit dhcp + /etc/init.d/dnsmasq restart + + loop: '{{ mgmt_firewall_interfaces.mgmt_firewall_lan }}' + +- name: Configure WAN interfaces + ansible.builtin.raw: | + section_id=$(uci add network interface) + uci set network.$section_id.proto='static' + uci set network.$section_id.area_type='wan' + uci set network.$section_id.peerdns='0' + uci set network.$section_id.device='{{ item.device }}' + uci set network.$section_id.metric='{{ item.metric }}' + uci set network.$section_id.ipaddr='{{ item.ipaddr }}' + uci set network.$section_id.netmask='{{ item.netmask }}' + uci set network.$section_id.gateway='{{ item.gateway }}' + uci set network.$section_id.name='{{ item.name }}' + {% for dns_server in item.dns %} + uci add_list network.$section_id.dns='{{ dns_server }}' + {% endfor %} + + uci add_list firewall.3.network="$section_id" + uci commit firewall + /etc/init.d/firewall restart + + uci commit network + /etc/init.d/network restart + loop: '{{ mgmt_firewall_interfaces.mgmt_firewall_wan.interfaces }}' + +- name: Apply Port-Forwards + ansible.builtin.raw: | + uci add firewall redirect + uci set firewall.@redirect[-1].src_dport='{{ item.src_dport }}' # External port + uci set firewall.@redirect[-1].dest_ip='{{ item.dest_ip }}' + uci set firewall.@redirect[-1].dest_port='{{ item.dest_port }}' + uci set firewall.@redirect[-1].src='{{ item.src }}' + uci set firewall.@redirect[-1].name='{{ item.name }}' + uci set firewall.@redirect[-1].target='DNAT' + uci set firewall.@redirect[-1].priority='{{ item.priority }}' # Order of rule + {% if item.dest is defined %} + uci set firewall.@redirect[-1].dest='{{ item.dest }}' # Internal Zone + {% endif %} + uci set firewall.@redirect[-1].enabled='1' + uci set firewall.@redirect[-1].reflection='{{ item.reflection }}' # Enable Nat Loopback + {% if item.src_ip is defined %} + {% for src_ip in item.src_ip %} + uci add_list firewall.@redirect[-1].src_ip="{{ src_ip }}" + {% endfor %} + {% endif %} + {% if item.src_dip is defined %} + uci set firewall.@redirect[-1].src_dip='{{ item.src_dip }}' # External IP, defaults to Any + {% endif %} + {% if item.proto is defined %} + {% for proto in item.proto %} + uci add_list firewall.@redirect[-1].proto="{{ proto }}" # Protocol setter (TCP, UDP) + {% endfor %} + {% endif %} + uci commit firewall + /etc/init.d/firewall restart + loop: '{{ mgmt_firewall_port_forwards }}' + +- name: Setup Static Routes + ansible.builtin.raw: | + uci set network.{{ item.network }}=route + uci set network.{{ item.network }}.table='254' + uci set network.{{ item.network }}.netmask='0.0.0.0' + uci set network.{{ item.network }}.target='0.0.0.0' + uci set network.{{ item.network }}.gateway='{{ item.gateway }}' + uci set network.{{ item.network }}.interface='wan' + loop: '{{ mgmt_firewall_static_routes }}' + when: mgmt_firewall_static_routes_enabled | default(false) + +- name: Setup static VLANs (VLAN 1 and 2) + ansible.builtin.raw: | + uci set network.@switch_vlan[0].ports='0t 4' + uci set network.@switch_vlan[1].ports='0t 5' + uci commit network + +- name: Setup dynamic VLANs + ansible.builtin.raw: | + uci add network switch_vlan + uci set network.@switch_vlan[-1].device='switch0' + uci set network.@switch_vlan[-1].vlan='{{ item.vlan }}' + uci set network.@switch_vlan[-1].vid='{{ item.vid }}' + uci set network.@switch_vlan[-1].ports='{{ item.ports }}' + uci commit network + loop: '{{ mgmt_firewall_vlans }}' + +- name: Restart Network + ansible.builtin.raw: | + /etc/init.d/network restart From e7a1e303be926eee3640a250696e1c553b2a26fd Mon Sep 17 00:00:00 2001 From: Ilja Rotar <77339620+iljarotar@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:53:01 +0200 Subject: [PATCH 2/4] Add config db to ZTP (#333) --- partition/roles/ztp/README.md | 21 +++++++++++++++++++++ partition/roles/ztp/defaults/main/main.yaml | 1 + partition/roles/ztp/files/config_db.json | 7 +++++++ partition/roles/ztp/files/reload.sh | 3 +++ partition/roles/ztp/tasks/main.yaml | 18 ++++++++++++++++++ partition/roles/ztp/templates/ztp.json.j2 | 21 +++++++++++++++++++++ 6 files changed, 71 insertions(+) create mode 100644 partition/roles/ztp/files/config_db.json create mode 100644 partition/roles/ztp/files/reload.sh create mode 100644 partition/roles/ztp/templates/ztp.json.j2 diff --git a/partition/roles/ztp/README.md b/partition/roles/ztp/README.md index 2de4b68a..f49ae6f1 100644 --- a/partition/roles/ztp/README.md +++ b/partition/roles/ztp/README.md @@ -9,7 +9,28 @@ Configures a server for providing zero-touch-provisioning scripts for switches. | ztp_nginx_image_name | yes | the docker image to use to serve ztp scripts. | | ztp_nginx_image_tag | yes | the tag of the docker image to use to serve ztp scripts. | | ztp_host_dir_path | | the path to serve ztp scripts from. | +| ztp_listen_address | | the address used to serve ztp requests | | ztp_port | | the port to serve ztp scripts on. | | ztp_authorized_keys | yes | the authorized keys that should be installed by ztp. | | ztp_admin_user | | the user for which the authorized keys will be provisioned. | | ztp_additional_files | | puts additional files into serve directory. | + +## Provisioning SONiC Switches via ztp.json + +On SONiC switches it is possible to describe the ZTP procedure in a file called `ztp.json`. +It contains all steps that should be performed during ZTP along with some additional options. +We use `ztp.json` to trigger a restart of the BGP service after the initial switch provisioning. +To use the `ztp.json` file, add a DHCP option with code 67 to the DHCP server that serves the file. +For example, add a section like the following to `/etc/dhcp/dhcpd.conf`: + +``` +option sonic_ztp code 67 = text; + +host leaf01 { + hardware ethernet aa:aa:aa:aa:aa:aa; + fixed-address 10.1.253.154; + option sonic_ztp "http://10.1.253.13:8080/ztp.json"; +} +``` + +For more information on the `ztp.json` format refer to the [documentation](https://github.com/sonic-net/SONiC/blob/master/doc/ztp/ztp.md). diff --git a/partition/roles/ztp/defaults/main/main.yaml b/partition/roles/ztp/defaults/main/main.yaml index 8cff09c2..01dfafc3 100644 --- a/partition/roles/ztp/defaults/main/main.yaml +++ b/partition/roles/ztp/defaults/main/main.yaml @@ -4,6 +4,7 @@ ztp_host_dir_path: /ztp ztp_authorized_keys: ztp_admin_user: admin +ztp_listen_address: "{{ ansible_host }}" ztp_port: 8080 ztp_additional_files: [] diff --git a/partition/roles/ztp/files/config_db.json b/partition/roles/ztp/files/config_db.json new file mode 100644 index 00000000..0d7ecddd --- /dev/null +++ b/partition/roles/ztp/files/config_db.json @@ -0,0 +1,7 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "docker_routing_config_mode": "split" + } + } +} \ No newline at end of file diff --git a/partition/roles/ztp/files/reload.sh b/partition/roles/ztp/files/reload.sh new file mode 100644 index 00000000..4712145e --- /dev/null +++ b/partition/roles/ztp/files/reload.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo systemctl restart bgp diff --git a/partition/roles/ztp/tasks/main.yaml b/partition/roles/ztp/tasks/main.yaml index 780c2956..1f17f311 100644 --- a/partition/roles/ztp/tasks/main.yaml +++ b/partition/roles/ztp/tasks/main.yaml @@ -23,6 +23,24 @@ dest: "{{ ztp_host_dir_path }}/config/ztp.sh" mode: 0644 +- name: copy config_db.json + copy: + src: "config_db.json" + dest: "{{ ztp_host_dir_path }}/config/config_db.json" + mode: 0644 + +- name: copy reload script + copy: + src: "reload.sh" + dest: "{{ ztp_host_dir_path }}/config/reload.sh" + mode: 0644 + +- name: render ztp.json + template: + src: "ztp.json.j2" + dest: "{{ ztp_host_dir_path }}/config/ztp.json" + mode: 0644 + - name: copy additional contents copy: dest: "{{ ztp_host_dir_path }}/config/{{ item.name }}" diff --git a/partition/roles/ztp/templates/ztp.json.j2 b/partition/roles/ztp/templates/ztp.json.j2 new file mode 100644 index 00000000..29da1d14 --- /dev/null +++ b/partition/roles/ztp/templates/ztp.json.j2 @@ -0,0 +1,21 @@ +{ + "ztp": { + "02-user": { + "plugin": { + "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/ztp.sh" + } + }, + "03-configdb-json": { + "url": { + "source": "http://{{ ztp_listen_address }}:{{ ztp_port }}/config_db.json" + }, + "clear-config": false + }, + "04-reload": { + "plugin": { + "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/reload.sh" + } + }, + "restart-ztp-no-config": false + } +} From ab93611d34196ebec081e9673d36d33564c61f5c Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 24 Oct 2024 09:12:14 +0200 Subject: [PATCH 3/4] Gardener v1.96. (#339) --- .../extension-provider-metal/controller-deployment.yaml | 7 +++---- .../templates/networking-cilium/controller-deployment.yaml | 2 +- .../os-metal-extension/controller-deployment.yaml | 6 +++--- .../templates/provider-gcp/controller-deployment.yaml | 7 +++---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml index 4010ed0b..1d56cb3f 100644 --- a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml @@ -1,11 +1,10 @@ --- -apiVersion: core.gardener.cloud/v1beta1 +apiVersion: core.gardener.cloud/v1 kind: ControllerDeployment metadata: name: provider-metal -type: helm -providerConfig: - chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/gardener-extension-provider-metal/' + gardener_extension_provider_metal_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" +helm: + rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/gardener-extension-provider-metal/' + gardener_extension_provider_metal_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}" values: image: repository: "{{ gardener_extension_provider_metal_image_name }}" diff --git a/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml b/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml index 0e832f65..e620be21 100644 --- a/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml @@ -15,4 +15,4 @@ providerConfig: imageVectorOverwrite: | images: {{ gardener_extension_networking_cilium_image_vector_overwrite | to_nice_yaml(indent=2) | indent(width=8, first=false) }} -{% endif %} \ No newline at end of file +{% endif %} diff --git a/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml b/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml index 9b4c8cd0..03f37294 100644 --- a/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml @@ -1,11 +1,11 @@ --- -apiVersion: core.gardener.cloud/v1beta1 +apiVersion: core.gardener.cloud/v1 kind: ControllerDeployment metadata: name: os-metal type: helm -providerConfig: - chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/os-metal-extension/' + gardener_os_controller_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" +helm: + rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/os-metal-extension/' + gardener_os_controller_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}" values: image: repository: "{{ gardener_os_controller_image_name }}" diff --git a/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml b/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml index fa237508..dc320558 100644 --- a/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml @@ -1,11 +1,10 @@ --- -apiVersion: core.gardener.cloud/v1beta1 +apiVersion: core.gardener.cloud/v1 kind: ControllerDeployment metadata: name: provider-gcp -type: helm -providerConfig: - chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-provider-gcp/' + gardener_extension_provider_gcp_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" +helm: + rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-provider-gcp/' + gardener_extension_provider_gcp_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}" values: image: tag: {{ gardener_extension_provider_gcp_image_tag }} From 91547de795701bcbb4c9f84c03e01b776cb681ce Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 25 Oct 2024 10:16:16 +0200 Subject: [PATCH 4/4] Provide configuration option to encrypt backup-restore-sidecar backups. (#340) --- .../auditing-meili/defaults/main/main.yaml | 1 + .../roles/auditing-meili/tasks/main.yaml | 1 + .../roles/headscale/defaults/main/db.yaml | 1 + control-plane/roles/headscale/tasks/main.yaml | 1 + .../roles/ipam-db/defaults/main/main.yaml | 1 + control-plane/roles/ipam-db/tasks/main.yaml | 2 +- .../masterdata-db/defaults/main/main.yaml | 1 + .../roles/masterdata-db/tasks/main.yaml | 1 + .../roles/meili-backup-restore/README.md | 50 ++++++------- .../defaults/main/main.yaml | 1 + .../roles/meili-backup-restore/tasks/main.yml | 1 + .../templates/meilisearch.yaml | 3 + .../roles/metal-db/defaults/main/main.yaml | 1 + control-plane/roles/metal-db/tasks/main.yaml | 1 + .../roles/postgres-backup-restore/README.md | 71 ++++++++++--------- .../defaults/main/main.yaml | 1 + .../postgres-backup-restore/tasks/main.yml | 1 + .../templates/postgres.yaml | 3 + .../roles/rethinkdb-backup-restore/README.md | 53 +++++++------- .../defaults/main/main.yaml | 1 + .../rethinkdb-backup-restore/tasks/main.yml | 1 + .../templates/rethinkdb.yaml | 3 + 22 files changed, 114 insertions(+), 86 deletions(-) diff --git a/control-plane/roles/auditing-meili/defaults/main/main.yaml b/control-plane/roles/auditing-meili/defaults/main/main.yaml index a1dd1713..2dab145f 100644 --- a/control-plane/roles/auditing-meili/defaults/main/main.yaml +++ b/control-plane/roles/auditing-meili/defaults/main/main.yaml @@ -17,6 +17,7 @@ auditing_meili_backup_restore_sidecar_backup_cron_schedule: "0 * * * *" auditing_meili_backup_restore_sidecar_log_level: debug auditing_meili_backup_restore_sidecar_object_prefix: "{{ auditing_meili_name }}-{{ metal_control_plane_stage_name }}" auditing_meili_backup_restore_sidecar_object_max_keep: +auditing_meili_backup_restore_sidecar_encryption_key: auditing_meili_backup_restore_sidecar_gcp_bucket_name: auditing_meili_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/auditing-meili/tasks/main.yaml b/control-plane/roles/auditing-meili/tasks/main.yaml index 1c4377a6..fe69d289 100644 --- a/control-plane/roles/auditing-meili/tasks/main.yaml +++ b/control-plane/roles/auditing-meili/tasks/main.yaml @@ -41,3 +41,4 @@ meilisearch_backup_restore_sidecar_gcp_serviceaccount_json: "{{ auditing_meili_backup_restore_sidecar_gcp_serviceaccount_json }}" meilisearch_resources: "{{ auditing_meili_resources }}" meilisearch_backup_restore_sidecar_object_max_keep: "{{ auditing_meili_backup_restore_sidecar_object_max_keep }}" + meilisearch_backup_restore_sidecar_encryption_key: "{{ auditing_meili_backup_restore_sidecar_encryption_key }}" diff --git a/control-plane/roles/headscale/defaults/main/db.yaml b/control-plane/roles/headscale/defaults/main/db.yaml index d08fb0e6..f96f2e09 100644 --- a/control-plane/roles/headscale/defaults/main/db.yaml +++ b/control-plane/roles/headscale/defaults/main/db.yaml @@ -12,6 +12,7 @@ headscale_db_backup_restore_sidecar_provider: local headscale_db_backup_restore_sidecar_backup_cron_schedule: "0 0 * * *" headscale_db_backup_restore_sidecar_log_level: debug headscale_db_backup_restore_sidecar_object_prefix: "{{ headscale_db_name }}" +headscale_db_backup_restore_sidecar_encryption_key: headscale_db_backup_restore_sidecar_gcp_bucket_name: headscale_db_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/headscale/tasks/main.yaml b/control-plane/roles/headscale/tasks/main.yaml index 73dec327..d395e70e 100644 --- a/control-plane/roles/headscale/tasks/main.yaml +++ b/control-plane/roles/headscale/tasks/main.yaml @@ -50,6 +50,7 @@ postgres_backup_restore_sidecar_gcp_backup_location: "{{ headscale_db_backup_restore_sidecar_gcp_backup_location }}" postgres_backup_restore_sidecar_gcp_project_id: "{{ headscale_db_backup_restore_sidecar_gcp_project_id }}" postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ headscale_db_backup_restore_sidecar_gcp_serviceaccount_json }}" + postgres_backup_restore_sidecar_encryption_key: "{{ headscale_db_backup_restore_sidecar_encryption_key }}" postgres_resources: "{{ headscale_db_resources }}" - name: Deploy headscale diff --git a/control-plane/roles/ipam-db/defaults/main/main.yaml b/control-plane/roles/ipam-db/defaults/main/main.yaml index a304e7b9..b5134a94 100644 --- a/control-plane/roles/ipam-db/defaults/main/main.yaml +++ b/control-plane/roles/ipam-db/defaults/main/main.yaml @@ -17,6 +17,7 @@ ipam_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *" ipam_db_backup_restore_sidecar_log_level: debug ipam_db_backup_restore_sidecar_object_prefix: "{{ ipam_db_name }}-{{ metal_control_plane_stage_name }}" ipam_db_backup_restore_sidecar_object_max_keep: +ipam_db_backup_restore_sidecar_encryption_key: ipam_db_backup_restore_sidecar_gcp_bucket_name: ipam_db_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/ipam-db/tasks/main.yaml b/control-plane/roles/ipam-db/tasks/main.yaml index ac77cf2c..23b068f6 100644 --- a/control-plane/roles/ipam-db/tasks/main.yaml +++ b/control-plane/roles/ipam-db/tasks/main.yaml @@ -42,4 +42,4 @@ postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ ipam_db_backup_restore_sidecar_gcp_serviceaccount_json }}" postgres_resources: "{{ ipam_db_resources }}" postgres_backup_restore_sidecar_object_max_keep: "{{ ipam_db_backup_restore_sidecar_object_max_keep }}" - + postgres_backup_restore_sidecar_encryption_key: "{{ ipam_db_backup_restore_sidecar_encryption_key }}" diff --git a/control-plane/roles/masterdata-db/defaults/main/main.yaml b/control-plane/roles/masterdata-db/defaults/main/main.yaml index 1e62ab0d..ccb43843 100644 --- a/control-plane/roles/masterdata-db/defaults/main/main.yaml +++ b/control-plane/roles/masterdata-db/defaults/main/main.yaml @@ -17,6 +17,7 @@ masterdata_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *" masterdata_db_backup_restore_sidecar_log_level: debug masterdata_db_backup_restore_sidecar_object_prefix: "{{ masterdata_db_name }}-{{ metal_control_plane_stage_name }}" masterdata_db_backup_restore_sidecar_object_max_keep: +masterdata_db_backup_restore_sidecar_encryption_key: masterdata_db_backup_restore_sidecar_gcp_bucket_name: masterdata_db_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/masterdata-db/tasks/main.yaml b/control-plane/roles/masterdata-db/tasks/main.yaml index 14def4b0..5326673b 100644 --- a/control-plane/roles/masterdata-db/tasks/main.yaml +++ b/control-plane/roles/masterdata-db/tasks/main.yaml @@ -42,3 +42,4 @@ postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ masterdata_db_backup_restore_sidecar_gcp_serviceaccount_json }}" postgres_resources: "{{ masterdata_db_resources }}" postgres_backup_restore_sidecar_object_max_keep: "{{ masterdata_db_backup_restore_sidecar_object_max_keep }}" + postgres_backup_restore_sidecar_encryption_key: "{{ masterdata_db_backup_restore_sidecar_encryption_key }}" diff --git a/control-plane/roles/meili-backup-restore/README.md b/control-plane/roles/meili-backup-restore/README.md index 7ee81d4e..965ab6a5 100644 --- a/control-plane/roles/meili-backup-restore/README.md +++ b/control-plane/roles/meili-backup-restore/README.md @@ -8,27 +8,29 @@ This role uses variables from [control-plane-defaults](/control-plane). So, make You can look up all the default values of this role [here](defaults/main/main.yaml). -| Name | Mandatory | Description | -| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------- | -| meilisearch_image_name | yes | Image version of the meilisearch | -| meilisearch_image_tag | yes | Image tag of the meilisearch | -| meilisearch_registry_auth_enabled | | Enables registry authentication | -| meilisearch_registry_auth | | The dockerconfigjson content used for registry authentication | -| meilisearch_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | -| meilisearch_name | | The name of the meilisearch instance | -| meilisearch_namespace | | The deployment's target namespace | -| meilisearch_storage_size | | The size of the PVC | -| meilisearch_storage_class | | The storage class of the PVC | -| meilisearch_api_key | | The api key for meilisearch | -| meilisearch_environment | | Sets the environment configuration for meilisearch | -| meilisearch_no_analytics | | Sets the no analytics configuration for meilisearch | -| meilisearch_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | -| meilisearch_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | -| meilisearch_backup_restore_sidecar_provider | | The backup provider | -| meilisearch_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | -| meilisearch_backup_restore_sidecar_log_level | | The log level of the sidecar | -| meilisearch_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | -| meilisearch_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | -| meilisearch_backup_restore_sidecar_gcp_project_id | | GCP project name | -| meilisearch_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | -| meilisearch_resources | | The kubernetes resources for the actual meilisearch container | +| Name | Mandatory | Description | +| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------- | +| meilisearch_image_name | yes | Image version of the meilisearch | +| meilisearch_image_tag | yes | Image tag of the meilisearch | +| meilisearch_registry_auth_enabled | | Enables registry authentication | +| meilisearch_registry_auth | | The dockerconfigjson content used for registry authentication | +| meilisearch_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | +| meilisearch_name | | The name of the meilisearch instance | +| meilisearch_namespace | | The deployment's target namespace | +| meilisearch_storage_size | | The size of the PVC | +| meilisearch_storage_class | | The storage class of the PVC | +| meilisearch_api_key | | The api key for meilisearch | +| meilisearch_environment | | Sets the environment configuration for meilisearch | +| meilisearch_no_analytics | | Sets the no analytics configuration for meilisearch | +| meilisearch_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | +| meilisearch_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | +| meilisearch_backup_restore_sidecar_provider | | The backup provider | +| meilisearch_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | +| meilisearch_backup_restore_sidecar_log_level | | The log level of the sidecar | +| meilisearch_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | +| meilisearch_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | +| meilisearch_backup_restore_sidecar_gcp_project_id | | GCP project name | +| meilisearch_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | +| meilisearch_resources | | The kubernetes resources for the actual meilisearch container | +| meilisearch_backup_restore_sidecar_object_max_keep | | The number of objects to keep at the cloud provider bucket | +| meilisearch_backup_restore_sidecar_encryption_key | | An optional encryption key to AES-encrypt the backups before uploading them to the backup provider (length == 32) | diff --git a/control-plane/roles/meili-backup-restore/defaults/main/main.yaml b/control-plane/roles/meili-backup-restore/defaults/main/main.yaml index e3600ec9..16fa8567 100644 --- a/control-plane/roles/meili-backup-restore/defaults/main/main.yaml +++ b/control-plane/roles/meili-backup-restore/defaults/main/main.yaml @@ -17,6 +17,7 @@ meilisearch_backup_restore_sidecar_backup_cron_schedule: "0 * * * *" meilisearch_backup_restore_sidecar_log_level: debug meilisearch_backup_restore_sidecar_object_prefix: "{{ meilisearch_name }}-{{ metal_control_plane_stage_name }}" meilisearch_backup_restore_sidecar_object_max_keep: +meilisearch_backup_restore_sidecar_encryption_key: meilisearch_backup_restore_sidecar_gcp_bucket_name: meilisearch_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/meili-backup-restore/tasks/main.yml b/control-plane/roles/meili-backup-restore/tasks/main.yml index 92a064e2..3db47b4a 100644 --- a/control-plane/roles/meili-backup-restore/tasks/main.yml +++ b/control-plane/roles/meili-backup-restore/tasks/main.yml @@ -11,6 +11,7 @@ - meilisearch_image_tag is defined - meilisearch_backup_restore_sidecar_image_name is defined - meilisearch_backup_restore_sidecar_image_tag is defined + - meilisearch_backup_restore_sidecar_encryption_key is none or meilisearch_backup_restore_sidecar_encryption_key | length == 32 - name: Deploy meilisearch (backup-restore) k8s: diff --git a/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml b/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml index d100275b..49067dc4 100644 --- a/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml +++ b/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml @@ -231,6 +231,9 @@ data: compression-method: targz {% if meilisearch_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ meilisearch_backup_restore_sidecar_object_max_keep }} +{% endif %} +{% if meilisearch_backup_restore_sidecar_encryption_key %} + encryption-key: {{ meilisearch_backup_restore_sidecar_encryption_key }} {% endif %} post-exec-cmds: - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files diff --git a/control-plane/roles/metal-db/defaults/main/main.yaml b/control-plane/roles/metal-db/defaults/main/main.yaml index 90219a82..b7ee9ef0 100644 --- a/control-plane/roles/metal-db/defaults/main/main.yaml +++ b/control-plane/roles/metal-db/defaults/main/main.yaml @@ -14,6 +14,7 @@ metal_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *" metal_db_backup_restore_sidecar_log_level: debug metal_db_backup_restore_sidecar_object_max_keep: +metal_db_backup_restore_sidecar_encryption_key: metal_db_backup_restore_sidecar_gcp_bucket_name: metal_db_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/metal-db/tasks/main.yaml b/control-plane/roles/metal-db/tasks/main.yaml index 3a121a41..5381ef1f 100644 --- a/control-plane/roles/metal-db/tasks/main.yaml +++ b/control-plane/roles/metal-db/tasks/main.yaml @@ -40,3 +40,4 @@ rethinkdb_ingress_dns: "{{ metal_db_ingress_dns }}" rethinkdb_resources: "{{ metal_db_resources }}" rethinkdb_backup_restore_sidecar_object_max_keep: "{{ metal_db_backup_restore_sidecar_object_max_keep }}" + rethinkdb_backup_restore_sidecar_encryption_key: "{{ metal_db_backup_restore_sidecar_encryption_key }}" diff --git a/control-plane/roles/postgres-backup-restore/README.md b/control-plane/roles/postgres-backup-restore/README.md index 6640b69e..4c6ab642 100644 --- a/control-plane/roles/postgres-backup-restore/README.md +++ b/control-plane/roles/postgres-backup-restore/README.md @@ -8,38 +8,39 @@ This role uses variables from [control-plane-defaults](/control-plane). So, make You can look up all the default values of this role [here](defaults/main/main.yaml). -| Name | Mandatory | Description | -| ------------------------------------------------------- | --------- | ------------------------------------------------------------------------ | -| postgres_image_name | yes | Image version of the postgres | -| postgres_image_tag | yes | Image tag of the postgres | -| postgres_registry_auth_enabled | | Enables registry authentication | -| postgres_registry_auth | | The dockerconfigjson content used for registry authentication | -| postgres_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | -| postgres_name | | The name of the postgres instance | -| postgres_namespace | | The deployment's target namespace | -| postgres_storage_size | | The size of the PVC | -| postgres_storage_class | | The storage class of the PVC | -| postgres_db | | The name of the database | -| postgres_user | | The user of the postgres database | -| postgres_password | | The password of the postgres database | -| postgres_max_connections | | The amount of max. connections possible, defaults to 100 | -| postgres_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | -| postgres_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | -| postgres_backup_restore_sidecar_provider | | The backup provider | -| postgres_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | -| postgres_backup_restore_sidecar_log_level | | The log level of the sidecar | -| postgres_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | -| postgres_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | -| postgres_backup_restore_sidecar_gcp_project_id | | GCP project name | -| postgres_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | -| postgres_expose_frontend | | Exposes the postgres over ingress (only use for dev environments) | -| postgres_ingress_dns | | The virtual host to reach the postgres frontend when exposed via ingress | -| postgres_resources | | The kubernetes resources for the actual postgres container | -| postgres_backup_restore_sidecar_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | -| postgres_shared_libraries_preload | | Allows setting shared libraries preload configuration | -| postgres_shared_buffers | | Allows setting shared buffer size | -| postgres_maintenance_work_mem | | Allows setting maintenance work memory | -| postgres_work_mem | | Allows setting work memory | -| postgres_effective_cache_size | | Allows setting effective cache size | -| postgres_backup_restore_sidecar_object_prefix | | The prefix to store the object in the cloud provider bucket | -| postgres_backup_restore_sidecar_object_max_keep | | The number of objects to keep at the cloud provider bucket | +| Name | Mandatory | Description | +| ------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------- | +| postgres_image_name | yes | Image version of the postgres | +| postgres_image_tag | yes | Image tag of the postgres | +| postgres_registry_auth_enabled | | Enables registry authentication | +| postgres_registry_auth | | The dockerconfigjson content used for registry authentication | +| postgres_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | +| postgres_name | | The name of the postgres instance | +| postgres_namespace | | The deployment's target namespace | +| postgres_storage_size | | The size of the PVC | +| postgres_storage_class | | The storage class of the PVC | +| postgres_db | | The name of the database | +| postgres_user | | The user of the postgres database | +| postgres_password | | The password of the postgres database | +| postgres_max_connections | | The amount of max. connections possible, defaults to 100 | +| postgres_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | +| postgres_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | +| postgres_backup_restore_sidecar_provider | | The backup provider | +| postgres_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | +| postgres_backup_restore_sidecar_log_level | | The log level of the sidecar | +| postgres_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | +| postgres_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | +| postgres_backup_restore_sidecar_gcp_project_id | | GCP project name | +| postgres_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | +| postgres_expose_frontend | | Exposes the postgres over ingress (only use for dev environments) | +| postgres_ingress_dns | | The virtual host to reach the postgres frontend when exposed via ingress | +| postgres_resources | | The kubernetes resources for the actual postgres container | +| postgres_backup_restore_sidecar_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | +| postgres_shared_libraries_preload | | Allows setting shared libraries preload configuration | +| postgres_shared_buffers | | Allows setting shared buffer size | +| postgres_maintenance_work_mem | | Allows setting maintenance work memory | +| postgres_work_mem | | Allows setting work memory | +| postgres_effective_cache_size | | Allows setting effective cache size | +| postgres_backup_restore_sidecar_object_prefix | | The prefix to store the object in the cloud provider bucket | +| postgres_backup_restore_sidecar_object_max_keep | | The number of objects to keep at the cloud provider bucket | +| postgres_backup_restore_sidecar_encryption_key | | An optional encryption key to AES-encrypt the backups before uploading them to the backup provider (length == 32) | diff --git a/control-plane/roles/postgres-backup-restore/defaults/main/main.yaml b/control-plane/roles/postgres-backup-restore/defaults/main/main.yaml index df0fc6a9..ac7b9084 100644 --- a/control-plane/roles/postgres-backup-restore/defaults/main/main.yaml +++ b/control-plane/roles/postgres-backup-restore/defaults/main/main.yaml @@ -23,6 +23,7 @@ postgres_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *" postgres_backup_restore_sidecar_log_level: debug postgres_backup_restore_sidecar_object_prefix: "{{ postgres_name }}-{{ metal_control_plane_stage_name }}" postgres_backup_restore_sidecar_object_max_keep: +postgres_backup_restore_sidecar_encryption_key: postgres_backup_restore_sidecar_gcp_bucket_name: postgres_backup_restore_sidecar_gcp_backup_location: diff --git a/control-plane/roles/postgres-backup-restore/tasks/main.yml b/control-plane/roles/postgres-backup-restore/tasks/main.yml index 8cea859f..3cb86531 100644 --- a/control-plane/roles/postgres-backup-restore/tasks/main.yml +++ b/control-plane/roles/postgres-backup-restore/tasks/main.yml @@ -11,6 +11,7 @@ - postgres_image_tag is defined - postgres_backup_restore_sidecar_image_name is defined - postgres_backup_restore_sidecar_image_tag is defined + - postgres_backup_restore_sidecar_encryption_key is none or postgres_backup_restore_sidecar_encryption_key | length == 32 - name: Deploy postgres (backup-restore) k8s: diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 566a8325..a6b1e464 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -240,6 +240,9 @@ data: compression-method: targz {% if postgres_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ postgres_backup_restore_sidecar_object_max_keep }} +{% endif %} +{% if postgres_backup_restore_sidecar_encryption_key %} + encryption-key: {{ postgres_backup_restore_sidecar_encryption_key }} {% endif %} post-exec-cmds: - docker-entrypoint.sh postgres {% if postgres_shared_libraries_preload %} -c shared_preload_libraries={{ postgres_shared_libraries_preload | join(',') }}{% endif %}{% if postgres_maintenance_work_mem %} -c maintenance_work_mem={{ postgres_maintenance_work_mem }}{% endif %}{% if postgres_shared_buffers %} -c shared_buffers={{ postgres_shared_buffers }}{% endif %}{% if postgres_effective_cache_size %} -c effective_cache_size={{ postgres_effective_cache_size }}{% endif %}{% if postgres_work_mem %} -c work_mem={{ postgres_work_mem }}{% endif %} -c max_connections={{ postgres_max_connections }} diff --git a/control-plane/roles/rethinkdb-backup-restore/README.md b/control-plane/roles/rethinkdb-backup-restore/README.md index 2147d214..5fef449b 100644 --- a/control-plane/roles/rethinkdb-backup-restore/README.md +++ b/control-plane/roles/rethinkdb-backup-restore/README.md @@ -8,29 +8,30 @@ This role uses variables from [control-plane-defaults](/control-plane). So, make You can look up all the default values of this role [here](defaults/main/main.yaml). -| Name | Mandatory | Description | -| -------------------------------------------------------- | --------- | ------------------------------------------------------------------------- | -| rethinkdb_image_name | yes | Image version of the rethinkdb | -| rethinkdb_image_tag | yes | Image tag of the rethinkdb | -| rethinkdb_registry_auth_enabled | | Enables registry authentication | -| rethinkdb_registry_auth | | The dockerconfigjson content used for registry authentication | -| rethinkdb_image_pull_policy | yes | Image pull policy (defaults to IfNotPresent) | -| rethinkdb_name | | The name of the rethinkdb instance | -| rethinkdb_namespace | | The deployment's target namespace | -| rethinkdb_storage_size | | The size of the PVC | -| rethinkdb_storage_class | | The storage class of the PVC | -| rethinkdb_password | | The password of the rethinkdb | -| rethinkdb_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | -| rethinkdb_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | -| rethinkdb_backup_restore_sidecar_provider | | The backup provider | -| rethinkdb_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | -| rethinkdb_backup_restore_sidecar_log_level | | The log level of the sidecar | -| rethinkdb_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | -| rethinkdb_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | -| rethinkdb_backup_restore_sidecar_gcp_project_id | | GCP project name | -| rethinkdb_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | -| rethinkdb_expose_frontend | | Exposes the rethinkdb over ingress (only use for dev environments) | -| rethinkdb_ingress_dns | | The virtual host to reach the rethinkdb frontend when exposed via ingress | -| rethinkdb_resources | | The kubernetes resources for the actual rethinkdb container | -| rethinkdb_backup_restore_sidecar_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | -| rethinkdb_backup_restore_sidecar_object_max_keep | | The number of objects to keep at the cloud provider bucket | +| Name | Mandatory | Description | +| -------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------- | +| rethinkdb_image_name | yes | Image version of the rethinkdb | +| rethinkdb_image_tag | yes | Image tag of the rethinkdb | +| rethinkdb_registry_auth_enabled | | Enables registry authentication | +| rethinkdb_registry_auth | | The dockerconfigjson content used for registry authentication | +| rethinkdb_image_pull_policy | yes | Image pull policy (defaults to IfNotPresent) | +| rethinkdb_name | | The name of the rethinkdb instance | +| rethinkdb_namespace | | The deployment's target namespace | +| rethinkdb_storage_size | | The size of the PVC | +| rethinkdb_storage_class | | The storage class of the PVC | +| rethinkdb_password | | The password of the rethinkdb | +| rethinkdb_backup_restore_sidecar_image_name | yes | Image version of the backup-restore-sidecar | +| rethinkdb_backup_restore_sidecar_image_tag | yes | Image tag of the backup-restore-sidecar | +| rethinkdb_backup_restore_sidecar_provider | | The backup provider | +| rethinkdb_backup_restore_sidecar_backup_cron_schedule | | The backup cron schedule | +| rethinkdb_backup_restore_sidecar_log_level | | The log level of the sidecar | +| rethinkdb_backup_restore_sidecar_gcp_bucket_name | | Bucket name of the GCP bucket | +| rethinkdb_backup_restore_sidecar_gcp_backup_location | | Location of the GCP bucket | +| rethinkdb_backup_restore_sidecar_gcp_project_id | | GCP project name | +| rethinkdb_backup_restore_sidecar_gcp_serviceaccount_json | | GCP Serviceaccount JSON string (service account requires bucket access) | +| rethinkdb_expose_frontend | | Exposes the rethinkdb over ingress (only use for dev environments) | +| rethinkdb_ingress_dns | | The virtual host to reach the rethinkdb frontend when exposed via ingress | +| rethinkdb_resources | | The kubernetes resources for the actual rethinkdb container | +| rethinkdb_backup_restore_sidecar_image_pull_policy | | Image pull policy (defaults to IfNotPresent) | +| rethinkdb_backup_restore_sidecar_object_max_keep | | The number of objects to keep at the cloud provider bucket | +| rethinkdb_backup_restore_sidecar_encryption_key | | An optional encryption key to AES-encrypt the backups before uploading them to the backup provider (length == 32) | diff --git a/control-plane/roles/rethinkdb-backup-restore/defaults/main/main.yaml b/control-plane/roles/rethinkdb-backup-restore/defaults/main/main.yaml index 6a30a1d5..852d536b 100644 --- a/control-plane/roles/rethinkdb-backup-restore/defaults/main/main.yaml +++ b/control-plane/roles/rethinkdb-backup-restore/defaults/main/main.yaml @@ -19,6 +19,7 @@ rethinkdb_backup_restore_sidecar_gcp_project_id: rethinkdb_backup_restore_sidecar_gcp_serviceaccount_json: rethinkdb_backup_restore_sidecar_object_max_keep: +rethinkdb_backup_restore_sidecar_encryption_key: rethinkdb_expose_frontend: no rethinkdb_ingress_dns: rethinkdb.{{ metal_control_plane_ingress_dns }} diff --git a/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml b/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml index ad7bf3f7..dff0c699 100644 --- a/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml +++ b/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml @@ -11,6 +11,7 @@ - rethinkdb_image_tag is defined - rethinkdb_backup_restore_sidecar_image_name is defined - rethinkdb_backup_restore_sidecar_image_tag is defined + - rethinkdb_backup_restore_sidecar_encryption_key is none or rethinkdb_backup_restore_sidecar_encryption_key | length == 32 - name: Check mandatory variables for this role are set assert: diff --git a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml index 035a4289..ab0c7fa1 100644 --- a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml +++ b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml @@ -191,6 +191,9 @@ data: {% if rethinkdb_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ rethinkdb_backup_restore_sidecar_object_max_keep }} {% endif %} +{% if rethinkdb_backup_restore_sidecar_encryption_key %} + encryption-key: {{ rethinkdb_backup_restore_sidecar_encryption_key }} +{% endif %} --- apiVersion: v1 kind: Secret