setup a cryptroot-installation including necessary steps for unlockin…

…g via ssh after reboot

ansible/roles/provision-hetzner/defaults/main.yml

@@ -4,14 +4,19 @@ hetzner_webservice_password:
 hetzner_hostname:               # needed for Hetzner server computer name
 hetzner_ip:                     # needed for Hetzner Web UI and Ansible
 hetzner_autosetup_file: templates/autosetup
+hetzner_postinst_file: templates/post-install
 hetzner_disk1: sda
 hetzner_disk2: sdb
 hetzner_raid_level: 0
 hetzner_vg_name: vg0
+
+hetzner_crypt_bootstrap_password: InitialCryptPW
+
 hetzner_image: "/root/.oldroot/nfs/install/../images/CentOS-90-stream-amd64-base.tar.gz"
 hetzner_image_ignore_errors: false
 hetzner_size_of_libvirt_images: all
 
+
 robot_base: https://robot-ws.your-server.de/
 needs_reprovision: false
 already_in_rescue: false

ansible/roles/provision-hetzner/tasks/provision-server.yml

@@ -142,21 +142,40 @@
     mode: 0644
   delegate_to: "{{ hetzner_ip }}"
 
+- name: Copy postinstall file for cryptroot
+  template:
+    src: "{{ hetzner_postinst_file }}"
+    dest: /root/post-install.ansible
+    owner: root
+    group: root
+    mode: 0755
+  delegate_to: "{{ hetzner_ip }}"
+  when: hetzner_crypt_password is defined
+
+
 - name: Run installimage
-  command: "/root/.oldroot/nfs/install/installimage -a -c /root/autosetup.ansible"
+  command: "/root/.oldroot/nfs/install/installimage -a -c /root/autosetup.ansible {{postinst_string}}"
   environment:
     TERM: "vt100"
   register: result
   changed_when: true
   failed_when: false
   delegate_to: "{{ hetzner_ip }}"
+  vars:
+    postinst_string: "{% if hetzner_crypt_password is defined %}-x /root/post-install.ansible{% endif %}"
 
 - name: Print installimage output with -v
   debug:
     var: result.stdout_lines
     verbosity: 1
   delegate_to: localhost
 
+- name: Print installimage stderroutput with -v
+  debug:
+    var: result.stderr_lines
+    verbosity: 1
+  delegate_to: localhost
+
 - name: Check stderr from installimage
   debug:
     msg: "Something want wrong at installimage: {{ result.stderr_lines | join('\n') }}"
@@ -191,6 +210,97 @@
   changed_when: '"updated" in output.stdout'
   delegate_to: localhost
 
+- name: Crypt-Unlocking twice
+  when: hetzner_crypt_password is defined
+  block:
+  ##
+  ## We have to unlock twice, because of auto-relabeling there are two boots
+  ##
+
+  # wait for first reboot
+  - name: Wait 600 seconds for port crypt-shell to become open
+    wait_for:
+      port: "{{ hetzner_crypt_network_ssh_port }}"
+      host: '{{ hetzner_ip }}'
+      delay: 10
+      timeout: 600
+    connection: local
+
+  # Show console before auth
+  - name: Read the console
+    ansible.builtin.raw: |
+      console_peek
+    timeout: 5
+    register: cryptroot_peek_reg
+    delegate_to: "{{ hetzner_ip }}"
+    ignore_errors: true
+  - debug: var=cryptroot_peek_reg
+
+  # this needs to be done manually, because the "console_auth" tool
+  # wants interactive input and we need to fake that
+  - name: Unlock cryptroot
+    ansible.builtin.shell: "echo {{ hetzner_crypt_bootstrap_password }} | ssh  -o StrictHostKeyChecking=no -l root -t {{ hetzner_ip }} 'console_auth'"
+    timeout: 5
+    register: cryptroot_unlock_reg
+    ignore_errors: true
+    delegate_to: "localhost"
+
+  # Show console after auth
+  - name: Pause a bit for the hardware reset to kick in
+    pause: seconds=2
+  - name: Read the console
+    ansible.builtin.raw: |
+      console_peek
+    timeout: 5
+    register: cryptroot_peek_reg
+    delegate_to: "{{ hetzner_ip }}"
+    ignore_errors: true
+  - debug: var=cryptroot_peek_reg
+
+  - name: Pause a bit for the hardware reset to kick in
+    pause: seconds=15
+
+  # wait for second reboot
+  - name: Wait 600 seconds for port crypt-shell to become open
+    wait_for:
+      port: "{{ hetzner_crypt_network_ssh_port }}"
+      host: '{{ hetzner_ip }}'
+      delay: 10
+      timeout: 600
+    connection: local
+
+  # Show console before auth
+  - name: Read the console
+    ansible.builtin.raw: |
+      console_peek
+    timeout: 5
+    register: cryptroot_peek_reg
+    delegate_to: "{{ hetzner_ip }}"
+    ignore_errors: true
+  - debug: var=cryptroot_peek_reg
+
+  # this needs to be done manually, because the "console_auth" tool
+  # wants interactive input and we need to fake that
+  - name: Unlock cryptroot
+    ansible.builtin.shell: "echo {{ hetzner_crypt_bootstrap_password }} | ssh  -o StrictHostKeyChecking=no -l root -t {{ hetzner_ip }} 'console_auth'"
+    timeout: 5
+    register: cryptroot_unlock_reg
+    ignore_errors: true
+    delegate_to: "localhost"
+
+  # Show console after auth
+  - name: Pause a bit for the hardware reset to kick in
+    pause: seconds=2
+  - name: Read the console
+    ansible.builtin.raw: |
+      console_peek
+    timeout: 5
+    register: cryptroot_peek_reg
+    delegate_to: "{{ hetzner_ip }}"
+    ignore_errors: true
+  - debug: var=cryptroot_peek_reg
+
+
 - name: Wait 600 seconds for port 22 to become open
   wait_for:
     port: 22
@@ -203,3 +313,46 @@
   ansible.builtin.gather_facts:
   register: host_facts
   delegate_to: "{{ hetzner_ip }}"
+
+- name: Change luks passphrase
+  when: hetzner_crypt_password is defined
+  delegate_to: "{{ hetzner_ip }}"
+  block:
+    - name: Set fact for the LVM PVs of vg0
+      set_fact:
+        luks_pv: "{{ ansible_facts.lvm.pvs | dict2items | selectattr('value.vg', 'equalto', hetzner_vg_name) | map(attribute='key') | list }}"
+      no_log: true
+
+    - name: Find parent of cryptdevice
+      ansible.builtin.shell:
+        cmd: "lsblk -l -o path,pkname | grep {{ luks_pv[0] }} | cut -d ' ' -f 2"
+        executable: /bin/bash
+      when: luks_pv | length > 0
+      register: luks_cryptdev_reg
+      ignore_errors: true
+
+    - set_fact:
+        luks_cryptdev: "{{luks_cryptdev_reg.stdout_lines[0]}}"
+
+    - name: Ensure the LUKS device variable is set
+      fail:
+        msg: "LUKS device could not be determined."
+      when: luks_pv | length == 0
+
+    - name: Add new passphrase to LUKS device
+      ansible.builtin.shell:
+        cmd: "echo -n '{{ hetzner_crypt_password }}' | cryptsetup luksAddKey /dev/{{ luks_cryptdev }} --key-file <(echo -n '{{ hetzner_crypt_bootstrap_password }}')"
+        executable: /bin/bash
+      no_log: true
+      when: luks_pv | length > 0
+      register: lukskey_add_reg
+      ignore_errors: true
+
+    - name: Remove old passphrase from LUKS device
+      ansible.builtin.shell:
+        cmd: "echo -n '{{ hetzner_crypt_bootstrap_password }}' | cryptsetup luksRemoveKey /dev/{{ luks_cryptdev }} --key-file <(echo -n '{{ hetzner_crypt_bootstrap_password }}')"
+        executable: /bin/bash
+      no_log: true
+      when: luks_pv | length > 0
+      register: lukskey_del_reg
+      ignore_errors: true

ansible/roles/provision-hetzner/templates/autosetup

@@ -5,12 +5,22 @@ SWRAIDLEVEL {{ hetzner_raid_level }}
 BOOTLOADER grub
 HOSTNAME {{ hetzner_hostname }}
 PART /boot ext3 1024M
-PART lvm {{ hetzner_vg_name }} all
+
+PART lvm {{ hetzner_vg_name }} all {% if hetzner_crypt_password is defined %} crypt{% endif %}
 
 LV {{ hetzner_vg_name }} root / xfs 50G
 LV {{ hetzner_vg_name }} swap swap swap 8G
 LV {{ hetzner_vg_name }} home   /home   xfs      10G
 LV {{ hetzner_vg_name }} var    /var xfs 50G
 LV {{ hetzner_vg_name }} libvirt /var/lib/libvirt/images xfs {{ hetzner_size_of_libvirt_images }}
 
+{#
+  we are setting an dummy password here, because this one is unsafe
+  This file remains on the server and could have been read by the
+  installimage, we're setting this one here and change it afterwards
+#}
+{% if hetzner_crypt_password is defined %}
+CRYPTPASSWORD {{ hetzner_crypt_bootstrap_password }}
+{% endif %}
+
 IMAGE {{ hetzner_image }}

ansible/roles/provision-hetzner/templates/post-install

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+NETWORK_STRING='ip={{ public_ip }}::{{ hetzner_crypt_network_gatewayv4 }}:255.255.255.255:{{ hetzner_hostname }}:{{ hetzner_crypt_network_interface }}:none rd.route={{ hetzner_crypt_network_gatewayv4 }}\/32:{{ hetzner_crypt_network_gatewayv4 }}:{{ hetzner_crypt_network_interface }}'
+
+dnf copr enable uriesk/dracut-crypt-ssh -y
+dnf install -y epel-release
+dnf install -y dracut-crypt-ssh
+
+echo "generating hostkeys"
+ssh-keygen -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key
+ssh-keygen -t ecdsa -N '' -f /etc/ssh/ssh_host_ecdsa_key
+ssh-keygen -t ed25519 -N '' -f /etc/ssh/ssh_host_ed25519_key
+
+echo adjusting kernel cmdline for early network
+sed -i '/^GRUB_CMDLINE_LINUX="/ {s/\<quiet\>//g;s/"$/ rd.neednet=1 '"$NETWORK_STRING"'"/;}' /etc/default/grub
+
+echo rebuilding grub.cfg
+grub2-mkconfig --output /boot/grub2/grub.cfg
+
+echo creating dracut/crypt-ssh.conf
+cat > /etc/dracut.conf.d/crypt-ssh.conf << EOT
+# NOTE: The defaults in this file MUST be carefully read and understood before using this module!
+# The defaults may NOT be appropriate for your site.  Carefully review and understand your threat model
+#  (ie, what attack scenarios you are protecting yourself against) and adjust accordingly!
+#
+
+# The port to run the ssh daemon on
+# Default: 222
+dropbear_port="{{ hetzner_crypt_network_ssh_port }}"
+
+# Where to get the RSA and/or ECDSA keys for dropbear, options are:
+#  GENERATE: generate a new one for each initrd run, the public key will be printed during the dracut build process
+#            and on boot
+#  SYSTEM:   use (convert) the host key from the host system's SSH daemon. This will make the initrd ssh indistinguishable
+#            from the running system - this may be a security risk, depending on your threat model, but simplifies
+#            your client-side ssh configuration
+#  /path/to/openssh_key:   an absolute path to a host key, in OpenSSH format as generated by ssh-keygen.
+#            A public key with '.pub' ending must be present too.
+#
+# It is recommend that you use the system one, or supply your own.  If using the system key, be aware that an attacker
+# that can access your initrd could use the host key to impersonate the running system.  This could allow them to attempt
+# an MITM attack.
+#
+# Default: GENERATE
+# dropbear_rsa_key="GENERATE"
+# dropbear_ecdsa_key="GENERATE"
+# dropbear_ed25519_key="GENERATE"
+#dropbear_rsa_key="SYSTEM"
+#dropbear_ecdsa_key="SYSTEM"
+#dropbear_ed25519_key="SYSTEM"
+
+# Location of the list of authorized public keys that can log into the initrd ssh daemon
+# Defaults to the authorized_keys list for root.  It may be advantageous to use a different authorized_keys list
+#  so that users/machines that can unlock the machine are not necessarily given full root access after boot.
+#  Note that root access to the initrd does give an attacker means to provide themselves with root access after boot,
+#  especially if they hold the encryption keys to the root drive - choose carefully!
+#
+# Default: /root/.ssh/authorized_keys
+dropbear_acl="/root/.ssh/authorized_keys"
+
+# Users wishing to unlock LUKS volumes remotely using the 'unlock' helper will need cryptsetup available in the initramfs.
+# Uncomment the below line to make sure that the application is available when needed.
+#
+install_items+=" /sbin/cryptsetup "
+EOT
+
+echo rebuilding initramfs
+dracut -v --force --kver="$(ls /lib/modules)"

cluster-example.yml

@@ -5,6 +5,12 @@ hetzner_webservice_password: # see docs
 hetzner_hostname: "changeme"
 hetzner_ip: "changeme"
 
+## the following block needs to be set for installation with cryptroot
+#hetzner_crypt_password:
+#hetzner_crypt_network_gatewayv4:
+#hetzner_crypt_network_interface: enp7s0
+#hetzner_crypt_network_ssh_port: 22
+
 cluster_name: ocp4
 public_domain: example.com