WIP - debug fapolicyd restart failures

tasks/main.yml

@@ -76,33 +76,6 @@
     - __fapolicyd_configcheck_supported | bool
     - __fapolicy_conf is changed
 
-- name: Start fapolicyd service
-  service:
-    name: "{{ __fapolicyd_services }}"
-    state: started
-    enabled: true
-  when: fapolicyd_setup_enable_service | bool
-  ignore_errors: true
-  register: __fapolicyd_start
-
-- name: Restart fapolicyd service
-  service:
-    name: "{{ __fapolicyd_services }}"
-    state: restarted
-    enabled: true
-  when:
-    - fapolicyd_setup_enable_service | bool
-    - __fapolicy_conf is changed
-  ignore_errors: true
-  register: __fapolicyd_restart
-
-- name: Check fapolicyd logs
-  command: journalctl -n5 -u {{ __fapolicyd_services | quote }}
-  register: __fapolicyd_results
-  changed_when: false
-  when: __fapolicyd_start is failed or __fapolicyd_restart is failed
-  failed_when: __fapolicyd_start is failed or __fapolicyd_restart is failed
-
 - name: Trustdb cleanup
   command: fapolicyd-cli --file delete /
   changed_when: true
@@ -115,51 +88,67 @@
   when: item | length > 0
   changed_when: true
 
-# The problem is that there is a race condition between calling `systemctl
-# restart fapolicyd`` and when fapolicyd will actually enforce the policy - so
-# we have to look for the right string in the fapolicyd logs.  Also - I don't
-# think we can move this into a script, because that script might be excluded by
-# policy!
+# The problem is that there is a race condition between calling
+# `systemctl restart fapolicyd` and when fapolicyd will actually
+# enforce the policy - so we have to look for the right string in the fapolicyd
+# logs.  Also - I don't think we can move this into a script, because that
+# script might be excluded by policy!
 # NOTE: I tried using `fapolicyd-cli --update` as recommended by the
 # documentation but it does not seem to work in all cases e.g. on RHEL 8.8 if
 # you are deleting entries but not adding entries, it seems to do nothing - the
 # only reliable way to update the trustdb is to restart the daemon and check for
 # "fapolicyd[...]: Starting to listen for events" in the journald output
-- name: Update fapolicyd db
+- name: Update fapolicyd db, restart, check for errors
   when: fapolicyd_setup_enable_service | bool
-  shell:
-    cmd: |
-      set -euo pipefail
-      # get current journal cursor
-      cursor=""
-      while [ -z "$cursor" ]; do
-        sleep 1
-        cursor="$(journalctl -u fapolicyd -n 0 --show-cursor |
-          awk '/^-- cursor:/ {print $3}')" || :
-      done
-      systemctl restart fapolicyd
-      search_str='fapolicyd[^:\ ]*:\ Starting to listen for events$'
-      # wait until we see the search_str - wait up to 30 seconds
-      waittime=30  # seconds
-      endtime="$(expr "$(date +%s)" + "$waittime")"
-      set +o pipefail  # the read will always return a failure code at EOF
-      journalctl -u fapolicyd --no-tail -f --after-cursor "$cursor" | \
-      while read -r line; do
-        if [[ "$line" =~ $search_str ]]; then
-          echo INFO: trustdb is updated
-          exit 0
-        fi
-      done & pid=$!
-      while ps -p "$pid"; do
-        if [ "$(date +%s)" -gt "$endtime" ]; then
-          echo ERROR: failed to update the trustdb
-          exit 1
-        fi
-        sleep 1
-      done
-      echo INFO: trustdb is updated
-      exit 0  # success
-  changed_when: true
+  block:
+    - name: Update fapolicyd db, restart fapolicyd, wait for readiness
+      shell:
+        cmd: |
+          set -euxo pipefail
+          # get current journal cursor, if any
+          cursor="$(journalctl -u fapolicyd -n 0 --show-cursor |
+            awk '/^-- cursor:/ {print $3}')" || :
+          systemctl restart fapolicyd
+          search_str='fapolicyd[^:\ ]*:\ Starting to listen for events$'
+          # wait until we see the search_str
+          waittime=60  # seconds
+          endtime="$(expr "$(date +%s)" + "$waittime")"
+          set +o pipefail  # the read will always return a failure code at EOF
+          journalctl_cmd=(journalctl -u fapolicyd -f --no-tail)
+          if [ -n "$cursor" ]; then
+            journalctl_cmd+=(--after-cursor "$cursor")
+          fi
+          "${journalctl_cmd[@]}" | while read -r line; do
+            if [[ "$line" =~ $search_str ]]; then
+              echo INFO: trustdb is updated
+              exit 0
+            fi
+          done & pid=$!
+          while ps -p "$pid" > /dev/null 2>&1; do
+            if [ "$(date +%s)" -gt "$endtime" ]; then
+              echo ERROR: failed to update the trustdb
+              exit 1
+            fi
+            sleep 1
+          done
+          echo INFO: trustdb is updated, server is ready
+          exit 0  # success
+      changed_when: true
+
+  rescue:
+    - name: Check system status
+      shell: |
+        set -uxo pipefail
+        exec 1>&2
+        journalctl -n10 -u {{ __fapolicyd_services | quote }}
+        systemctl status --full {{ __fapolicyd_services | quote }}
+        systemctl show --all {{ __fapolicyd_services | quote }}
+        journalctl --header
+      changed_when: false
+
+    # - name: Fail
+    #   fail:
+    #     msg: "{{ ansible_failed_result }}"
 
 - name: Making sure fapolicyd does not run if it was set so
   service:

vars/main.yml

@@ -9,6 +9,7 @@
 __fapolicyd_services: fapolicyd.service
 __fapolicyd_dir: /etc/fapolicyd
 __fapolicyd_conf: fapolicyd.conf
+__fapolicyd_state_file: /var/run/fapolicyd.state
 
 __fapolicyd_packages: [fapolicyd]
 __fapolicyd_selinux_packages: [fapolicyd-selinux]