Commits to investigate CI

Signed-off-by: Utkarsh Bhatt <[email protected]>

.github/workflows/tests.yml

@@ -97,6 +97,8 @@ jobs:
         grep -q ceph-version $meta
 
         # Verify health and auto crush rule
+        sleep 60
+        sudo microceph.ceph status
         sudo microceph.ceph health | grep -q "OSD count 0 < osd_pool_default_size 3"
         sudo microceph.ceph osd crush rule ls | grep -F microceph_auto_osd
 
@@ -215,6 +217,11 @@ jobs:
         output=$(sudo microceph log get-level)
         if [[ "$output" != "3" ]] ; then echo "incorrect log level: $output"; exit 1; fi 
 
+    - name: Upload artifacts on failure
+      if: failure()
+      run: |
+        sudo snap logs microceph -n 1000
+
   multi-node-tests:
     name: Multi node testing
     runs-on: ubuntu-22.04
@@ -286,6 +293,17 @@ jobs:
       run: |
         set -uex
         lxc exec node-wrk0 -- sh -c "microceph.ceph osd crush rule ls" | grep -F microceph_auto_host
+        # Wait for storage to settle
+        for i in $(seq 1 10); do
+            res=$(lxc exec node-wrk0 -- sh -c "microceph.ceph osd pool ls detail" | grep -c "crush_rule 2" || true)
+            if [[ $res -eq 1 ]] ; then
+                echo "Crush rules reconfigured"
+                break
+            else
+                echo -n '.'
+                sleep 5
+            fi
+        done
         lxc exec node-wrk0 -- sh -c "microceph.ceph osd pool ls detail" | grep -F "crush_rule 2"
 
     - name: Add another OSD
@@ -334,6 +352,11 @@ jobs:
     - name: Test client configurations
       run: ~/actionutils.sh check_client_configs
 
+    - name: Upload artifacts on failure
+      if: failure()
+      run: |
+        sudo snap logs microceph -n 1000
+
   multi-node-tests-with-custom-microceph-ip:
     name: Multi node testing with custom microceph IP
     runs-on: ubuntu-22.04
@@ -434,6 +457,11 @@ jobs:
         ~/actionutils.sh wait_for_osds 3
         sudo microceph.ceph -s
 
+    - name: Upload artifacts on failure
+      if: failure()
+      run: |
+        sudo snap logs microceph -n 1000
+
   wal-db-tests:
     name: Test WAL/DB device usage
     runs-on: ubuntu-22.04