MDBF-814 for install/upgrade save logs on failure

This applies to Columnstore logs, data

core files also saved.

Also saves the systemd journal of applicable services.

Being an trap ERR it will execute only if an err occurs.

master-libvirt/master.cfg

@@ -185,29 +185,47 @@ def getMajorVersionStep():
     )
 
 
+def getSaveArtifactsStep():
+    return steps.DirectoryUpload(
+        name="save mariadb log files",
+        doStepIf=hasFailed,
+        workersrc="/home/buildbot/logs/",
+        masterdest=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/logs/"
+            + "%(prop:buildername)s"
+        ),
+    )
+
+
 # FACTORY
 
 ## f_deb_install
 f_deb_install = util.BuildFactory()
 f_deb_install.addStep(getScript("deb-install.sh"))
 f_deb_install.addStep(getDebInstallStep())
+f_deb_install.addStep(getSaveArtifactsStep())
 
 ## f_deb_upgrade
 f_deb_upgrade = util.BuildFactory()
 f_deb_upgrade.addStep(getMajorVersionStep())
 f_deb_upgrade.addStep(getScript("deb-upgrade.sh"))
 f_deb_upgrade.addStep(getDebUpgradeStep())
+f_deb_upgrade.addStep(getSaveArtifactsStep())
 
 ## f_rpm_install
 f_rpm_install = util.BuildFactory()
 f_rpm_install.addStep(getScript("rpm-install.sh"))
 f_rpm_install.addStep(getRpmInstallStep())
+f_rpm_install.addStep(getSaveArtifactsStep())
 
 ## f_rpm_upgrade
 f_rpm_upgrade = util.BuildFactory()
 f_rpm_upgrade.addStep(getMajorVersionStep())
 f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh"))
 f_rpm_upgrade.addStep(getRpmUpgradeStep())
+f_rpm_upgrade.addStep(getSaveArtifactsStep())
 
 ####### WORKERS and BUILDERS
 

scripts/bash_lib.sh

@@ -4,6 +4,8 @@
 # Include with:
 # . ./bash_lib.sh
 
+trap save_failure_logs ERR
+
 bb_log_info() {
   set +x
   echo >&1 "INFO: $*"
@@ -437,22 +439,40 @@ upgrade_test_type() {
   esac
 }
 
-get_columnstore_logs() {
+save_failure_logs() {
+  local logdir=/home/buildbot/logs/
+  local logfile=$logdir/$test_mode.log
+  mkdir -p $logdir
   if [[ $test_mode == "columnstore" ]]; then
     bb_log_info "storing Columnstore logs in columnstore_logs"
     set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
-    done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    if [ -d /var/log/mariadb/columnstore ]; then
+      for f in $(sudo find /var/log/mariadb/columnstore -type f); do
+        echo "----------- $f -----------" >>"$logfile"
+        sudo cat "$f" 1>>"$logfile" 2>&1
+      done
+    fi
+    if [ -d /tmp/columnstore_tmp_files ]; then
+      for f in $(sudo find /tmp/columnstore_tmp_files -type f); do
+        echo "----------- $f -----------" >>"$logfile"
+        sudo cat "$f" | sudo tee -a "$logfile" 2>&1
+      done
+    fi
+    for s in mariadb.service mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \
+            mcs-primproc.service [email protected]; do
+        echo "----------- $s -----------" >>"$logfile"
+        sudo journalctl -u "$s" | sudo tee -a "$logfile" 2>&1
     done
+    if [ -d /var/lib/columnstore ]; then
+      tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore
+    fi
+  fi
+  echo "----------- mariadb.service -----------" >>"$logfile"
+  sudo journalctl -u mariadb.service | sudo tee -a "$logfile" 2>&1
+  if [ -f "$logfile" ]; then
+    bzip2 "$logfile"
   fi
+  sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \;
 }
 
 check_mariadb_server_and_create_structures() {
@@ -468,10 +488,7 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"
   fi
   set +e
 }
@@ -495,10 +512,7 @@ check_mariadb_server_and_verify_structures() {
   sudo mariadb -e "SELECT db.f()"
 
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"
   fi
   set +e
 }

scripts/deb-upgrade.sh

@@ -125,7 +125,6 @@ fi
 if [[ $systemdCapability == "yes" ]]; then
   if ! sudo systemctl status mariadb --no-pager; then
     sudo journalctl -xe --no-pager
-    get_columnstore_logs
     bb_log_err "mariadb service didn't start properly after installation"
     exit 1
   fi