Merge pull request #138 from fnordahl/upgrade-dp-test

Measure data path uptime as part of upgrade test.

tests/test_helper/bats/ovsdb_schema_upgrade.bats

@@ -57,14 +57,6 @@ setup() {
         old_member_msg_sb="$original_sb"
     fi
 
-    # Before we proceed with upgrade, make sure that cluster is fully converged. Starting
-    # the upgrade before the cluster is ready may lead to unexpectedly long convergence
-    # after the upgrade.
-    local container=""
-    for container in $TEST_CONTAINERS; do
-        run wait_microovn_online "$container" 60
-    done
-
     local container_index=0
     local upgrade_container_index=0
     local old_container_index=0

tests/test_helper/common.bash

@@ -306,3 +306,18 @@ function collect_coverage() {
         lxc_pull_dir "$container/var/snap/microovn/common/data/coverage" "$output_dir"
     done
 }
+
+# ping_packets_lost CONTAINER DST [ NETNS ]
+#
+# Stop ping process previously started by a call to ``ping_start`` and print
+# how many packets were lost, if any.
+function ping_packets_lost() {
+    local container=$1; shift
+    local dst=$1; shift
+    local netns=$1
+
+    local n_lost
+    n_lost=$(ping_reap "$container" "$dst" "$netns" \
+        | awk '/packets/{print$1-$4}')
+    echo "$n_lost"
+}

tests/test_helper/microovn.bash

@@ -547,18 +547,41 @@ function wait_ovsdb_cluster_container_join() {
     return $rc
 }
 
+# microovn_status_is_schema_ok CONTAINER NBSB
+#
+# Checks whether schema for NBSB (nb|sb) on CONTAINER is OK from the
+# perspective of the `microovn status` command.
+function microovn_status_is_schema_ok() {
+    local container=$1; shift
+    local nbsb=$1; shift
+
+    local schema_name
+    schema_name=$(_ovn_schema_name "$nbsb")
+
+    local cmd
+    printf -v cmd 'microovn status | grep -q %s:\ OK' "${schema_name//_/\\ }"
+
+    lxc_exec "$container" "$cmd"
+}
+
 MICROOVN_PREFIX_LS=sw
 MICROOVN_PREFIX_LR=lr
 MICROOVN_PREFIX_LRP=lrp-sw
 
 MICROOVN_SUFFIX_LRP_LSP=lrp
 
-function microovn_extract_ctn_n__() {
+# microovn_extract_ctn_n CONTAINER
+#
+# When CONTAINER is a string ending with "-n" this function will extract and
+# validate that n is an integer before printing it.
+#
+# Note that it is up to the caller to ensure that the prerequisite mentioned
+# above is met.
+function microovn_extract_ctn_n() {
     local container=$1; shift
 
     local n=${container##*-}
     assert test "$n" -ge 0
-    assert test "$n" -le 9
 
     echo "$n"
 }
@@ -575,11 +598,14 @@ function microovn_add_gw_router() {
     local container=$1; shift
 
     local n
-    n=$(microovn_extract_ctn_n__ "$container")
+    n=$(microovn_extract_ctn_n "$container")
+    assert test "$n" -le 255
     local ls_name="${MICROOVN_PREFIX_LS}-${container}"
     local lr_name="${MICROOVN_PREFIX_LR}-${container}"
     local lrp_name="${MICROOVN_PREFIX_LRP}-${container}"
     local lrp_lsp_name="${ls_name}-${MICROOVN_SUFFIX_LRP_LSP}"
+    local lrp_addresses
+    printf -v lrp_addresses "00:00:02:00:00:%02x 10.42.%d.1/24" "$n" "$n"
 
     lxc_exec "$container" \
         "microovn.ovn-nbctl \
@@ -590,8 +616,7 @@ function microovn_add_gw_router() {
          -- \
          set Logical_Router $lr_name options:chassis=$container \
          -- \
-         lrp-add $lr_name $lrp_name \
-             00:00:02:00:00:0$n 10.42.$n.1/24 \
+         lrp-add $lr_name $lrp_name $lrp_addresses \
          -- \
          lsp-add $ls_name $lrp_lsp_name \
          -- \
@@ -650,9 +675,12 @@ function microovn_add_vif() {
     local if_name=$1; shift
 
     local n
-    n=$(microovn_extract_ctn_n__ "$container")
-    local lladdr="00:00:02:00:01:0$n"
-    local cidr="10.42.$n.10/24"
+    n=$(microovn_extract_ctn_n "$container")
+    assert test "$n" -le 255
+    local lladdr
+    printf -v lladdr "00:00:02:00:01:%02x" "$n"
+    local cidr
+    printf -v cidr "10.42.%d.10/24" "$n"
     local ls_name="${MICROOVN_PREFIX_LS}-${container}"
     local lsp_name="${container}-${ns_name}-${if_name}"
 

tests/test_helper/setup_teardown/upgrade.bash

@@ -22,7 +22,8 @@ setup_file() {
     export CENTRAL_CONTAINERS
     export CHASSIS_CONTAINERS
 
-    launch_containers $TEST_CONTAINERS
+    launch_containers_args \
+        "${TEST_LXD_LAUNCH_ARGS:--c security.nesting=true}" $TEST_CONTAINERS
     wait_containers_ready $TEST_CONTAINERS
     install_microovn_from_store "$MICROOVN_SNAP_CHANNEL" $TEST_CONTAINERS
     bootstrap_cluster $TEST_CONTAINERS
@@ -38,31 +39,114 @@ setup_file() {
         fi
     done
 
+    # Make sure that microcluster is fully converged before proceeding.
+    # Performing further actions before the microcluster is ready may lead to
+    # unexpectedly long convergence after a microcluster schema upgrade.
+    for container in $TEST_CONTAINERS; do
+        wait_microovn_online "$container" 60
+    done
+
     # detect and export initial MicroOVN snap revision
     container=$(echo "$TEST_CONTAINERS" | awk '{print $1;}' )
     export MICROOVN_SNAP_REV=""
     MICROOVN_SNAP_REV=$(lxc_exec "$container" "snap list | grep microovn | awk '{print \$3;}'")
     assert [ -n "$MICROOVN_SNAP_REV" ]
 
     if [ -n "$UPGRADE_DO_UPGRADE" ]; then
-        echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV" >&3
-        install_microovn "$MICROOVN_SNAP_PATH" $TEST_CONTAINERS
+        assert [ -n "$CENTRAL_CONTAINERS" ]
+        assert [ -n "$CHASSIS_CONTAINERS" ]
+
+        # Export names used locally on chassis containers for use in
+        # teardown_file().
+        export UPGRADE_NS_NAME="upgrade_ns0"
+        export UPGRADE_VIF_NAME="upgrade_vif0"
+
+        # Set up gateway router, workload and background ping on each chassis.
+        for container in $CHASSIS_CONTAINERS; do
+            local ctn_n
+            ctn_n=$(microovn_extract_ctn_n "$container")
+            microovn_add_gw_router "$container"
+            netns_add "$container" "$UPGRADE_NS_NAME"
+            microovn_add_vif "$container" \
+                "$UPGRADE_NS_NAME" "$UPGRADE_VIF_NAME"
+            ping_start "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME"
+        done
+
+        echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV "\
+             "on central container(s)." >&3
+        install_microovn "$MICROOVN_SNAP_PATH" $CENTRAL_CONTAINERS
+
+        for container in $CENTRAL_CONTAINERS; do
+            microovn_wait_ovndb_state "$container" nb connected 32
+            microovn_wait_ovndb_state "$container" sb connected 32
+        done
+
+        maybe_perform_manual_upgrade_steps $CENTRAL_CONTAINERS
 
+        # Reap ping and assert on result.
+        #
+        # Start background ping for next measurement.
+        for container in $CHASSIS_CONTAINERS; do
+            local ctn_n
+            ctn_n=$(microovn_extract_ctn_n "$container")
+            local n_lost
+            n_lost=$(ping_packets_lost \
+                "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME")
+            # Apart from the one packet that can be lost while stopping
+            # ``ping``, we expect zero loss.
+            assert test "$n_lost" -le 1
+
+            ping_start "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME"
+        done
+
+        echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV "\
+             "on chassis container(s)." >&3
+        install_microovn "$MICROOVN_SNAP_PATH" $CHASSIS_CONTAINERS
+
+        # Now that the remaining containers have been upgraded any pending
+        # schema conversions will be performed both for the microcluster and
+        # OVSDB databases.  Ensure these processes are complete before
+        # measuring the result.
         for container in $TEST_CONTAINERS; do
-            local container_services
-            container_services=$(microovn_get_cluster_services "$container")
-            if [[ "$container_services" != *"central"* ]]; then
-                continue
-            fi
-            microovn_wait_ovndb_state "$container" nb connected 15
-            microovn_wait_ovndb_state "$container" sb connected 15
+            wait_microovn_online "$container" 60
+            for db in nb sb; do
+                local cmd
+                printf -v cmd \
+                    'microovn_status_is_schema_ok %s %s' "$container" "$db"
+                wait_until "$cmd"
+            done
         done
 
-        perform_manual_upgrade_steps $TEST_CONTAINERS
+        # Reap ping and assert on result.
+        for container in $CHASSIS_CONTAINERS; do
+            local ctn_n
+            ctn_n=$(microovn_extract_ctn_n "$container")
+            local max_lost=8
+            local n_lost
+            n_lost=$(ping_packets_lost \
+                "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME")
+            # Upgrading the node with the instance being monitored will
+            # inevitably cause some data path interruption as Open vSwitch
+            # restarts.
+            assert test "$n_lost" -le "$max_lost"
+            echo "# Upgrade induced packet loss: $n_lost packets " \
+                 "(threshold $max_lost)" >&3
+        done
     fi
 }
 
 teardown_file() {
     collect_coverage $TEST_CONTAINERS
+
+    if [ -n "$UPGRADE_NS_NAME" ] && [ -n "$UPGRADE_VIF_NAME" ]; then
+        local container
+        for container in $CHASSIS_CONTAINERS; do
+            microovn_delete_vif "$container" \
+                "$UPGRADE_NS_NAME" "$UPGRADE_VIF_NAME"
+            netns_delete "$container" "$UPGRADE_NS_NAME"
+            microovn_delete_gw_router "$container"
+        done
+    fi
+
     delete_containers $TEST_CONTAINERS
 }

tests/test_helper/upgrade_procedures.bash

@@ -121,11 +121,11 @@ function revision_111_upgrade_tls() {
     ovsdb_rebuild_tls_cluster "sb" "$central_containers"
 }
 
-# perform_manual_upgrade_steps CONTAINER1 [CONTAINER2 ...]
+# maybe_perform_manual_upgrade_steps CONTAINER1 [CONTAINER2 ...]
 #
 # Sequentially execute manual steps that are required for upgrade
 # between certain MicroOVN snap revisions.
-function perform_manual_upgrade_steps() {
+function maybe_perform_manual_upgrade_steps() {
     local containers=$*; shift
 
     if [ "$MICROOVN_SNAP_REV" -lt 111 ]; then