diff --git a/tests/test_helper/bats/ovsdb_schema_upgrade.bats b/tests/test_helper/bats/ovsdb_schema_upgrade.bats index 44375de1..dd0b155b 100644 --- a/tests/test_helper/bats/ovsdb_schema_upgrade.bats +++ b/tests/test_helper/bats/ovsdb_schema_upgrade.bats @@ -57,14 +57,6 @@ setup() { old_member_msg_sb="$original_sb" fi - # Before we proceed with upgrade, make sure that cluster is fully converged. Starting - # the upgrade before the cluster is ready may lead to unexpectedly long convergence - # after the upgrade. - local container="" - for container in $TEST_CONTAINERS; do - run wait_microovn_online "$container" 60 - done - local container_index=0 local upgrade_container_index=0 local old_container_index=0 diff --git a/tests/test_helper/common.bash b/tests/test_helper/common.bash index 212f2c1a..109060ba 100644 --- a/tests/test_helper/common.bash +++ b/tests/test_helper/common.bash @@ -306,3 +306,18 @@ function collect_coverage() { lxc_pull_dir "$container/var/snap/microovn/common/data/coverage" "$output_dir" done } + +# ping_packets_lost CONTAINER DST [ NETNS ] +# +# Stop ping process previously started by a call to ``ping_start`` and print +# how many packets were lost, if any. +function ping_packets_lost() { + local container=$1; shift + local dst=$1; shift + local netns=$1 + + local n_lost + n_lost=$(ping_reap "$container" "$dst" "$netns" \ + | awk '/packets/{print$1-$4}') + echo "$n_lost" +} diff --git a/tests/test_helper/microovn.bash b/tests/test_helper/microovn.bash index 0c2ee344..b90a8430 100644 --- a/tests/test_helper/microovn.bash +++ b/tests/test_helper/microovn.bash @@ -547,18 +547,41 @@ function wait_ovsdb_cluster_container_join() { return $rc } +# microovn_status_is_schema_ok CONTAINER NBSB +# +# Checks whether schema for NBSB (nb|sb) on CONTAINER is OK from the +# perspective of the `microovn status` command. +function microovn_status_is_schema_ok() { + local container=$1; shift + local nbsb=$1; shift + + local schema_name + schema_name=$(_ovn_schema_name "$nbsb") + + local cmd + printf -v cmd 'microovn status | grep -q %s:\ OK' "${schema_name//_/\\ }" + + lxc_exec "$container" "$cmd" +} + MICROOVN_PREFIX_LS=sw MICROOVN_PREFIX_LR=lr MICROOVN_PREFIX_LRP=lrp-sw MICROOVN_SUFFIX_LRP_LSP=lrp -function microovn_extract_ctn_n__() { +# microovn_extract_ctn_n CONTAINER +# +# When CONTAINER is a string ending with "-n" this function will extract and +# validate that n is an integer before printing it. +# +# Note that it is up to the caller to ensure that the prerequisite mentioned +# above is met. +function microovn_extract_ctn_n() { local container=$1; shift local n=${container##*-} assert test "$n" -ge 0 - assert test "$n" -le 9 echo "$n" } @@ -575,11 +598,14 @@ function microovn_add_gw_router() { local container=$1; shift local n - n=$(microovn_extract_ctn_n__ "$container") + n=$(microovn_extract_ctn_n "$container") + assert test "$n" -le 255 local ls_name="${MICROOVN_PREFIX_LS}-${container}" local lr_name="${MICROOVN_PREFIX_LR}-${container}" local lrp_name="${MICROOVN_PREFIX_LRP}-${container}" local lrp_lsp_name="${ls_name}-${MICROOVN_SUFFIX_LRP_LSP}" + local lrp_addresses + printf -v lrp_addresses "00:00:02:00:00:%02x 10.42.%d.1/24" "$n" "$n" lxc_exec "$container" \ "microovn.ovn-nbctl \ @@ -590,8 +616,7 @@ function microovn_add_gw_router() { -- \ set Logical_Router $lr_name options:chassis=$container \ -- \ - lrp-add $lr_name $lrp_name \ - 00:00:02:00:00:0$n 10.42.$n.1/24 \ + lrp-add $lr_name $lrp_name $lrp_addresses \ -- \ lsp-add $ls_name $lrp_lsp_name \ -- \ @@ -650,9 +675,12 @@ function microovn_add_vif() { local if_name=$1; shift local n - n=$(microovn_extract_ctn_n__ "$container") - local lladdr="00:00:02:00:01:0$n" - local cidr="10.42.$n.10/24" + n=$(microovn_extract_ctn_n "$container") + assert test "$n" -le 255 + local lladdr + printf -v lladdr "00:00:02:00:01:%02x" "$n" + local cidr + printf -v cidr "10.42.%d.10/24" "$n" local ls_name="${MICROOVN_PREFIX_LS}-${container}" local lsp_name="${container}-${ns_name}-${if_name}" diff --git a/tests/test_helper/setup_teardown/upgrade.bash b/tests/test_helper/setup_teardown/upgrade.bash index 37ff29b0..ea3c2f73 100644 --- a/tests/test_helper/setup_teardown/upgrade.bash +++ b/tests/test_helper/setup_teardown/upgrade.bash @@ -22,7 +22,8 @@ setup_file() { export CENTRAL_CONTAINERS export CHASSIS_CONTAINERS - launch_containers $TEST_CONTAINERS + launch_containers_args \ + "${TEST_LXD_LAUNCH_ARGS:--c security.nesting=true}" $TEST_CONTAINERS wait_containers_ready $TEST_CONTAINERS install_microovn_from_store "$MICROOVN_SNAP_CHANNEL" $TEST_CONTAINERS bootstrap_cluster $TEST_CONTAINERS @@ -38,6 +39,13 @@ setup_file() { fi done + # Make sure that microcluster is fully converged before proceeding. + # Performing further actions before the microcluster is ready may lead to + # unexpectedly long convergence after a microcluster schema upgrade. + for container in $TEST_CONTAINERS; do + wait_microovn_online "$container" 60 + done + # detect and export initial MicroOVN snap revision container=$(echo "$TEST_CONTAINERS" | awk '{print $1;}' ) export MICROOVN_SNAP_REV="" @@ -45,24 +53,100 @@ setup_file() { assert [ -n "$MICROOVN_SNAP_REV" ] if [ -n "$UPGRADE_DO_UPGRADE" ]; then - echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV" >&3 - install_microovn "$MICROOVN_SNAP_PATH" $TEST_CONTAINERS + assert [ -n "$CENTRAL_CONTAINERS" ] + assert [ -n "$CHASSIS_CONTAINERS" ] + + # Export names used locally on chassis containers for use in + # teardown_file(). + export UPGRADE_NS_NAME="upgrade_ns0" + export UPGRADE_VIF_NAME="upgrade_vif0" + + # Set up gateway router, workload and background ping on each chassis. + for container in $CHASSIS_CONTAINERS; do + local ctn_n + ctn_n=$(microovn_extract_ctn_n "$container") + microovn_add_gw_router "$container" + netns_add "$container" "$UPGRADE_NS_NAME" + microovn_add_vif "$container" \ + "$UPGRADE_NS_NAME" "$UPGRADE_VIF_NAME" + ping_start "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME" + done + + echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV "\ + "on central container(s)." >&3 + install_microovn "$MICROOVN_SNAP_PATH" $CENTRAL_CONTAINERS + + for container in $CENTRAL_CONTAINERS; do + microovn_wait_ovndb_state "$container" nb connected 32 + microovn_wait_ovndb_state "$container" sb connected 32 + done + + maybe_perform_manual_upgrade_steps $CENTRAL_CONTAINERS + # Reap ping and assert on result. + # + # Start background ping for next measurement. + for container in $CHASSIS_CONTAINERS; do + local ctn_n + ctn_n=$(microovn_extract_ctn_n "$container") + local n_lost + n_lost=$(ping_packets_lost \ + "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME") + # Apart from the one packet that can be lost while stopping + # ``ping``, we expect zero loss. + assert test "$n_lost" -le 1 + + ping_start "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME" + done + + echo "# Upgrading MicroOVN from revision $MICROOVN_SNAP_REV "\ + "on chassis container(s)." >&3 + install_microovn "$MICROOVN_SNAP_PATH" $CHASSIS_CONTAINERS + + # Now that the remaining containers have been upgraded any pending + # schema conversions will be performed both for the microcluster and + # OVSDB databases. Ensure these processes are complete before + # measuring the result. for container in $TEST_CONTAINERS; do - local container_services - container_services=$(microovn_get_cluster_services "$container") - if [[ "$container_services" != *"central"* ]]; then - continue - fi - microovn_wait_ovndb_state "$container" nb connected 15 - microovn_wait_ovndb_state "$container" sb connected 15 + wait_microovn_online "$container" 60 + for db in nb sb; do + local cmd + printf -v cmd \ + 'microovn_status_is_schema_ok %s %s' "$container" "$db" + wait_until "$cmd" + done done - perform_manual_upgrade_steps $TEST_CONTAINERS + # Reap ping and assert on result. + for container in $CHASSIS_CONTAINERS; do + local ctn_n + ctn_n=$(microovn_extract_ctn_n "$container") + local max_lost=8 + local n_lost + n_lost=$(ping_packets_lost \ + "$container" 10.42.${ctn_n}.1 "$UPGRADE_NS_NAME") + # Upgrading the node with the instance being monitored will + # inevitably cause some data path interruption as Open vSwitch + # restarts. + assert test "$n_lost" -le "$max_lost" + echo "# Upgrade induced packet loss: $n_lost packets " \ + "(threshold $max_lost)" >&3 + done fi } teardown_file() { collect_coverage $TEST_CONTAINERS + + if [ -n "$UPGRADE_NS_NAME" ] && [ -n "$UPGRADE_VIF_NAME" ]; then + local container + for container in $CHASSIS_CONTAINERS; do + microovn_delete_vif "$container" \ + "$UPGRADE_NS_NAME" "$UPGRADE_VIF_NAME" + netns_delete "$container" "$UPGRADE_NS_NAME" + microovn_delete_gw_router "$container" + done + fi + delete_containers $TEST_CONTAINERS } diff --git a/tests/test_helper/upgrade_procedures.bash b/tests/test_helper/upgrade_procedures.bash index 9c000e47..d83f95bd 100644 --- a/tests/test_helper/upgrade_procedures.bash +++ b/tests/test_helper/upgrade_procedures.bash @@ -121,11 +121,11 @@ function revision_111_upgrade_tls() { ovsdb_rebuild_tls_cluster "sb" "$central_containers" } -# perform_manual_upgrade_steps CONTAINER1 [CONTAINER2 ...] +# maybe_perform_manual_upgrade_steps CONTAINER1 [CONTAINER2 ...] # # Sequentially execute manual steps that are required for upgrade # between certain MicroOVN snap revisions. -function perform_manual_upgrade_steps() { +function maybe_perform_manual_upgrade_steps() { local containers=$*; shift if [ "$MICROOVN_SNAP_REV" -lt 111 ]; then