add communication probes

.github/buildomat/jobs/a4x2-deploy.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+#:
+#: name = "a4x2-deploy"
+#: variety = "basic"
+#: target = "lab-2.0-opte-0.27"
+#: rust_toolchain = "stable"
+#: output_rules = [
+#:	"/out/falcon/*.log",
+#:	"/out/falcon/*.err",
+#:  "/out/connectivity-report.json",
+#   "/out/*.log",
+#: ]
+#: skip_clone = true
+#:
+#: [dependencies.a4x2]
+#: job = "a4x2-prepare"
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+pfexec mkdir -p /out
+pfexec chown "$UID" /out
+
+#
+# If we fail, try to collect some debugging information
+#
+_exit_trap() {
+	local status=$?
+	[[ $status -eq 0 ]] && exit 0
+
+    set +o errexit
+
+    mkdir -p /out/falcon
+    cp .falcon/* /out/falcon/
+    for x in ce cr1 cr2 g0 g1 g2 g3; do
+        mv /out/falcon/$x.out /out/falcon/$x.log
+    done
+    cp connectivity-report.json /out/
+
+    for gimlet in g0 g1 g2 g3; do
+        ./a4x2 exec                                           \
+            $gimlet                                           \
+            "cat /var/svc/log/oxide-sled-agent:default.log" > \
+            /out/$gimlet-sled-agent.log
+    done
+}
+trap _exit_trap EXIT
+
+#
+# Install propolis
+#
+curl -fOL https://buildomat.eng.oxide.computer/wg/0/artefact/01HJ4BJJY2Q9EKXHYV6HQZ8XPN/qQS2fnkS9LebcL4cDLeHRWdleSiXaGKEXGLDucRoab8pwBSi/01HJ4BJY5F995ET252YSD4NJWV/01HJ4CGFH946THBF0ZRH6SRM8X/propolis-server
+chmod +x propolis-server
+pfexec mv propolis-server /usr/bin/
+
+#
+# Make space for CI work
+#
+export DISK=${DISK:-c1t1d0}
+pfexec diskinfo
+pfexec zpool create -f cpool $DISK
+pfexec zfs create -o mountpoint=/ci cpool/ci
+pfexec chown "$UID" /ci
+cd /ci
+
+#
+# Fetch and decompress the cargo bay from the a4x2-prepeare job
+#
+for x in ce cr1 cr2 omicron-common g0 g1 g2 g3 tools; do
+    tar -xvzf /input/a4x2/out/cargo-bay-$x.tgz
+done
+
+for sled in g0 g1 g2 g3; do
+    cp -r cargo-bay/omicron-common/omicron/out/* cargo-bay/$sled/omicron/out/
+done
+ls -R
+
+#
+# Fetch the a4x2 topology manager program
+#
+buildomat_url=https://buildomat.eng.oxide.computer
+testbed_artifact_path=public/file/oxidecomputer/testbed/topo/
+testbed_rev=667a7474968b15dafc8369e6cdc523b8d25630b2
+curl -fOL $buildomat_url/$testbed_artifact_path/$testbed_rev/a4x2
+chmod +x a4x2
+
+#
+# Create a zpool for falcon images and disks
+#
+export FALCON_DATASET=cpool/falcon
+
+#
+# Install falcon base images
+#
+github_raw=https://raw.githubusercontent.com
+falcon_path=oxidecomputer/falcon
+falcon_branch=main
+curl -sSf $github_raw/$falcon_path/$falcon_branch/setup-base-images.sh | bash
+curl -sSf $github_raw/$falcon_path/$falcon_branch/get-ovmf.sh | bash
+
+#
+# Fetch the arista image
+#
+curl -OL https://oxide-falcon-assets.s3.us-west-2.amazonaws.com/arista.gz.xz
+unxz arista.gz.xz
+pfexec zfs receive cpool/falcon/img/arista@base < arista.gz
+
+#
+# Run the VM dhcp server
+#
+export EXT_INTERFACE=${EXT_INTERFACE:-igb0}
+
+cp /input/a4x2/out/dhcp-server .
+chmod +x dhcp-server
+first=`bmat address ls -f extra -Ho first`
+last=`bmat address ls -f extra -Ho last`
+gw=`bmat address ls -f extra -Ho gateway`
+server=`ipadm show-addr $EXT_INTERFACE/dhcp -po ADDR | sed 's#/.*##g'`
+pfexec ./dhcp-server $first $last $gw $server &> /out/dhcp-server.log &
+
+#
+# Run the topology
+#
+pfexec ./a4x2 launch
+
+#
+# Add a route to the rack ip pool
+#
+
+# XXX i think this will be on the CI machine config by default, but leaving a
+# breadcrumb here just in case.
+#
+# Get a DHCP address for the external interface on the LAB network.
+# ipadm create-addr -t -T dhcp $EXT_INTERFACE/testbed
+
+# Get the DHCP address for the external interface of the customer edge VM. This
+# VM interface is attached to the host machine's external interface via viona.
+customer_edge_addr=$(./a4x2 exec ce \
+    "ip -4 -j addr show enp0s10 | jq -r '.[0].addr_info[] | select(.dynamic == true) | .local'")
+
+# Add the route to the rack via the customer edge VM
+pfexec dladm
+pfexec ipadm
+pfexec netstat -nr
+pfexec route add 198.51.100.0/24 $customer_edge_addr
+
+#
+# Run the communications test program
+#
+cp /input/a4x2/out/commtest .
+chmod +x commtest
+pfexec ./commtest http://198.51.100.23 run \
+    --ip-pool-begin 198.51.100.40 \
+    --ip-pool-end 198.51.100.70 \
+    --icmp-loss-tolerance 10 \
+    --test-duration 200s \
+    --packet-rate 10
+
+cp connectivity-report.json /out/

.github/buildomat/jobs/a4x2-prepare.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+#:
+#: name = "a4x2-prepare"
+#: variety = "basic"
+#: target = "helios-2.0"
+#: rust_toolchain = "stable"
+#: output_rules = [
+#:	"=/out/cargo-bay-ce.tgz",
+#:	"=/out/cargo-bay-cr1.tgz",
+#:	"=/out/cargo-bay-cr2.tgz",
+#:	"=/out/cargo-bay-g0.tgz",
+#:	"=/out/cargo-bay-g1.tgz",
+#:	"=/out/cargo-bay-g2.tgz",
+#:	"=/out/cargo-bay-g3.tgz",
+#:	"=/out/cargo-bay-tools.tgz",
+#:	"=/out/cargo-bay-omicron-common.tgz",
+#:	"=/out/commtest",
+#:	"=/out/dhcp-server",
+#: ]
+#: access_repos = [
+#:	"oxidecomputer/testbed",
+#: ]
+
+source ./env.sh
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+pfexec mkdir -p /out
+pfexec chown "$UID" /out
+
+#
+# Prep to build omicron
+#
+banner "prerequisites"
+set -o xtrace
+./tools/install_builder_prerequisites.sh -y
+
+#
+# Build the commtest program and place in the output
+#
+banner "commtest"
+cargo build -p end-to-end-tests --bin commtest --bin dhcp-server --release
+cp target/release/commtest /out/
+cp target/release/dhcp-server /out/
+
+#
+# Clone the testbed repo
+#
+banner "testbed"
+cd /work/oxidecomputer
+rm -rf testbed
+git clone https://github.com/oxidecomputer/testbed
+cd testbed/a4x2
+
+#
+# Build the a4x2 cargo bay using the omicron sources in this branch, fetch the
+# softnpu artifacts into the cargo bay, zip up the cargo bay and place it in the
+# output.
+#
+OMICRON=/work/oxidecomputer/omicron ./config/build-packages.sh
+
+# Create an omicron archive that captures common assets
+
+pushd cargo-bay
+mkdir -p omicron-common/omicron/
+cp -r g0/omicron/out omicron-common/omicron/
+# sled agent archive is sled-specific
+rm omicron-common/omicron/out/omicron-sled-agent.tar 
+popd
+
+# Remove everything in $sled/omicron/out except sled agent tar, these common
+# elements are in the omicron-common archive
+for sled in g0 g1 g2 g3; do
+    find cargo-bay/$sled/omicron/out/ -maxdepth 1 -mindepth 1 \
+        | grep -v sled-agent | xargs -l rm -rf
+done
+
+# Put the softnpu artifacts in place.
+./config/fetch-softnpu-artifacts.sh
+
+# Archive everything up and place it in the output
+for x in ce cr1 cr2 g0 g1 g2 g3 tools omicron-common; do
+    tar -czf cargo-bay-$x.tgz cargo-bay/$x
+    mv cargo-bay-$x.tgz /out/
+done
+

.github/buildomat/jobs/ci-tools.sh

@@ -8,6 +8,7 @@
 #:	"=/work/end-to-end-tests/*.gz",
 #:	"=/work/caboose-util.gz",
 #:	"=/work/tufaceous.gz",
+#:  "=/work/commtest",
 #: ]
 
 set -o errexit
@@ -33,6 +34,10 @@ export CARGO_INCREMENTAL=0
 ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \
 	--message-format json-render-diagnostics >/tmp/output.end-to-end.json
 
+mkdir -p /work
+ptime -m cargo build --locked -p end-to-end-tests --tests --bin commtest
+cp target/debug/commtest /work/commtest
+
 mkdir -p /work/end-to-end-tests
 for p in target/debug/bootstrap $(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json); do
 	# shellcheck disable=SC2094

.github/buildomat/jobs/package.sh

@@ -124,6 +124,7 @@ zones=(
   out/omicron-gateway-softnpu.tar.gz
   out/omicron-gateway-asic.tar.gz
   out/overlay.tar.gz
+  out/probe.tar.gz
 )
 cp "${zones[@]}" /work/zones/
 

.gitignore

@@ -15,3 +15,4 @@ debug.out
 rusty-tags.vi
 *.sw*
 tags
+connectivity-report.json