Skip to content

Commit

Permalink
Report results after charm func tests
Browse files Browse the repository at this point in the history
Report contains info about whether tests are voting or not and
also the commit id the tests were run against.
  • Loading branch information
dosaboy committed Sep 10, 2024
1 parent 0336a37 commit d408767
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 3 deletions.
31 changes: 29 additions & 2 deletions openstack/tools/charmed_openstack_functest_runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,13 @@ do
shift
done

TOOLS_PATH=$(realpath $(dirname $0))/func_test_tools
CHARM_PATH=$(pwd)

# Get commit we are running tests against.
COMMIT_ID=$(git -C $CHARM_PATH rev-parse --short HEAD)
CHARM_NAME=$(egrep -o "^name: .+" metadata.yaml| cut -f 2 -d ' ')

source ~/novarc
export {,TEST_}CIDR_EXT=`openstack subnet show subnet_${OS_USERNAME}-psd-extra -c cidr -f value`
FIP_MAX=$(ipcalc $CIDR_EXT| awk '$1=="HostMax:" {print $2}')
Expand Down Expand Up @@ -135,10 +142,12 @@ if [[ -n $FUNC_TEST_PR ]]; then
)
fi

declare -A func_targets=()
if [[ -n $FUNC_TEST_TARGET ]]; then
func_targets=( $FUNC_TEST_TARGET )
func_targets[$FUNC_TEST_TARGET]=null
else
func_targets=( $(python3 $(realpath $(dirname $0))/identify_charm_func_tests.py) )
for target in $(python3 $TOOLS_PATH/identify_charm_func_tests.py):
func_targets[target]=null
fi

if $MODIFY_BUNDLE_CONSTRAINTS; then
Expand All @@ -148,6 +157,7 @@ if $MODIFY_BUNDLE_CONSTRAINTS; then
)
fi

echo "Running functional tests for charm $CHARM_NAME commit $COMMIT_ID"
for target in ${func_targets[@]}; do
[[ -d src ]] && pushd src || true
tox -re func-target -- $target
Expand All @@ -158,3 +168,20 @@ for target in ${func_targets[@]}; do
juju destroy-model --no-prompt $model --force --no-wait --destroy-storage
done

# Report results
echo "Test results for charm $CHARM_NAME functional tests @ commit $COMMIT_ID"
for target in ${!func_targets[@]}; do
if $(python3 $TOOLS_PATH/test_is_voting.py) $target; then
voting_info=" (non-voting)"
else
voting_info=""

if [[ ${func_targets[$target]} = null ]]; then
echo "$target: SKIPPED$voting_info"
elif [[ ${func_targets[$target]} = success ]]; then
echo "$target: SUCCESS$voting_info"
else
echo "$target: FAILURE$voting_info"
fi
done

Empty file.
34 changes: 34 additions & 0 deletions openstack/tools/func_test_tools/test_is_voting.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""
Takes a func test target name as input and exits 0 if the test is voting and
1 if not.
"""
import os
import sys

import yaml

if __name__ == "__main__":
target_name = sys.argv[1]
if not os.path.exists('osci.yaml'):
sys.stderr.write(f"ERROR: osci.yaml not found - assuming "
f"{target_name} is voting.\n")
sys.exit(0)

with open('osci.yaml', encoding='utf-8') as fd:
osci_config = yaml.safe_load(fd)

try:
jobs = osci_config[0]['project']['check']['jobs']
if target_name in jobs:
# default is voting=True
sys.exit(0)

for check in jobs:
if isinstance(check, dict) and target_name in check:
if not check[target_name]['voting']:
sys.exit(1)
except KeyError as exc:
sys.stderr.write(f"ERROR: failed to process osci.yaml - assuming "
f"{target_name} is voting (key {exc} not found).\n")

sys.exit(0)
2 changes: 1 addition & 1 deletion tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import-order-style = pep8
[testenv]
basepython = {env:TOX_PYTHON:python3}
pyfiles =
{toxinidir}/openstack/tools/identify_charm_func_tests.py
{toxinidir}/openstack/tools/func_test_tools
{toxinidir}/tools/parse-bundle.py
{toxinidir}/tools/juju-bundle-applications.py

Expand Down

0 comments on commit d408767

Please sign in to comment.