Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tox with pep8,pylint and bashate #224

Merged
merged 1 commit into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 0 additions & 36 deletions .github/workflows/lint.yaml

This file was deleted.

47 changes: 47 additions & 0 deletions .github/workflows/run-tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# This is a templated file and must be kept up-to-date with the original
# from upstream at https://github.com/canonical/se-tooling-ci-common.
name: Run Tests
on:
- push
- pull_request
- workflow_dispatch

jobs:
test:
strategy:
matrix:
python-version: ['3.8', '3.10', '3.12']
os: [ubuntu-24.04, ubuntu-22.04, ubuntu-20.04]
exclude:
- os: ubuntu-20.04
python-version: '3.10'
- os: ubuntu-20.04
python-version: '3.12'
- os: ubuntu-22.04
python-version: '3.8'
- os: ubuntu-22.04
python-version: '3.12'
- os: ubuntu-24.04
python-version: '3.8'
- os: ubuntu-24.04
python-version: '3.10'
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r gh-test-requirements.txt
- name: Run pylint
run: tox -e pylint
if: matrix.python-version == '3.10'
- name: Run pep8
run: tox -e pep8
if: matrix.python-version == '3.10'
- name: Run bashate
run: tox -e bashate
if: matrix.python-version == '3.10'
3 changes: 2 additions & 1 deletion common/generate-bundle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ MOD_DIR=$(realpath $(dirname $0))
. $MOD_DIR/pipeline/02configure
. $MOD_DIR/pipeline/03build
# Ensure no unrendered variables
out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config --exclude-dir=p| egrep -v '^.*#'`" || exit 0
out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config \
--exclude-dir=p| egrep -v '^.*#'`" || exit 0
echo -e "ERROR: there are unrendered variables in your bundle:\n$out"
exit 1
51 changes: 35 additions & 16 deletions common/generate_bundle_base
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ update_master_opts ${MOD_PASSTHROUGH_OPTS[@]}
vip_start=${MASTER_OPTS[VIP_ADDR_START]}
if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
# prodstack
cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd -c cidr -f value 2>/dev/null)
cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd \
-c cidr -f value 2>/dev/null)
if [[ -z $cidr ]]; then
# stsstack
cidr=$(source ~/novarc; openstack subnet show ${OS_USERNAME}_admin_subnet -c cidr -f value 2>/dev/null)
cidr=$(source ~/novarc; openstack subnet show \
${OS_USERNAME}_admin_subnet \
-c cidr -f value 2>/dev/null)
if [[ -n $cidr ]]; then
vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+).+/\1/g').150.0
fi
Expand All @@ -24,7 +27,9 @@ if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
# last 20 addresses for vips which is prone to collisions but
# we have no alternative currently.
net_end=$(awk -F'.' '/HostMax/{print $NF}' <<<$(ipcalc -b $cidr))
vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end - 19))
vip_start=$(echo $cidr|
sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end -
19))
fi
fi
VIP_START_PREFIX=${vip_start%\.*}
Expand Down Expand Up @@ -67,7 +72,8 @@ finish ()
{
local target

echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > ${bundles_dir}/generate-command
echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > \
${bundles_dir}/generate-command
if has_opt --replay; then
target=${bundles_dir}/command
echo -e "INFO: replaying last known command (from $target)\n"
Expand Down Expand Up @@ -160,54 +166,67 @@ aggregate_mysql_interface_parts $bundles_dir/o
# generate bundle list
base_bundle=$bundles_dir/`basename $bundle`
overlay_list+=( $base_bundle )
readarray -t application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" ${overlay_list[@]})
readarray -t \
application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" \
${overlay_list[@]})

# Generate placement overlay for use with MAAS provider
if ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then
cp $MOD_DIR/overlays/unit_placement/header.yaml.template $bundles_dir/unit-placement.yaml
cp $MOD_DIR/overlays/unit_placement/header.yaml.template \
$bundles_dir/unit-placement.yaml
if [[ $MOD_NAME = openstack ]]; then
# these two represent total machines since e.g. ceph is deployed on compute but compute and gateway are never colocated
num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+
${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
elif [[ $MOD_NAME = kubernetes ]]; then
num_placement_machines=$((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
num_placement_machines=\
$((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+
${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
elif [[ $MOD_NAME = ceph ]]; then
num_placement_machines=$((${MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]}))
else
echo "ERROR: module '$MOD_NAME' does not yet have support for hyperconverged mode" 1>&2
echo -n "ERROR: module '$MOD_NAME'" 1>&2
echo " does not yet have support for hyperconverged mode" 1>&2
exit 1
fi
# detect all apps used and generate placement info by doing:
# * search for all app names in generated bundles/overlays
# * search unit_placement template with same name
for app in ${application_list[@]}; do
# filter juju keywords
[[ $app == "options" ]] || [[ $app == "to" ]] || [[ $app == "storage" ]] && continue
[[ $app =~ ^(options|to|storage)$ ]] && continue
app_placement=${app}.yaml
t=$MOD_DIR/overlays/unit_placement/${app_placement}.template
[ -r "$t" ] || continue
# load template
cp $t $PLACEMENT_OVERLAYS_DIR/$app_placement
# apply all renderers
render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement \
$num_placement_machines
render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement \
$num_placement_machines
render $PLACEMENT_OVERLAYS_DIR/$app_placement
# add to master placement overlay
cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> $bundles_dir/unit-placement.yaml
cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> \
$bundles_dir/unit-placement.yaml
done
# finally render master machine list
render_placement_machines $bundles_dir/unit-placement.yaml $num_placement_machines
render_placement_machines $bundles_dir/unit-placement.yaml \
$num_placement_machines
render $bundles_dir/unit-placement.yaml
# and add to list of overlays
overlay_opts+=( --overlay $bundles_dir/unit-placement.yaml )

# add default binding to all applications
bindings="\ bindings:\n '': ${MASTER_OPTS[DEFAULT_BINDING]}"
find $bundles_dir -name \*.yaml| xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
find $bundles_dir -name \*.yaml|\
xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
fi

((${#overlay_opts[@]})) && overlay_opts+=("") # right padding

echo -e "juju deploy${JUJU_DEPLOY_OPTS} ${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
echo -e "juju deploy${JUJU_DEPLOY_OPTS} \
${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
finish

for f in $INTERNAL_BUNDLE_CONFIG; do
Expand Down
1 change: 1 addition & 0 deletions gh-test-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tox<4.0.0
43 changes: 28 additions & 15 deletions openstack/tools/identify_charm_func_tests.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,45 @@
# Get names of test targets that OSCI would run for the given charm. Should be
# run from within the charm root.
#
# Outputs space seperated list of target names.
#
"""
Get names of test targets that OSCI would run for the given charm. Should be
run from within the charm root.
Outputs space separated list of target names.
"""
import os

import yaml

CLASSIC_TESTS_YAML = 'tests/tests.yaml'
REACTIVE_TESTS_YAML = os.path.join('src', CLASSIC_TESTS_YAML)

def extract_values(bundle_list):

def extract_targets(bundle_list):
"""
Targets are provided as strings or dicts where the target name is the
value so this accounts for both formats.
"""
extracted = []
for item in bundle_list:
if isinstance(item, dict):
extracted.append(list(item.values())[0])
else:
extracted.append(item)

return extracted

if os.path.exists(REACTIVE_TESTS_YAML):
bundles = yaml.safe_load(open(REACTIVE_TESTS_YAML))
else:
bundles = yaml.safe_load(open(CLASSIC_TESTS_YAML))

smoke_bundles = extract_values(bundles['smoke_bundles'])
gate_bundles = extract_values(bundles['gate_bundles'])
dev_bundles = extract_values(bundles['dev_bundles'])
if __name__ == "__main__":
if os.path.exists(REACTIVE_TESTS_YAML):
TESTS_FILE = REACTIVE_TESTS_YAML
else:
TESTS_FILE = CLASSIC_TESTS_YAML

with open(TESTS_FILE, encoding='utf-8') as fd:
bundles = yaml.safe_load(fd)

smoke_bundles = extract_targets(bundles['smoke_bundles'])
gate_bundles = extract_targets(bundles['gate_bundles'])
dev_bundles = extract_targets(bundles['dev_bundles'])

targets = set(smoke_bundles + gate_bundles + dev_bundles)
targets = set(smoke_bundles + gate_bundles + dev_bundles)

print(' '.join(sorted(targets)))
print(' '.join(sorted(targets)))
28 changes: 28 additions & 0 deletions pylintrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# This is a templated file and must be kept up-to-date with the original
# from upstream at https://github.com/canonical/se-tooling-ci-common.
[MAIN]
jobs=0
ignore=.git

# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins=pylint.extensions.no_self_use

# When enabled, pylint would attempt to guess common misconfiguration and emit
# user-friendly hints instead of false-positive error messages.
suggestion-mode=yes

[FORMAT]
max-line-length=79
# Allow doctrings containing long urls
ignore-long-lines=^\s+.+<?https?://\S+>?$

[REPORTS]
#reports=yes
score=yes

[MESSAGES CONTROL]
disable=

[DESIGN]
min-public-methods=1
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pyyaml
6 changes: 6 additions & 0 deletions test-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# This is a templated file and must be kept up-to-date with the original
# from upstream at https://github.com/canonical/se-tooling-ci-common.
bashate
flake8==6.1.0
flake8-import-order==0.18.2
pylint==3.1.0
24 changes: 14 additions & 10 deletions tools/juju-bundle-applications.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
#!/usr/bin/env python3

# pylint: disable=invalid-name
"""
Get Juju Bundle or Overlay Applications.
"""
import sys
import yaml

application_list = set()
import yaml

for filename in sys.argv[1:]:
with open(filename, 'r') as f:
data = yaml.load_all(f, Loader=yaml.SafeLoader)
for d in data:
if 'applications' in d:
application_list.update(d['applications'].keys())
print('\n'.join(application_list))
if __name__ == "__main__":
application_list = set()
for filename in sys.argv[1:]:
with open(filename, 'r', encoding='utf-8') as f:
data = yaml.load_all(f, Loader=yaml.SafeLoader)
for d in data:
if 'applications' in d:
application_list.update(d['applications'].keys())
print('\n'.join(application_list))
Loading