Add tox with pep8,pylint and bashate

Tests all python scripts and starts adding coverage for
bash scripts.

.github/workflows/run-tests.yaml

@@ -0,0 +1,47 @@
+# This is a templated file and must be kept up-to-date with the original
+# from upstream at https://github.com/canonical/se-tooling-ci-common.
+name: Run Tests
+on:
+  - push
+  - pull_request
+  - workflow_dispatch
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.10', '3.12']
+        os: [ubuntu-24.04, ubuntu-22.04, ubuntu-20.04]
+        exclude:
+          - os: ubuntu-20.04
+            python-version: '3.10'
+          - os: ubuntu-20.04
+            python-version: '3.12'
+          - os: ubuntu-22.04
+            python-version: '3.8'
+          - os: ubuntu-22.04
+            python-version: '3.12'
+          - os: ubuntu-24.04
+            python-version: '3.8'
+          - os: ubuntu-24.04
+            python-version: '3.10'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r gh-test-requirements.txt
+    - name: Run pylint
+      run: tox -e pylint
+      if: matrix.python-version == '3.10'
+    - name: Run pep8
+      run: tox -e pep8
+      if: matrix.python-version == '3.10'
+    - name: Run bashate
+      run: tox -e bashate
+      if: matrix.python-version == '3.10'

common/generate-bundle.sh

@@ -6,6 +6,7 @@ MOD_DIR=$(realpath $(dirname $0))
 . $MOD_DIR/pipeline/02configure
 . $MOD_DIR/pipeline/03build
 # Ensure no unrendered variables
-out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config --exclude-dir=p| egrep -v '^.*#'`" || exit 0
+out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config \
+        --exclude-dir=p| egrep -v '^.*#'`" || exit 0
 echo -e "ERROR: there are unrendered variables in your bundle:\n$out"
 exit 1

common/generate_bundle_base

@@ -12,10 +12,13 @@ update_master_opts ${MOD_PASSTHROUGH_OPTS[@]}
 vip_start=${MASTER_OPTS[VIP_ADDR_START]}
 if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
     # prodstack
-    cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd -c cidr -f value 2>/dev/null)
+    cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd \
+                            -c cidr -f value 2>/dev/null)
     if [[ -z $cidr ]]; then
         # stsstack
-        cidr=$(source ~/novarc; openstack subnet show ${OS_USERNAME}_admin_subnet -c cidr -f value 2>/dev/null)
+        cidr=$(source ~/novarc; openstack subnet show \
+                                ${OS_USERNAME}_admin_subnet \
+                                -c cidr -f value 2>/dev/null)
         if [[ -n $cidr ]]; then
             vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+).+/\1/g').150.0
         fi
@@ -24,7 +27,9 @@ if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
         # last 20 addresses for vips which is prone to collisions but
         # we have no alternative currently.
         net_end=$(awk -F'.' '/HostMax/{print $NF}' <<<$(ipcalc -b $cidr))
-        vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end - 19))
+        vip_start=$(echo $cidr|
+                    sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end -
+                    19))
     fi
 fi
 VIP_START_PREFIX=${vip_start%\.*}
@@ -67,7 +72,8 @@ finish ()
 {
     local target
 
-    echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > ${bundles_dir}/generate-command
+    echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > \
+            ${bundles_dir}/generate-command
     if has_opt --replay; then
         target=${bundles_dir}/command
         echo -e "INFO: replaying last known command (from $target)\n"
@@ -160,54 +166,68 @@ aggregate_mysql_interface_parts $bundles_dir/o
 # generate bundle list
 base_bundle=$bundles_dir/`basename $bundle`
 overlay_list+=( $base_bundle )
-readarray -t application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" ${overlay_list[@]})
+readarray -t \
+    application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" \
+    ${overlay_list[@]})
 
 # Generate placement overlay for use with MAAS provider
 if ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then
-    cp  $MOD_DIR/overlays/unit_placement/header.yaml.template $bundles_dir/unit-placement.yaml
+    cp $MOD_DIR/overlays/unit_placement/header.yaml.template \
+       $bundles_dir/unit-placement.yaml
     if [[ $MOD_NAME = openstack ]]; then
         # these two represent total machines since e.g. ceph is deployed on compute but compute and gateway are never colocated
-        num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
+        num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+
+                                    ${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
     elif [[ $MOD_NAME = kubernetes ]]; then
-        num_placement_machines=$((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
+        num_placement_machines=\
+            $((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+
+                ${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
     elif [[ $MOD_NAME = ceph ]]; then
         num_placement_machines=$((${MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]}))
     else
-        echo "ERROR: module '$MOD_NAME' does not yet have support for hyperconverged mode" 1>&2
+        echo -n "ERROR: module '$MOD_NAME'" 1>&2
+        echo " does not yet have support for hyperconverged mode" 1>&2
         exit 1
     fi
     # detect all apps used and generate placement info by doing:
     #  * search for all app names in generated bundles/overlays
     #  * search unit_placement template with same name
     for app in ${application_list[@]}; do
         # filter juju keywords
-        [[ $app == "options" ]] || [[ $app == "to" ]] || [[ $app == "storage" ]] && continue
+        [[ $app == "options" ]] || [[ $app == "to" ]] || \
+            [[ $app == "storage" ]] && continue
         app_placement=${app}.yaml
         t=$MOD_DIR/overlays/unit_placement/${app_placement}.template
         [ -r "$t" ] || continue
         # load template
         cp $t $PLACEMENT_OVERLAYS_DIR/$app_placement
         # apply all renderers
-        render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
-        render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
+        render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement \
+                                   $num_placement_machines
+        render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement \
+                                     $num_placement_machines
         render $PLACEMENT_OVERLAYS_DIR/$app_placement
         # add to master placement overlay
-        cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> $bundles_dir/unit-placement.yaml
+        cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> \
+                $bundles_dir/unit-placement.yaml
     done
     # finally render master machine list
-    render_placement_machines $bundles_dir/unit-placement.yaml $num_placement_machines
+    render_placement_machines $bundles_dir/unit-placement.yaml \
+                              $num_placement_machines
     render $bundles_dir/unit-placement.yaml
     # and add to list of overlays
     overlay_opts+=( --overlay $bundles_dir/unit-placement.yaml )
 
     # add default binding to all applications
     bindings="\    bindings:\n      '': ${MASTER_OPTS[DEFAULT_BINDING]}"
-    find $bundles_dir -name \*.yaml| xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
+    find $bundles_dir -name \*.yaml|\
+        xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
 fi
 
 ((${#overlay_opts[@]})) && overlay_opts+=("")  # right padding
 
-echo -e "juju deploy${JUJU_DEPLOY_OPTS} ${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
+echo -e "juju deploy${JUJU_DEPLOY_OPTS} \
+    ${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
 finish
 
 for f in $INTERNAL_BUNDLE_CONFIG; do

gh-test-requirements.txt

@@ -0,0 +1 @@
+tox<4.0.0

openstack/tools/identify_charm_func_tests.py

@@ -1,32 +1,45 @@
-# Get names of test targets that OSCI would run for the given charm. Should be
-# run from within the charm root.
-#
-# Outputs space seperated list of target names.
-#
+"""
+Get names of test targets that OSCI would run for the given charm. Should be
+run from within the charm root.
+
+Outputs space separated list of target names.
+"""
 import os
+
 import yaml
 
 CLASSIC_TESTS_YAML = 'tests/tests.yaml'
 REACTIVE_TESTS_YAML = os.path.join('src', CLASSIC_TESTS_YAML)
 
-def extract_values(bundle_list):
+
+def extract_targets(bundle_list):
+    """
+    Targets are provided as strings or dicts where the target name is the
+    value so this accounts for both formats.
+    """
     extracted = []
     for item in bundle_list:
         if isinstance(item, dict):
             extracted.append(list(item.values())[0])
         else:
             extracted.append(item)
+
     return extracted
 
-if os.path.exists(REACTIVE_TESTS_YAML):
-    bundles = yaml.safe_load(open(REACTIVE_TESTS_YAML))
-else:
-    bundles = yaml.safe_load(open(CLASSIC_TESTS_YAML))
 
-smoke_bundles = extract_values(bundles['smoke_bundles'])
-gate_bundles = extract_values(bundles['gate_bundles'])
-dev_bundles = extract_values(bundles['dev_bundles'])
+if __name__ == "__main__":
+    if os.path.exists(REACTIVE_TESTS_YAML):
+        TESTS_FILE = REACTIVE_TESTS_YAML
+    else:
+        TESTS_FILE = CLASSIC_TESTS_YAML
+
+    with open(TESTS_FILE, encoding='utf-8') as fd:
+        bundles = yaml.safe_load(fd)
+
+    smoke_bundles = extract_targets(bundles['smoke_bundles'])
+    gate_bundles = extract_targets(bundles['gate_bundles'])
+    dev_bundles = extract_targets(bundles['dev_bundles'])
 
-targets = set(smoke_bundles + gate_bundles + dev_bundles)
+    targets = set(smoke_bundles + gate_bundles + dev_bundles)
 
-print(' '.join(sorted(targets)))
+    print(' '.join(sorted(targets)))

pylintrc

@@ -0,0 +1,28 @@
+# This is a templated file and must be kept up-to-date with the original
+# from upstream at https://github.com/canonical/se-tooling-ci-common.
+[MAIN]
+jobs=0
+ignore=.git
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=pylint.extensions.no_self_use
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+[FORMAT]
+max-line-length=79
+# Allow doctrings containing long urls
+ignore-long-lines=^\s+.+<?https?://\S+>?$
+
+[REPORTS]
+#reports=yes
+score=yes
+
+[MESSAGES CONTROL]
+disable=
+
+[DESIGN]
+min-public-methods=1

requirements.txt

@@ -0,0 +1 @@
+pyyaml

test-requirements.txt

@@ -0,0 +1,6 @@
+# This is a templated file and must be kept up-to-date with the original
+# from upstream at https://github.com/canonical/se-tooling-ci-common.
+bashate
+flake8==6.1.0
+flake8-import-order==0.18.2
+pylint==3.1.0

tools/juju-bundle-applications.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python3
-
+# pylint: disable=invalid-name
+"""
+Get Juju Bundle or Overlay Applications.
+"""
 import sys
-import yaml
 
-application_list = set()
+import yaml
 
-for filename in sys.argv[1:]:
-    with open(filename, 'r') as f:
-        data = yaml.load_all(f, Loader=yaml.SafeLoader)
-        for d in data:
-            if 'applications' in d:
-                application_list.update(d['applications'].keys())
-print('\n'.join(application_list))
+if __name__ == "__main__":
+    application_list = set()
+    for filename in sys.argv[1:]:
+        with open(filename, 'r', encoding='utf-8') as f:
+            data = yaml.load_all(f, Loader=yaml.SafeLoader)
+            for d in data:
+                if 'applications' in d:
+                    application_list.update(d['applications'].keys())
+    print('\n'.join(application_list))