Merge branch 'canonical:main' into unseal-vault

.github/workflows/run-tests.yaml

@@ -0,0 +1,47 @@
+# This is a templated file and must be kept up-to-date with the original
+# from upstream at https://github.com/canonical/se-tooling-ci-common.
+name: Run Tests
+on:
+  - push
+  - pull_request
+  - workflow_dispatch
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.10', '3.12']
+        os: [ubuntu-24.04, ubuntu-22.04, ubuntu-20.04]
+        exclude:
+          - os: ubuntu-20.04
+            python-version: '3.10'
+          - os: ubuntu-20.04
+            python-version: '3.12'
+          - os: ubuntu-22.04
+            python-version: '3.8'
+          - os: ubuntu-22.04
+            python-version: '3.12'
+          - os: ubuntu-24.04
+            python-version: '3.8'
+          - os: ubuntu-24.04
+            python-version: '3.10'
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r gh-test-requirements.txt
+    - name: Run pylint
+      run: tox -e pylint
+      if: matrix.python-version == '3.10'
+    - name: Run pep8
+      run: tox -e pep8
+      if: matrix.python-version == '3.10'
+    - name: Run bashate
+      run: tox -e bashate
+      if: matrix.python-version == '3.10'

.gitignore

@@ -2,3 +2,4 @@ b
 bundles
 ssl/*/
 microk8s/dockercreds
+__pycache__

.module_defaults/cos

@@ -0,0 +1 @@
+../cos/module_defaults

common/ceph_release_info

@@ -13,5 +13,6 @@ export -A APP_RELEASE_NAMES=( [firefly]=icehouse
                               [octopus]=ussuri
                               [pacific]=wallaby # xena and wallaby both have pacific
                               [quincy]=yoga
-                              [reef]=bobcat )   # bobcat and caracal both have reef
+                              [reef]=bobcat
+                              [squid]=caracal )
 

common/ch_channel_map/any_series

@@ -12,5 +12,9 @@ for c in ${CEPH_CHARMS[@]}; do
     CHARM_CHANNEL[$c]=$ceph_release/edge
 done
 
+for c in ${IAM_CHARMS[@]}; do
+    CHARM_CHANNEL[$c]=latest/edge
+done
+
 CHARM_CHANNEL[pacemaker-remote]=${series}/edge
 CHARM_CHANNEL[microk8s]=1.28/stable

common/ch_channel_map/jammy-caracal

@@ -0,0 +1,8 @@
+# These are charmhub mappings for charms that are outside of the main large
+# groupings like openstack and ceph etc.
+
+# Versions are based on https://docs.openstack.org/charm-guide/latest/project/charm-delivery.html
+
+CHARM_CHANNEL[ovn-central]=24.03/candidate
+CHARM_CHANNEL[ovn-chassis]=24.03/candidate
+CHARM_CHANNEL[ovn-dedicated-chassis]=24.03/candidate

common/charm_lists

@@ -88,3 +88,14 @@ prometheus
 ro
 zookeeper
 )
+
+declare -a IAM_CHARMS=(
+hydra
+identity-platform-login-ui-operator
+kratos
+kratos-external-idp-integrator
+oathkeeper
+postgresql-k8s
+self-signed-certificates
+traefik-k8s
+)

common/generate-bundle.sh

@@ -6,6 +6,7 @@ MOD_DIR=$(realpath $(dirname $0))
 . $MOD_DIR/pipeline/02configure
 . $MOD_DIR/pipeline/03build
 # Ensure no unrendered variables
-out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config --exclude-dir=p| egrep -v '^.*#'`" || exit 0
+out="`grep -r __ $MOD_DIR/b/${MASTER_OPTS[BUNDLE_NAME]} --exclude=config \
+        --exclude-dir=p| egrep -v '^.*#'`" || exit 0
 echo -e "ERROR: there are unrendered variables in your bundle:\n$out"
 exit 1

common/generate_bundle_base

@@ -9,13 +9,17 @@ declare -a overlay_opts=()
 # Catch any changes/additions to master opts
 update_master_opts ${MOD_PASSTHROUGH_OPTS[@]}
 
+type=$(get_cloud_type)
 vip_start=${MASTER_OPTS[VIP_ADDR_START]}
-if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
+if [[ -z $vip_start ]] && [[ -e ~/novarc ]] && [[ $type = openstack  ]]; then
     # prodstack
-    cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd -c cidr -f value 2>/dev/null)
+    cidr=$(source ~/novarc; openstack subnet show subnet_${OS_USERNAME}-psd \
+                            -c cidr -f value 2>/dev/null)
     if [[ -z $cidr ]]; then
         # stsstack
-        cidr=$(source ~/novarc; openstack subnet show ${OS_USERNAME}_admin_subnet -c cidr -f value 2>/dev/null)
+        cidr=$(source ~/novarc; openstack subnet show \
+                                ${OS_USERNAME}_admin_subnet \
+                                -c cidr -f value 2>/dev/null)
         if [[ -n $cidr ]]; then
             vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+).+/\1/g').150.0
         fi
@@ -24,7 +28,9 @@ if [[ -z $vip_start ]] && [[ -e ~/novarc ]]; then
         # last 20 addresses for vips which is prone to collisions but
         # we have no alternative currently.
         net_end=$(awk -F'.' '/HostMax/{print $NF}' <<<$(ipcalc -b $cidr))
-        vip_start=$(echo $cidr| sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end - 19))
+        vip_start=$(echo $cidr|
+                    sed -r 's/([0-9]+\.[0-9]+\.[0-9]+).+/\1/g').$((net_end -
+                    19))
     fi
 fi
 VIP_START_PREFIX=${vip_start%\.*}
@@ -42,6 +48,12 @@ if [ -n "${MASTER_OPTS[MODEL_CONFIG]}" ]; then
     juju model-config ${MASTER_OPTS[MODEL_CONFIG]}
 fi
 
+if [ -n "${MASTER_OPTS[MODEL_CONSTRAINTS]}" ]; then
+    juju set-model-constraints ${MASTER_OPTS[MODEL_CONSTRAINTS]}
+elif ! ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then
+    juju set-model-constraints root-disk-source=volume root-disk=20G
+fi
+
 if has_opt --list; then
     state_root=`get_bundle_state_root`
     if [ -d "$state_root" ]; then
@@ -61,7 +73,8 @@ finish ()
 {
     local target
 
-    echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > ${bundles_dir}/generate-command
+    echo "${MOD_DIR}/generate-bundle.sh ${CACHED_STDIN[@]}" > \
+            ${bundles_dir}/generate-command
     if has_opt --replay; then
         target=${bundles_dir}/command
         echo -e "INFO: replaying last known command (from $target)\n"
@@ -125,7 +138,9 @@ if ((${#MOD_OVERLAYS[@]})); then
             unset subdir
         fi
         cp $MOD_DIR/overlays/$overlay $bundles_dir/o/${subdir:-""}
-        ((${#overlay_opts[@]}==0)) && overlay_opts+=("")  # left padding
+        if ((${#overlay_opts[@]}==0)); then
+            overlay_opts+=("")  # left padding
+        fi
         overlay_opts+=( --overlay $bundles_dir/o/$overlay )
         overlay_list+=( "$bundles_dir/o/$overlay" )
         render $bundles_dir/o/$overlay
@@ -154,54 +169,69 @@ aggregate_mysql_interface_parts $bundles_dir/o
 # generate bundle list
 base_bundle=$bundles_dir/`basename $bundle`
 overlay_list+=( $base_bundle )
-readarray -t application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" ${overlay_list[@]})
+readarray -t \
+    application_list < <("$MOD_DIR/../tools/juju-bundle-applications.py" \
+    ${overlay_list[@]})
 
 # Generate placement overlay for use with MAAS provider
 if ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then
-    cp  $MOD_DIR/overlays/unit_placement/header.yaml.template $bundles_dir/unit-placement.yaml
+    cp $MOD_DIR/overlays/unit_placement/header.yaml.template \
+       $bundles_dir/unit-placement.yaml
     if [[ $MOD_NAME = openstack ]]; then
         # these two represent total machines since e.g. ceph is deployed on compute but compute and gateway are never colocated
-        num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
+        num_placement_machines=$((${MOD_PARAMS[__NUM_NEUTRON_GATEWAY_UNITS__]}+
+                                    ${MOD_PARAMS[__NUM_COMPUTE_UNITS__]}))
     elif [[ $MOD_NAME = kubernetes ]]; then
-        num_placement_machines=$((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
-    elif [[ $MOD_NAME = ceph ]]; then
+        num_placement_machines=\
+            $((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+
+                ${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
+    elif [[ $MOD_NAME = ceph ]] || [[ $MOD_NAME = cos ]]; then
         num_placement_machines=$((${MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]}))
     else
-        echo "ERROR: module '$MOD_NAME' does not yet have support for hyperconverged mode" 1>&2
+        echo -n "ERROR: module '$MOD_NAME'" 1>&2
+        echo " does not yet have support for hyperconverged mode" 1>&2
         exit 1
     fi
     # detect all apps used and generate placement info by doing:
     #  * search for all app names in generated bundles/overlays
     #  * search unit_placement template with same name
     for app in ${application_list[@]}; do
         # filter juju keywords
-        [[ $app == "options" ]] || [[ $app == "to" ]] || [[ $app == "storage" ]] && continue
+        [[ $app =~ ^(options|to|storage)$ ]] && continue
         app_placement=${app}.yaml
         t=$MOD_DIR/overlays/unit_placement/${app_placement}.template
         [ -r "$t" ] || continue
         # load template
         cp $t $PLACEMENT_OVERLAYS_DIR/$app_placement
         # apply all renderers
-        render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
-        render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement $num_placement_machines
+        render_placement_units_lxd $PLACEMENT_OVERLAYS_DIR/$app_placement \
+                                   $num_placement_machines
+        render_placement_units_metal $PLACEMENT_OVERLAYS_DIR/$app_placement \
+                                     $num_placement_machines
         render $PLACEMENT_OVERLAYS_DIR/$app_placement
         # add to master placement overlay
-        cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> $bundles_dir/unit-placement.yaml
+        cat $PLACEMENT_OVERLAYS_DIR/$app_placement >> \
+                $bundles_dir/unit-placement.yaml
     done
     # finally render master machine list
-    render_placement_machines $bundles_dir/unit-placement.yaml $num_placement_machines
+    render_placement_machines $bundles_dir/unit-placement.yaml \
+                              $num_placement_machines
     render $bundles_dir/unit-placement.yaml
     # and add to list of overlays
     overlay_opts+=( --overlay $bundles_dir/unit-placement.yaml )
 
     # add default binding to all applications
     bindings="\    bindings:\n      '': ${MASTER_OPTS[DEFAULT_BINDING]}"
-    find $bundles_dir -name \*.yaml| xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
+    find $bundles_dir -name \*.yaml|\
+        xargs -l sed -i -r "/^\s+charm:.+/a$bindings"
 fi
 
-((${#overlay_opts[@]})) && overlay_opts+=("")  # right padding
+if ((${#overlay_opts[@]})); then
+    overlay_opts+=("")  # right padding
+fi
 
-echo -e "juju deploy${JUJU_DEPLOY_OPTS} ${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
+echo -e "juju deploy${JUJU_DEPLOY_OPTS} \
+    ${base_bundle}${overlay_opts[@]:- }\n " > ${bundles_dir}/command
 finish
 
 for f in $INTERNAL_BUNDLE_CONFIG; do

common/helpers

@@ -5,6 +5,7 @@ declare -A MASTER_OPTS=(
     [DEFAULT_BINDING]=
     [HYPERCONVERGED_DEPLOYMENT]=false
     [MODEL_CONFIG]='test-mode=true'
+    [MODEL_CONSTRAINTS]=''
     [BUNDLE_NAME]=''
     [CLOUD_NAME]=''
     [TARGET_RELEASE_NAME]=''
@@ -42,6 +43,13 @@ list_overlays ()
     } | sort -u
 }
 
+get_cloud_type ()
+{
+   local cloud=`juju show-model| sed -rn 's/.+cloud:\s*(.+).*/\1/p'| uniq`
+   local type=`juju show-cloud $cloud| sed -rn 's/^type:\s*(.+).*/\1/p'| uniq`
+   echo "$type"
+}
+
 _usage () {
     cat << EOF
 USAGE: `basename $0` OPTIONS [OVERLAYS] [MODULE_OPTS]
@@ -181,6 +189,14 @@ filter_master_opts ()
                 fi
                 shift
                 ;;
+            --model-constraints)
+                if [ -z "${MASTER_OPTS[MODEL_CONSTRAINTS]}" ]; then
+                    MASTER_OPTS[MODEL_CONSTRAINTS]="$2"
+                else
+                    MASTER_OPTS[MODEL_CONSTRAINTS]+=" $2"
+                fi
+                shift
+                ;;
             --name|-n)
                 # give bundle set a name and store under named dir
                 MASTER_OPTS[BUNDLE_NAME]="${2%%:*}"
@@ -782,8 +798,7 @@ ensure_model
 # Establish what cloud provider is in use and if it's MAAS, use unit placement.
 # NOTE: this has to be done AFTER the model has been created and we have
 #       switched context.
-cloud=`juju show-model| sed -rn 's/.+cloud:\s*(.+).*/\1/p'| uniq`
-type=`juju show-cloud $cloud| sed -rn 's/^type:\s*(.+).*/\1/p'| uniq`
+type=$(get_cloud_type)
 if [[ $type = maas ]]; then
     echo "INFO: maas provider detected - enabling hyperconverged deployment"
     MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]=true

common/render.d/all

@@ -45,7 +45,8 @@ render_mod_params ()
 
     if ((${#MOD_PARAMS[@]})); then
         for p in ${!MOD_PARAMS[@]}; do
-            echo -n "-e 's,$p,${MOD_PARAMS[$p]},g' " >> $config_renderer
+            # Escape any comma characters in the value, otherwise it breaks
+            echo -n "-e 's,$p,${MOD_PARAMS[$p]//,/\\,},g' " >> $config_renderer
             echo "${p}: \"${MOD_PARAMS[$p]}\"" >> $INTERNAL_BUNDLE_CONFIG
         done
     fi

cos/common

@@ -0,0 +1 @@
+../common