Merge pull request #259 from dosaboy/func-tests-fix-target-name-handling

Fix func test target name identification

openstack/tools/charmed_openstack_functest_runner.sh

@@ -192,13 +192,14 @@ if [[ -n $FUNC_TEST_PR ]]; then
     )
 fi
 
-declare -A func_targets=()
+declare -A func_target_state=()
+declare -a func_target_order
 if [[ -n $FUNC_TEST_TARGET ]]; then
-    func_targets[$FUNC_TEST_TARGET]=null
+    func_target_state[$FUNC_TEST_TARGET]=null
 else
     voting_targets=()
     non_voting_targets=()
-    for target in $(python3 $TOOLS_PATH/identify_charm_func_tests.py); do
+    for target in $(python3 $TOOLS_PATH/identify_charm_func_test_jobs.py); do
         if $(python3 $TOOLS_PATH/test_is_voting.py $target); then
             voting_targets+=( $target )
         else
@@ -207,7 +208,8 @@ else
     done
     # Ensure voting targets processed first.
     for target in ${voting_targets[@]} ${non_voting_targets[@]}; do
-        func_targets[$target]=null
+        func_target_order+=( $target )
+        func_target_state[$target]=null
     done
 fi
 
@@ -231,7 +233,7 @@ fi
 
 first=true
 init_noop_target=true
-for target in ${!func_targets[@]}; do
+for target in ${func_target_order[@]}; do
     # Destroy any existing zaza models to ensure we have all the resources we
     # need.
     destroy_zaza_models
@@ -245,21 +247,20 @@ for target in ${!func_targets[@]}; do
     fi
     [[ -d src ]] && pushd src &>/dev/null || true
     fail=false
-    # Remove substitutions and replace with whitespace
-    target=${target//+/ }
+    _target="$(python3 $TOOLS_PATH/extract_job_target.py $target)"
     if ! $MANUAL_FUNCTESTS; then
-        tox ${tox_args} -- $target || fail=true
+        tox ${tox_args} -- $_target || fail=true
         model=$(juju list-models| egrep -o "^zaza-\S+"|tr -d '*')
     else
-        $TOOLS_PATH/manual_functests_runner.sh $target $SLEEP $init_noop_target || fail=true
+        $TOOLS_PATH/manual_functests_runner.sh "$_target" $SLEEP $init_noop_target || fail=true
         model=test-$target
         init_noop_target=false
     fi
 
     if $fail; then
-        func_targets[$target]='fail'
+        func_target_state[$target]='fail'
     else
-        func_targets[$target]='success'
+        func_target_state[$target]='success'
     fi
 
     if $WAIT_ON_DESTROY; then
@@ -273,16 +274,16 @@ popd &>/dev/null || true
 
 # Report results
 echo -e "\nTest results for charm $CHARM_NAME functional tests @ commit $COMMIT_ID:"
-for target in ${!func_targets[@]}; do
+for target in ${func_target_order[@]}; do
     if $(python3 $TOOLS_PATH/test_is_voting.py $target); then
         voting_info=""
     else
         voting_info=" (non-voting)"
     fi
 
-    if [[ ${func_targets[$target]} = null ]]; then
+    if [[ ${func_target_state[$target]} = null ]]; then
         echo "  * $target: SKIPPED$voting_info"
-    elif [[ ${func_targets[$target]} = success ]]; then
+    elif [[ ${func_target_state[$target]} = success ]]; then
         echo "  * $target: SUCCESS$voting_info"
     else
         echo "  * $target: FAILURE$voting_info"

openstack/tools/func_test_tools/common.py

@@ -23,12 +23,7 @@ def project_check_jobs(self):
             if 'check' not in item['project']:
                 continue
 
-            for job in item['project']['check'].get('jobs', []):
-                # can be a dict with voting info
-                if isinstance(job, dict):
-                    yield list(job.keys())[0]
-                else:
-                    yield job
+            yield from item['project']['check'].get('jobs', [])
 
     @property
     def jobs(self):
@@ -37,6 +32,17 @@ def jobs(self):
             if 'job' in item:
                 yield item['job']
 
+    def get_job(self, name):
+        """ Get job by name.
+
+        @param name: string name
+        """
+        for job in self.jobs:
+            if job['name'] == name:
+                return job
+
+        return None
+
 
 class ProjectTemplatesConfig():
     """ Extract information from project_templates.yaml """

openstack/tools/func_test_tools/extract_job_target.py

@@ -0,0 +1,33 @@
+"""
+If a job has an accompanying vars section that specifies a tox command with
+target names we need to run those instead of the job name.
+"""
+import re
+import sys
+
+from common import OSCIConfig  # pylint: disable=import-error
+
+
+def extract_job_target(testjob):
+    """
+    Some jobs map directly to target names and some needs to be de-refenced by
+    looking for the job definition and extracting the target from the tox
+    command. Returns jobname if no dereference available.
+
+    @param job: job name
+    """
+    osci = OSCIConfig()
+    job = osci.get_job(testjob)
+    if not job or 'vars' not in job or 'tox_extra_args' not in job['vars']:
+        return testjob
+
+    ret = re.search(r"-- (.+)",
+                    str(job['vars']['tox_extra_args']))
+    if not ret:
+        return testjob
+
+    return ret.group(1)
+
+
+if __name__ == "__main__":
+    print(extract_job_target(sys.argv[1]))

openstack/tools/func_test_tools/identify_charm_func_tests.py → openstack/tools/func_test_tools/identify_charm_func_test_jobs.py

@@ -1,11 +1,10 @@
 """
-Get names of test targets that OSCI would run for the given charm. Should be
+Get names of test jobs that OSCI would run for the given charm. Should be
 run from within the charm root.
 
-Outputs space separated list of target names.
+Outputs space separated list of job names.
 """
 import os
-import re
 
 import yaml
 from common import OSCIConfig  # pylint: disable=import-error
@@ -37,7 +36,7 @@ def extract_targets(bundle_list):
     return extracted
 
 
-def get_aliased_targets(bundles):
+def get_job_deps(bundles):
     """
     Extract aliased targets. A charm can define aliased targets which is where
     Zaza tests are run and use configuration steps from an alias section rather
@@ -49,41 +48,23 @@ def get_aliased_targets(bundles):
 
     @param bundles: list of extracted bundles
     """
-    targets = []
+    deps = []
     osci = OSCIConfig()
     project_check_jobs = list(osci.project_check_jobs)
     jobs = project_check_jobs + bundles
     for jobname in jobs:
-        for job in osci.jobs:
-            if job['name'] != jobname:
-                continue
-
-            if 'tox_extra_args' not in job['vars']:
-                continue
-
-            ret = re.search(r"-- (.+)",
-                            str(job['vars']['tox_extra_args']))
-            if ret:
-                target = ret.group(1)
-                # NOTE: will need to reverse this when we use the target name
-                target = target.replace(' ', '+')
-                targets.append(target)
-                for _target in target.split():
-                    name = _target.partition(':')[2]
-                    if name in bundles:
-                        bundles.remove(name)
-
-                if jobname in bundles:
-                    bundles.remove(jobname)
-
-                # Some jobs will depend on other tests that need to be run but
-                # are not defined in tests.yaml so we need to add them from
-                # here as well.
-                for name in job.get('dependencies', []):
-                    if name in project_check_jobs:
-                        bundles.append(name)
-
-    return targets + bundles
+        job = osci.get_job(jobname)
+        if not job:
+            continue
+
+        # Some jobs will depend on other tests that need to be run but
+        # are not defined in tests.yaml so we need to add them from
+        # here as well.
+        for name in job.get('dependencies', []):
+            if name in project_check_jobs:
+                deps.append(name)
+
+    return deps + bundles
 
 
 def get_tests_bundles():
@@ -106,5 +87,5 @@ def get_tests_bundles():
 
 if __name__ == "__main__":
     _bundles = get_tests_bundles()
-    _bundles = get_aliased_targets(list(set(_bundles)))
+    _bundles = get_job_deps(list(set(_bundles)))
     print(' '.join(sorted(set(_bundles))))

openstack/tools/func_test_tools/test_is_voting.py

@@ -14,7 +14,7 @@
 
 
 if __name__ == "__main__":
-    target_name = sys.argv[1]
+    test_job = sys.argv[1]
     zosci_path = os.path.join(os.environ['HOME'], "zosci-config")
     project_templates = os.path.join(zosci_path,
                                      "zuul.d/project-templates.yaml")
@@ -25,19 +25,17 @@
         osci_config = OSCIConfig()
         try:
             jobs = osci_config.project_check_jobs
-            if target_name in jobs:
-                # default is voting=True
-                sys.exit(0)
-
-            for check in jobs:
-                if isinstance(check, dict) and target_name in check:
-                    if not check[target_name]['voting']:
+            for job in jobs:
+                if isinstance(job, dict) and test_job in job:
+                    if not job[test_job]['voting']:
                         sys.exit(1)
                     else:
+                        # default is true
                         sys.exit(0)
+
         except KeyError as exc:
             sys.stderr.write(f"ERROR: failed to process osci.yaml - assuming "
-                             f"{target_name} is voting (key {exc} not found)."
+                             f"{test_job} is voting (key {exc} not found)."
                              "\n")
 
     # If the target was not found in osci.yaml then osci will fallback to
@@ -47,6 +45,6 @@
         for project in config.project_templates:
             if project['name'] == "charm-functional-jobs":
                 for job in project['check']['jobs']:
-                    if target_name in job:
-                        if not job[target_name].get('voting', True):
+                    if test_job in job:
+                        if not job[test_job].get('voting', True):
                             sys.exit(1)