From 759462520e911f4db0beca4f432b9347c00b042a Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 2 Oct 2024 16:31:04 +0100 Subject: [PATCH] More fix to how we identify branch jobs tests/tests.yaml is actually ignored by osci and jobs are in fact determined from a combination of zosci-config and local osci.yaml. --- openstack/tools/func_test_tools/common.py | 32 ++++++++ .../identify_charm_func_test_jobs.py | 82 ++++++------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/openstack/tools/func_test_tools/common.py b/openstack/tools/func_test_tools/common.py index 5ccb4fa8..88a856b8 100644 --- a/openstack/tools/func_test_tools/common.py +++ b/openstack/tools/func_test_tools/common.py @@ -1,9 +1,41 @@ """ Common helpers for func test runners. """ +from functools import cached_property import os import yaml +class ZOSCIConfig(): + """ Extract information from zosci-config """ + def __init__(self, path): + self.path = path + + @cached_property + def project_templates(self): + """ + Generator returning each project-template defined. + """ + with open(os.path.join(self.path, 'zuul.d/project-templates.yaml'), + encoding='utf-8') as fd: + yield from yaml.safe_load(fd) + + def get_branch_jobs(self, branch): + """ + For a given branch name, find all jobs that need to be run against that + branch. + """ + test_jobs = [] + for t in self.project_templates: + t = t['project-template'] + if t['name'] == 'charm-functional-jobs': + for jobs in t['check']['jobs']: + for job, info in jobs.items(): + if branch in info['branches']: + test_jobs.append(job) + + return test_jobs + + class OSCIConfig(): """ Extract information from osci.yaml """ def __init__(self): diff --git a/openstack/tools/func_test_tools/identify_charm_func_test_jobs.py b/openstack/tools/func_test_tools/identify_charm_func_test_jobs.py index 31a0ee5f..7fecab06 100644 --- a/openstack/tools/func_test_tools/identify_charm_func_test_jobs.py +++ b/openstack/tools/func_test_tools/identify_charm_func_test_jobs.py @@ -4,59 +4,33 @@ Outputs space separated list of job names. """ +import configparser import os -import yaml -from common import OSCIConfig # pylint: disable=import-error +from common import ZOSCIConfig, OSCIConfig # pylint: disable=import-error -CLASSIC_TESTS_YAML = 'tests/tests.yaml' -REACTIVE_TESTS_YAML = os.path.join('src', CLASSIC_TESTS_YAML) - -def extract_targets(bundle_list): - """ - Targets are provided as strings or dicts where the target name is the - value so this accounts for both formats. +def get_local_jobs_and_deps(jobs): """ - extracted = [] - for item in bundle_list or []: - if isinstance(item, dict): - values = list(item.values()) - for value in values: - if isinstance(value, list): - # its a list of overlays so we get the key name and go find - # the corresponding job in osci.yaml - extracted.append(list(item.keys())[0]) - else: - # its a bundle name - extracted.append(value) - else: - extracted.append(item) - - return extracted + Get any locally defined jobs and add them to the list of jobs provided. - -def get_job_deps(bundles): - """ - Extract aliased targets. A charm can define aliased targets which is where - Zaza tests are run and use configuration steps from an alias section rather - than the default (see 'configure:' section in tests.yaml for aliases). An - alias is run by specifying the target to be run as a tox command using a - job definition in osci.yaml where the target name has a : prefix. - - We extract any aliased targets here and return as a list. - - @param bundles: list of extracted bundles + @param jobs: list of already identified jobs. """ deps = [] + local_jobs = [] osci = OSCIConfig() project_check_jobs = list(osci.project_check_jobs) - jobs = project_check_jobs + bundles - for jobname in jobs: + all_jobs = project_check_jobs + jobs + for jobname in all_jobs: + if isinstance(jobname, dict): + jobname = list(jobname.keys())[0] + job = osci.get_job(jobname) if not job: continue + local_jobs.append(jobname) + # Some jobs will depend on other tests that need to be run but # are not defined in tests.yaml so we need to add them from # here as well. @@ -64,28 +38,22 @@ def get_job_deps(bundles): if name in project_check_jobs: deps.append(name) - return deps + bundles + return deps + jobs + local_jobs -def get_tests_bundles(): +def get_default_jobs(): """ - Extract test targets from primary location i.e. {src/}test/tests.yaml. + Get all jobs we need to run by default for the given branch. """ - if os.path.exists(REACTIVE_TESTS_YAML): - tests_file = REACTIVE_TESTS_YAML - else: - tests_file = CLASSIC_TESTS_YAML - - with open(tests_file, encoding='utf-8') as fd: - bundles = yaml.safe_load(fd) - - smoke_bundles = extract_targets(bundles['smoke_bundles']) - gate_bundles = extract_targets(bundles['gate_bundles']) - dev_bundles = extract_targets(bundles['dev_bundles']) - return smoke_bundles + gate_bundles + dev_bundles + path = os.path.join(os.environ['HOME'], 'zosci-config') + c = configparser.ConfigParser() + c.read('.gitreview') + branch = c['gerrit']['defaultbranch'] + jobs = ZOSCIConfig(path).get_branch_jobs(branch) + return jobs if __name__ == "__main__": - _bundles = get_tests_bundles() - _bundles = get_job_deps(list(set(_bundles))) - print(' '.join(sorted(set(_bundles)))) + _jobs = get_default_jobs() + _jobs = get_local_jobs_and_deps(list(set(_jobs))) + print(' '.join(sorted(set(_jobs))))