From 5f99700b6ea0da15aa63d7c15823e8ea7ff97713 Mon Sep 17 00:00:00 2001 From: Fred Tingaud Date: Thu, 12 Dec 2024 17:55:27 +0100 Subject: [PATCH 1/8] RULEAPI-816 Create a rule / product mapping file --- rspec-tools/rspec_tools/cli.py | 10 +- rspec-tools/rspec_tools/coverage.py | 136 +++++++++++++++++- .../rspec_tools/repo_plugin_mapping.json | 119 +++++++++++++++ 3 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 rspec-tools/rspec_tools/repo_plugin_mapping.json diff --git a/rspec-tools/rspec_tools/cli.py b/rspec-tools/rspec_tools/cli.py index 71abb95b6d4..eab59584dd6 100644 --- a/rspec-tools/rspec_tools/cli.py +++ b/rspec-tools/rspec_tools/cli.py @@ -7,9 +7,12 @@ import rspec_tools.create_rule import rspec_tools.modify_rule from rspec_tools.checklinks import check_html_links -from rspec_tools.coverage import (update_coverage_for_all_repos, - update_coverage_for_repo, - update_coverage_for_repo_version) +from rspec_tools.coverage import ( + collect_coverage_per_product, + update_coverage_for_all_repos, + update_coverage_for_repo, + update_coverage_for_repo_version, +) from rspec_tools.errors import RuleValidationError from rspec_tools.notify_failure_on_slack import notify_slack from rspec_tools.rules import LanguageSpecificRule, RulesRepository @@ -147,6 +150,7 @@ def update_coverage(rulesdir: str, repository: Optional[str], version: Optional[ update_coverage_for_repo(repository, Path(rulesdir)) else: update_coverage_for_repo_version(repository, version, Path(rulesdir)) + collect_coverage_per_product(Path(rulesdir)) @cli.command() @click.option('--message', required=True) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 5728251e7f8..1e768e3c310 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -49,6 +49,12 @@ RULES_FILENAME = 'covered_rules.json' +DEPENDENCY_RE = re.compile(r'\s*dependency\s+\'(com|org)\.sonarsource\.([\w-]+):([\w-]+):(\d+(\.\d+)+)\'') + +BUNDLED_SIMPLE = r'[\'"](com|org)\.sonarsource\.([\w.-]+):([\w-]+)[\'"]' +BUNDLED_MULTI = r'\(\s*group:\s*[\'"]([\w.-]+)[\'"],\s*name:\s*[\'"]([\w-]+)[\'"],\s*classifier:\s*[\'"][\w-]+\'\s*\)' +BUNDLED_RE = re.compile(rf'\s*bundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') + def get_rule_id(filename): rule_id = filename[:-5] @@ -197,9 +203,13 @@ def is_version_tag(name): def comparable_version(key): - if not is_version_tag(key): - return [0] - return list(map(int, key.split('.'))) + v = key.removeprefix('sqcb-').removeprefix('sqs-') + if not is_version_tag(v): + if v == 'master': + return [0] + else: + sys.exit(f'Unexpected version {key}') + return list(map(int, v.split('.'))) def collect_coverage_for_all_versions(repo, coverage): @@ -246,3 +256,123 @@ def update_coverage_for_repo_version(repo, version, rules_dir): collect_coverage_for_version(repo, git_repo, version, coverage) coverage.save_to_file(RULES_FILENAME) + +def get_plugin_versions(git_repo, version): + g = Git(git_repo) + repo_dir = git_repo.working_tree_dir + try: + with pushd(repo_dir): + content = g.show(f'{version}:build.gradle') + versions = {} + for m in re.findall(DEPENDENCY_RE, content): + versions[m[2]] = m[3] + return versions + except Exception as e: + print(f"Sonar Enterprise {version} checkout failed: {e}") + raise + +def get_packaged_plugins(git_repo): + g = Git(git_repo) + repo_dir = git_repo.working_tree_dir + with pushd(repo_dir): + BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle', + 'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle', + 'Developer': 'private/edition-developer/bundled_plugins.gradle', + 'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'} + bundle_map = {} + for key, bundle in BUNDLES.items(): + bundle_map[key] = [] + content = g.show(f'master:{bundle}') + for m in re.findall(BUNDLED_RE, content): + if m[3] != '': + bundle_map[key].append(m[3]) + else: + bundle_map[key].append(m[5]) + return bundle_map + +def lowest_cb(plugin_versions, plugin, version): + tags = list(filter(lambda k: not k.startswith('sqs-'), plugin_versions.keys())) + tags.sort(key = comparable_version) + for t in tags: + if plugin in plugin_versions[t]: + pvv = plugin_versions[t][plugin] + if comparable_version(pvv) >= comparable_version(version): + return t + return None + + +def lowest_server(plugin_versions, plugin, version): + tags = list(filter(lambda k: not k.startswith('sqcb-'), plugin_versions.keys())) + tags.sort(key = comparable_version) + for t in tags: + if plugin in plugin_versions[t]: + pvv = plugin_versions[t][plugin] + if comparable_version(pvv) >= comparable_version(version): + return t + return None + + +def build_rule_per_product(rules_dir, bundle_map, plugin_versions): + coverage = Coverage(RULES_FILENAME, rules_dir) + rule_per_product = {} + repo_plugin_mapping = load_json(os.path.join(Path(__file__).parent, 'repo_plugin_mapping.json')) + for lang, rules in coverage.rules.items(): + for rule, version in rules.items(): + if isinstance(version, str): + if rule not in rule_per_product: + rule_per_product[rule] = {} + if lang not in rule_per_product[rule]: + rule_per_product[rule][lang] = {} + target_repo, v = version.split(' ') + if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: + print(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}") + continue + plugin = repo_plugin_mapping[lang][target_repo] + if plugin in bundle_map['Community Build']: + rule_per_product[rule][lang]['SonarQube Community Build'] = lowest_cb(plugin_versions, plugin, v) + rule_per_product[rule][lang]['SonarQube Server'] = { + 'Developer': lowest_server(plugin_versions, plugin, v) + } + elif plugin in bundle_map['Developer']: + rule_per_product[rule][lang]['SonarQube Server'] = { + 'Developer': lowest_server(plugin_versions, plugin, v)} + elif plugin in bundle_map['Enterprise']: + rule_per_product[rule][lang]['SonarQube Server'] = { + 'Enterprise': lowest_server(plugin_versions, plugin, v)} + elif plugin in bundle_map['Datacenter']: + rule_per_product[rule][lang]['SonarQube Server'] = { + 'Datacenter': lowest_server(plugin_versions, plugin, v) + } + else: + print(f'Couldnt find plugin {plugin}') + with open('rule_product_mapping.json', 'w', encoding='utf-8') as outfile: + json.dump(rule_per_product, outfile, indent=2, sort_keys=True) + + +def is_interesting_version(version): + if version.startswith('sqs-'): + # Sonarqube Server Release + return True + if version.startswith('sqcb-'): + # Sonarqube Community Build Release + return True + if not is_version_tag(version): + # Non official version + return False + try: + # Official release before Dec 2024 + major = int(version[:version.find('.')]) + except ValueError: + return False + return major >= 8 + +def collect_coverage_per_product(rules_dir): + git_repo = checkout_repo('sonar-enterprise') + bundle_map = get_packaged_plugins(git_repo) + tags = git_repo.tags + tags.sort(key = lambda t: t.commit.committed_date) + versions = [tag.name for tag in tags if is_interesting_version(tag.name)] + plugin_versions = {} + for version in versions: + plugin_versions[version] = get_plugin_versions(git_repo, version) + build_rule_per_product(rules_dir, bundle_map, plugin_versions) diff --git a/rspec-tools/rspec_tools/repo_plugin_mapping.json b/rspec-tools/rspec_tools/repo_plugin_mapping.json new file mode 100644 index 00000000000..c94e01d4f2d --- /dev/null +++ b/rspec-tools/rspec_tools/repo_plugin_mapping.json @@ -0,0 +1,119 @@ +{ + "ABAP": { + "sonar-abap": "sonar-abap-plugin" + }, + "ANSIBLE": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "APEX": { + "sonar-apex": "sonar-apex-plugin" + }, + "AZURE_RESOURCE_MANAGER": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "C": { + "sonar-cpp": "sonar-cfamily-plugin" + }, + "CLOUDFORMATION": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "COBOL": { + "sonar-cobol": "sonar-cobol-plugin" + }, + "CPP": { + "sonar-cpp": "sonar-cfamily-plugin" + }, + "CSH": { + "sonar-dotnet-enterprise": "sonar-csharp-plugin", + "sonar-security": "sonar-security-plugin" + }, + "CSS": { + "SonarJS": "sonar-javascript-plugin" + }, + "DART": { + "sonar-dart": "sonar-dart-plugin" + }, + "DOCKER": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "FLEX": { + "sonar-flex": "sonar-flex-plugin" + }, + "GO": { + "sonar-go": "sonar-go-plugin" + }, + "HTML": { + "sonar-html": "sonar-html-plugin" + }, + "JAVA": { + "sonar-architecture": "sonar-architecture-plugin", + "sonar-dataflow-bug-detection": "sonar-dbd-java-frontend-plugin", + "sonar-java": "sonar-java-plugin", + "sonar-security": "sonar-security-plugin" + }, + "JAVASCRIPT": { + "SonarJS": "sonar-javascript-plugin", + "sonar-security": "sonar-security-plugin" + }, + "KOTLIN": { + "sonar-kotlin": "sonar-kotlin-plugin" + }, + "KUBERNETES": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "OBJC": { + "sonar-cpp": "sonar-cfamily-plugin" + }, + "PHP": { + "sonar-php": "sonar-php-plugin", + "sonar-security": "sonar-security-plugin" + }, + "PLI": { + "sonar-pli": "sonar-pli-plugin" + }, + "PLSQL": { + "sonar-plsql": "sonar-plsql-plugin" + }, + "PY": { + "sonar-dataflow-bug-detection": "sonar-dbd-python-frontend-plugin", + "sonar-python": "sonar-python-plugin", + "sonar-security": "sonar-security-plugin" + }, + "RPG": { + "sonar-rpg": "sonar-rpg-plugin" + }, + "RUBY": { + "sonar-ruby": "sonar-ruby-plugin" + }, + "SCALA": { + "sonar-scala": "sonar-scala-plugin" + }, + "SECRETS": { + "sonar-text-enterprise": "sonar-text-enterprise-plugin" + }, + "SWIFT": { + "sonar-swift": "sonar-swift-plugin" + }, + "TERRAFORM": { + "sonar-iac-enterprise": "sonar-iac-enterprise-plugin" + }, + "TEXT": { + "sonar-text-enterprise": "sonar-text-enterprise-plugin" + }, + "TSQL": { + "sonar-tsql": "sonar-tsql-plugin" + }, + "TYPESCRIPT": { + "SonarJS": "sonar-javascript-plugin", + "sonar-security": "sonar-security-plugin" + }, + "VB": { + "sonar-vb": "sonar-vb-plugin" + }, + "VBNET": { + "sonar-dotnet-enterprise": "sonar-vbnet-enterprise-plugin" + }, + "XML": { + "sonar-xml": "sonar-xml-plugin" + } +} \ No newline at end of file From 755ffff5957390d360baf4eb3eac957be1d92582 Mon Sep 17 00:00:00 2001 From: Fred Tingaud Date: Fri, 13 Dec 2024 18:37:29 +0100 Subject: [PATCH 2/8] Apply suggestions --- rspec-tools/rspec_tools/cli.py | 3 +- rspec-tools/rspec_tools/coverage.py | 147 ++++++++++++++-------------- rspec-tools/rspec_tools/utils.py | 17 ++-- 3 files changed, 83 insertions(+), 84 deletions(-) diff --git a/rspec-tools/rspec_tools/cli.py b/rspec-tools/rspec_tools/cli.py index eab59584dd6..0fc1a0a1178 100644 --- a/rspec-tools/rspec_tools/cli.py +++ b/rspec-tools/rspec_tools/cli.py @@ -3,7 +3,6 @@ from typing import Optional import click - import rspec_tools.create_rule import rspec_tools.modify_rule from rspec_tools.checklinks import check_html_links @@ -150,7 +149,7 @@ def update_coverage(rulesdir: str, repository: Optional[str], version: Optional[ update_coverage_for_repo(repository, Path(rulesdir)) else: update_coverage_for_repo_version(repository, version, Path(rulesdir)) - collect_coverage_per_product(Path(rulesdir)) + collect_coverage_per_product() @cli.command() @click.option('--message', required=True) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 1e768e3c310..04e9aced31b 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -1,12 +1,12 @@ import collections -import json import os import re import sys +from collections import defaultdict from pathlib import Path from git import Git, Repo -from rspec_tools.utils import load_json, pushd +from rspec_tools.utils import load_json, pushd, save_json REPOS = [ 'sonar-abap', @@ -49,11 +49,11 @@ RULES_FILENAME = 'covered_rules.json' -DEPENDENCY_RE = re.compile(r'\s*dependency\s+\'(com|org)\.sonarsource\.([\w-]+):([\w-]+):(\d+(\.\d+)+)\'') +DEPENDENCY_RE = re.compile(r'''\bdependency\s+['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+):(?P\d+(\.\d+)+)['"]''') -BUNDLED_SIMPLE = r'[\'"](com|org)\.sonarsource\.([\w.-]+):([\w-]+)[\'"]' -BUNDLED_MULTI = r'\(\s*group:\s*[\'"]([\w.-]+)[\'"],\s*name:\s*[\'"]([\w-]+)[\'"],\s*classifier:\s*[\'"][\w-]+\'\s*\)' -BUNDLED_RE = re.compile(rf'\s*bundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') +BUNDLED_SIMPLE = r'''['"](?:com|org)\.sonarsource\.(?:[\w.-]+):(?P[\w-]+)['"]''' +BUNDLED_MULTI = r'''\(\s*group:\s*['"]([\w.-]+)['"],\s*name:\s*['"](?P[\w-]+)['"],\s*classifier:\s*['"][\w-]+['"]\s*\)''' +BUNDLED_RE = re.compile(rf'\bbundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') def get_rule_id(filename): @@ -126,8 +126,7 @@ def __init__(self, filename, rules_dir): self.rules = load_json(filename) def save_to_file(self, filename): - with open(filename, 'w') as outfile: - json.dump(self.rules, outfile, indent=2, sort_keys=True) + save_json(self.rules, filename) def _rule_implemented_for_intermediate_version(self, rule_id, language, repo_and_version): if rule_id not in self.rules[language]: @@ -260,93 +259,89 @@ def update_coverage_for_repo_version(repo, version, rules_dir): def get_plugin_versions(git_repo, version): g = Git(git_repo) repo_dir = git_repo.working_tree_dir - try: - with pushd(repo_dir): - content = g.show(f'{version}:build.gradle') - versions = {} - for m in re.findall(DEPENDENCY_RE, content): - versions[m[2]] = m[3] - return versions - except Exception as e: - print(f"Sonar Enterprise {version} checkout failed: {e}") - raise + with pushd(repo_dir): + content = g.show(f'{version}:build.gradle') + versions = {} + for m in re.finditer(DEPENDENCY_RE, content): + versions[m.groupdict()['plugin_name']] = m.groupdict()['version'] + return versions + + +BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle', + 'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle', + 'Developer': 'private/edition-developer/bundled_plugins.gradle', + 'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'} + def get_packaged_plugins(git_repo): g = Git(git_repo) repo_dir = git_repo.working_tree_dir with pushd(repo_dir): - BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle', - 'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle', - 'Developer': 'private/edition-developer/bundled_plugins.gradle', - 'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'} bundle_map = {} for key, bundle in BUNDLES.items(): bundle_map[key] = [] content = g.show(f'master:{bundle}') - for m in re.findall(BUNDLED_RE, content): - if m[3] != '': - bundle_map[key].append(m[3]) + for m in re.finditer(BUNDLED_RE, content): + if m.groupdict()['plugin_name'] != None: + bundle_map[key].append(m.groupdict()['plugin_name']) else: - bundle_map[key].append(m[5]) + bundle_map[key].append(m.groupdict()['plugin_name2']) return bundle_map -def lowest_cb(plugin_versions, plugin, version): - tags = list(filter(lambda k: not k.startswith('sqs-'), plugin_versions.keys())) + +def lowest_version(plugin_versions, plugin, version, skip_suffix): + tags = list(filter(lambda k: not k.startswith(skip_suffix), plugin_versions.keys())) tags.sort(key = comparable_version) for t in tags: if plugin in plugin_versions[t]: pvv = plugin_versions[t][plugin] if comparable_version(pvv) >= comparable_version(version): return t - return None + sys.exit(f'failed finding the oldest version of {plugin} containing {version} from {plugin_versions}') -def lowest_server(plugin_versions, plugin, version): - tags = list(filter(lambda k: not k.startswith('sqcb-'), plugin_versions.keys())) - tags.sort(key = comparable_version) - for t in tags: - if plugin in plugin_versions[t]: - pvv = plugin_versions[t][plugin] - if comparable_version(pvv) >= comparable_version(version): - return t - return None +def lowest_community_build_version(plugin_versions, plugin, version): + return lowest_version(plugin_versions, plugin, version, 'sqs-') -def build_rule_per_product(rules_dir, bundle_map, plugin_versions): - coverage = Coverage(RULES_FILENAME, rules_dir) - rule_per_product = {} - repo_plugin_mapping = load_json(os.path.join(Path(__file__).parent, 'repo_plugin_mapping.json')) - for lang, rules in coverage.rules.items(): - for rule, version in rules.items(): - if isinstance(version, str): - if rule not in rule_per_product: - rule_per_product[rule] = {} - if lang not in rule_per_product[rule]: - rule_per_product[rule][lang] = {} - target_repo, v = version.split(' ') - if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: - print(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}") - continue - plugin = repo_plugin_mapping[lang][target_repo] - if plugin in bundle_map['Community Build']: - rule_per_product[rule][lang]['SonarQube Community Build'] = lowest_cb(plugin_versions, plugin, v) - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Developer': lowest_server(plugin_versions, plugin, v) - } - elif plugin in bundle_map['Developer']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Developer': lowest_server(plugin_versions, plugin, v)} - elif plugin in bundle_map['Enterprise']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Enterprise': lowest_server(plugin_versions, plugin, v)} - elif plugin in bundle_map['Datacenter']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Datacenter': lowest_server(plugin_versions, plugin, v) - } - else: - print(f'Couldnt find plugin {plugin}') - with open('rule_product_mapping.json', 'w', encoding='utf-8') as outfile: - json.dump(rule_per_product, outfile, indent=2, sort_keys=True) +def lowest_server_version(plugin_versions, plugin, version): + return lowest_version(plugin_versions, plugin, version, 'sqcb-') + + +EDITIONS =['Developer', 'Enterprise', 'Datacenter'] + + +def fill_product_mapping(plugin: str, bundle_map: dict, version: str, plugin_versions: dict, product_per_rule_per_lang: dict): + if plugin in bundle_map['Community Build']: + product_per_rule_per_lang['SonarQube Community Build'] = lowest_community_build_version(plugin_versions, plugin, version) + product_per_rule_per_lang['SonarQube Server'] = { + 'minimal-edition': EDITIONS[0], + 'since-version': lowest_server_version(plugin_versions, plugin, version) + } + return + for edition in EDITIONS: + if plugin in bundle_map[edition]: + product_per_rule_per_lang['SonarQube Server'] = { + 'minimal-edition': edition, + 'since-version': lowest_server_version(plugin_versions, plugin, version) + } + return + sys.exit(f'Couldnt find plugin {plugin}') + + +def build_rule_per_product(bundle_map, plugin_versions): + rules_coverage = load_json(RULES_FILENAME) + rule_per_product = defaultdict(lambda: defaultdict(lambda: {})) + repo_plugin_mapping = load_json(Path(__file__).parent / 'repo_plugin_mapping.json') + for lang, rules in rules_coverage.items(): + for rule, since in rules.items(): + if not isinstance(since, str): + continue + target_repo, version = since.split(' ') + if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: + sys.exit(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}") + fill_product_mapping(repo_plugin_mapping[lang][target_repo], bundle_map, version, plugin_versions, rule_per_product[rule][lang]) + save_json(rule_per_product, 'rule_product_mapping.json') def is_interesting_version(version): @@ -366,13 +361,13 @@ def is_interesting_version(version): return False return major >= 8 -def collect_coverage_per_product(rules_dir): +def collect_coverage_per_product(): git_repo = checkout_repo('sonar-enterprise') bundle_map = get_packaged_plugins(git_repo) tags = git_repo.tags - tags.sort(key = lambda t: t.commit.committed_date) versions = [tag.name for tag in tags if is_interesting_version(tag.name)] + versions.sort(key = comparable_version) plugin_versions = {} for version in versions: plugin_versions[version] = get_plugin_versions(git_repo, version) - build_rule_per_product(rules_dir, bundle_map, plugin_versions) + build_rule_per_product(bundle_map, plugin_versions) diff --git a/rspec-tools/rspec_tools/utils.py b/rspec-tools/rspec_tools/utils.py index a80caac0db9..f24f037ecf3 100644 --- a/rspec-tools/rspec_tools/utils.py +++ b/rspec-tools/rspec_tools/utils.py @@ -1,11 +1,12 @@ -from pathlib import Path -from typing import List -from contextlib import contextmanager -import shutil -import re -import tempfile import json import os +import re +import shutil +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import List + from rspec_tools.errors import InvalidArgumentError SUPPORTED_LANGUAGES_FILENAME = '../supported_languages.adoc' @@ -162,6 +163,10 @@ def load_json(file): with open(file, encoding='utf8') as json_file: return json.load(json_file) +def save_json(content, filename): + with open(filename, 'w', encoding='utf8') as outfile: + json.dump(content, outfile, indent=2, sort_keys=True) + @contextmanager def pushd(new_dir): previous_dir = os.getcwd() From 1abfb8f923221fbe286e84934d46291abfbd9aff Mon Sep 17 00:00:00 2001 From: Marco Borgeaud Date: Mon, 16 Dec 2024 11:58:56 +0100 Subject: [PATCH 3/8] Update rspec-tools/rspec_tools/coverage.py --- rspec-tools/rspec_tools/coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 04e9aced31b..ce7bcd803db 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -336,6 +336,8 @@ def build_rule_per_product(bundle_map, plugin_versions): for lang, rules in rules_coverage.items(): for rule, since in rules.items(): if not isinstance(since, str): + # The rule has an "until", therefore it does not exist anymore + # and should not appear in the product mapping. continue target_repo, version = since.split(' ') if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: From e056c0f888e7d15e985176be779fc6fd0add5293 Mon Sep 17 00:00:00 2001 From: Marco Borgeaud Date: Mon, 16 Dec 2024 13:11:22 +0100 Subject: [PATCH 4/8] Simplify regex: drop unnecessary groups --- rspec-tools/rspec_tools/coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index ce7bcd803db..4c4d1ff5eaf 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -51,8 +51,8 @@ DEPENDENCY_RE = re.compile(r'''\bdependency\s+['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+):(?P\d+(\.\d+)+)['"]''') -BUNDLED_SIMPLE = r'''['"](?:com|org)\.sonarsource\.(?:[\w.-]+):(?P[\w-]+)['"]''' -BUNDLED_MULTI = r'''\(\s*group:\s*['"]([\w.-]+)['"],\s*name:\s*['"](?P[\w-]+)['"],\s*classifier:\s*['"][\w-]+['"]\s*\)''' +BUNDLED_SIMPLE = r'''['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+)['"]''' +BUNDLED_MULTI = r'''\(\s*group:\s*['"][\w.-]+['"],\s*name:\s*['"](?P[\w-]+)['"],\s*classifier:\s*['"][\w-]+['"]\s*\)''' BUNDLED_RE = re.compile(rf'\bbundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') From f5d9bbf6ab20008f2802f89cb35c25c67cf39ce1 Mon Sep 17 00:00:00 2001 From: Marco Borgeaud Date: Mon, 16 Dec 2024 13:13:00 +0100 Subject: [PATCH 5/8] Make comparable_version more linear --- rspec-tools/rspec_tools/coverage.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 4c4d1ff5eaf..8ec752270ca 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -203,12 +203,11 @@ def is_version_tag(name): def comparable_version(key): v = key.removeprefix('sqcb-').removeprefix('sqs-') - if not is_version_tag(v): - if v == 'master': - return [0] - else: - sys.exit(f'Unexpected version {key}') - return list(map(int, v.split('.'))) + if is_version_tag(v): + return list(map(int, v.split('.'))) + if v == 'master': + return [0] + sys.exit(f'Unexpected version {key}') def collect_coverage_for_all_versions(repo, coverage): From 69127ba2905d6819edcabc6341e6f3413e617176 Mon Sep 17 00:00:00 2001 From: Marco Borgeaud Date: Mon, 16 Dec 2024 14:37:25 +0100 Subject: [PATCH 6/8] Simplify how regex groups are accessed --- rspec-tools/rspec_tools/coverage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 8ec752270ca..70cc866f645 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -262,7 +262,7 @@ def get_plugin_versions(git_repo, version): content = g.show(f'{version}:build.gradle') versions = {} for m in re.finditer(DEPENDENCY_RE, content): - versions[m.groupdict()['plugin_name']] = m.groupdict()['version'] + versions[m['plugin_name']] = m['version'] return versions @@ -281,10 +281,10 @@ def get_packaged_plugins(git_repo): bundle_map[key] = [] content = g.show(f'master:{bundle}') for m in re.finditer(BUNDLED_RE, content): - if m.groupdict()['plugin_name'] != None: - bundle_map[key].append(m.groupdict()['plugin_name']) + if m['plugin_name'] != None: + bundle_map[key].append(m['plugin_name']) else: - bundle_map[key].append(m.groupdict()['plugin_name2']) + bundle_map[key].append(m['plugin_name2']) return bundle_map From 3b745315e47c13c3ec5e8ae0d7fb416da2884d34 Mon Sep 17 00:00:00 2001 From: Marco Borgeaud Date: Mon, 16 Dec 2024 15:38:14 +0100 Subject: [PATCH 7/8] Only support tags >= 9 Loosen regex to detect unexpectedly missing version in the future. --- rspec-tools/rspec_tools/coverage.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 70cc866f645..9c24ea929cb 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -49,7 +49,7 @@ RULES_FILENAME = 'covered_rules.json' -DEPENDENCY_RE = re.compile(r'''\bdependency\s+['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+):(?P\d+(\.\d+)+)['"]''') +DEPENDENCY_RE = re.compile(r'''\bdependency\s+['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+):(?P\d+(\.\d+)+)?''') BUNDLED_SIMPLE = r'''['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+)['"]''' BUNDLED_MULTI = r'''\(\s*group:\s*['"][\w.-]+['"],\s*name:\s*['"](?P[\w-]+)['"],\s*classifier:\s*['"][\w-]+['"]\s*\)''' @@ -262,6 +262,10 @@ def get_plugin_versions(git_repo, version): content = g.show(f'{version}:build.gradle') versions = {} for m in re.finditer(DEPENDENCY_RE, content): + if m['plugin_name'] in ['sonar-plugin-api', 'sonar-plugin-api-test-fixtures']: + # Ignore these "plugins". They may not have a numerical version. + continue + assert m['version'], f'Failed to find version from dependency {m[0]}' versions[m['plugin_name']] = m['version'] return versions @@ -360,7 +364,7 @@ def is_interesting_version(version): major = int(version[:version.find('.')]) except ValueError: return False - return major >= 8 + return major >= 9 def collect_coverage_per_product(): git_repo = checkout_repo('sonar-enterprise') From af5edc79aa5bf1a6f4e9d277fce573e0d04de247 Mon Sep 17 00:00:00 2001 From: Fred Tingaud Date: Tue, 17 Dec 2024 11:57:29 +0100 Subject: [PATCH 8/8] Fix version for unreleased rules and handle gracefully unreleased rules in mapping --- rspec-tools/rspec_tools/coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 9c24ea929cb..9d82024baed 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -206,7 +206,7 @@ def comparable_version(key): if is_version_tag(v): return list(map(int, v.split('.'))) if v == 'master': - return [0] + return [sys.maxsize] sys.exit(f'Unexpected version {key}') @@ -300,7 +300,7 @@ def lowest_version(plugin_versions, plugin, version, skip_suffix): pvv = plugin_versions[t][plugin] if comparable_version(pvv) >= comparable_version(version): return t - sys.exit(f'failed finding the oldest version of {plugin} containing {version} from {plugin_versions}') + return "Coming soon" def lowest_community_build_version(plugin_versions, plugin, version):