From 5df35b5421b6c42cd8592446bbf275d0f37decdd Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 16 Jun 2023 11:10:55 +0200 Subject: [PATCH] Ignore prodtype when building Benchmark and include only rules that are part of a Profile WIP --- .../rule.yml | 2 +- .../oval/shared.xml | 8 +- .../oval/shared.xml | 14 +- .../oval/shared.xml | 24 +-- .../oval/shared.xml | 34 +-- .../oval/shared.xml | 14 +- ssg/build_yaml.py | 18 +- ssg/entities/profile_base.py | 8 + utils/convert_prodtype_ds_check.py | 72 +++++++ ...ert_prodtype_into_default_profile_entry.py | 197 ++++++++++++++++++ utils/convert_prodtype_remove_prodtype.sh | 14 ++ 11 files changed, 356 insertions(+), 49 deletions(-) create mode 100755 utils/convert_prodtype_ds_check.py create mode 100755 utils/convert_prodtype_into_default_profile_entry.py create mode 100755 utils/convert_prodtype_remove_prodtype.sh diff --git a/linux_os/guide/system/auditing/auditd_configure_rules/audit_privileged_commands/audit_rules_privileged_commands_unix2_chkpwd/rule.yml b/linux_os/guide/system/auditing/auditd_configure_rules/audit_privileged_commands/audit_rules_privileged_commands_unix2_chkpwd/rule.yml index 47d4add9ebc0..ed8ecfc2cc7c 100644 --- a/linux_os/guide/system/auditing/auditd_configure_rules/audit_privileged_commands/audit_rules_privileged_commands_unix2_chkpwd/rule.yml +++ b/linux_os/guide/system/auditing/auditd_configure_rules/audit_privileged_commands/audit_rules_privileged_commands_unix2_chkpwd/rule.yml @@ -63,4 +63,4 @@ ocil: |- template: name: audit_rules_privileged_commands vars: - path@sle15: /sbin/unix2_chkpwd + path: /sbin/unix2_chkpwd diff --git a/linux_os/guide/system/software/gnome/gnome_login_screen/gnome_gdm_disable_unattended_automatic_login/oval/shared.xml b/linux_os/guide/system/software/gnome/gnome_login_screen/gnome_gdm_disable_unattended_automatic_login/oval/shared.xml index 05f30bb0e2d2..55b5fd248a81 100644 --- a/linux_os/guide/system/software/gnome/gnome_login_screen/gnome_gdm_disable_unattended_automatic_login/oval/shared.xml +++ b/linux_os/guide/system/software/gnome/gnome_login_screen/gnome_gdm_disable_unattended_automatic_login/oval/shared.xml @@ -3,17 +3,17 @@ {{{ oval_metadata("Disable the GNOME Display Manager (GDM) ability to allow users to automatically login.") }}} - + - + id="test_disable_unattended_automatic_login" version="1"> + - /etc/sysconfig/displaymanager ^DISPLAYMANAGER_AUTOLOGIN=""$ diff --git a/linux_os/guide/system/software/updating/ensure_fedora_gpgkey_installed/oval/shared.xml b/linux_os/guide/system/software/updating/ensure_fedora_gpgkey_installed/oval/shared.xml index 9f006ffd4347..fdbefee62e99 100644 --- a/linux_os/guide/system/software/updating/ensure_fedora_gpgkey_installed/oval/shared.xml +++ b/linux_os/guide/system/software/updating/ensure_fedora_gpgkey_installed/oval/shared.xml @@ -1,18 +1,18 @@ {{% macro fedora_gpgkey_criterion(fedora_version, pkg_release, pkg_version) %}} + test_ref="test_fedora_package_gpgkey-{{{ pkg_version }}}-{{{ pkg_release }}}_installed" /> {{% endmacro %}} {{% macro fedora_gpgkey_check(fedora_version, pkg_release, pkg_version) %}} - - + + - + {{{ pkg_release }}} {{{ pkg_version }}} @@ -32,8 +32,8 @@ - - + + gpg-pubkey diff --git a/linux_os/guide/system/software/updating/ensure_oracle_gpgkey_installed/oval/shared.xml b/linux_os/guide/system/software/updating/ensure_oracle_gpgkey_installed/oval/shared.xml index 625014621513..e119d9911cbe 100644 --- a/linux_os/guide/system/software/updating/ensure_oracle_gpgkey_installed/oval/shared.xml +++ b/linux_os/guide/system/software/updating/ensure_oracle_gpgkey_installed/oval/shared.xml @@ -9,29 +9,29 @@ + test_ref="test_oracle_package_gpgkey-{{{ pkg_version }}}-{{{ pkg_release }}}_installed" /> {{% if aux_pkg_version %}} + test_ref="test_oracle_package_gpgkey-{{{ aux_pkg_version }}}-{{{ aux_pkg_release }}}_installed" /> {{% endif %}} - - + + gpg-pubkey - - + + - + {{{ pkg_release }}} {{{ pkg_version }}} @@ -39,13 +39,13 @@ {{% if aux_pkg_version %}} - - + + - + {{{ aux_pkg_release }}} {{{ aux_pkg_version }}} diff --git a/linux_os/guide/system/software/updating/ensure_redhat_gpgkey_installed/oval/shared.xml b/linux_os/guide/system/software/updating/ensure_redhat_gpgkey_installed/oval/shared.xml index dd514ad95fc1..96cb0ff5b12e 100644 --- a/linux_os/guide/system/software/updating/ensure_redhat_gpgkey_installed/oval/shared.xml +++ b/linux_os/guide/system/software/updating/ensure_redhat_gpgkey_installed/oval/shared.xml @@ -12,63 +12,63 @@ + test_ref="test_redhat_package_gpgkey-{{{ pkg_version }}}-{{{ pkg_release }}}_installed" /> + test_ref="test_redhat_package_gpgkey-{{{ aux_pkg_version }}}-{{{ aux_pkg_release }}}_installed" /> {{%- if centos_major_version %}} + test_ref="test_redhat_package_gpgkey-{{{ centos_pkg_version }}}-{{{ centos_pkg_release }}}_installed" /> {{%- endif %}} - - + + gpg-pubkey - - + + - + {{{ pkg_release }}} {{{ pkg_version }}} - - + + - + {{{ aux_pkg_release }}} {{{ aux_pkg_version }}} {{%- if centos_major_version %}} - - + + - + {{{ centos_pkg_release }}} {{{ centos_pkg_version }}} diff --git a/linux_os/guide/system/software/updating/ensure_suse_gpgkey_installed/oval/shared.xml b/linux_os/guide/system/software/updating/ensure_suse_gpgkey_installed/oval/shared.xml index 23f85c5ad169..9f9ed521dd13 100644 --- a/linux_os/guide/system/software/updating/ensure_suse_gpgkey_installed/oval/shared.xml +++ b/linux_os/guide/system/software/updating/ensure_suse_gpgkey_installed/oval/shared.xml @@ -9,26 +9,26 @@ + test_ref="test_suse_package_gpgkey-{{{ pkg_version }}}-{{{ pkg_release }}}_installed" /> - - + + gpg-pubkey - - + + - + {{{ pkg_release }}} {{{ pkg_version }}} diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index fde38c00f3ff..87fd0607db1d 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -336,6 +336,13 @@ def unselect_empty_groups(self): for p in self.profiles: p.unselect_empty_groups(self) + def drop_rules_not_included_in_a_profile(self): + selected_profiles = set() + for p in self.profiles: + selected_profiles.update(p.selected) + for g in self.groups.values(): + g.remove_rules_with_ids_not_listed(selected_profiles) + def to_xml_element(self, env_yaml=None, product_cpes=None): root = ET.Element('{%s}Benchmark' % XCCDF12_NS) root.set('id', OSCAP_BENCHMARK + self.id_) @@ -631,6 +638,11 @@ def _add_child(self, child, childs, env_yaml=None, product_cpes=None): child.inherited_platforms.update(self.platforms, self.inherited_platforms) childs[child.id_] = child + def remove_rules_with_ids_not_listed(self, rule_ids_list): + self.rules = dict(filter(lambda el, ids=rule_ids_list: el[0] in ids, self.rules.items())) + for group in self.groups.values(): + group.remove_rules_with_ids_not_listed(rule_ids_list) + def __str__(self): return self.id_ @@ -1364,7 +1376,10 @@ def _process_rule(self, rule): (rule.id_, self.components_dir)) prodtypes = parse_prodtype(rule.prodtype) if "all" not in prodtypes and self.product not in prodtypes: - return False + pass +# print(f"Rule prodtype is not compatible: {rule.id_}") +# rule.incompatible = True +# rule.conflicts.append('prodtype') self.all_rules[rule.id_] = rule self.loaded_group.add_rule( rule, env_yaml=self.env_yaml, product_cpes=self.product_cpes) @@ -1456,6 +1471,7 @@ def load_benchmark(self, directory): except KeyError as exc: # Add only the groups we have compiled and loaded pass + self.benchmark.drop_rules_not_included_in_a_profile() self.benchmark.unselect_empty_groups() def load_compiled_content(self): diff --git a/ssg/entities/profile_base.py b/ssg/entities/profile_base.py index 1d217a4cc9c2..bf970c271349 100644 --- a/ssg/entities/profile_base.py +++ b/ssg/entities/profile_base.py @@ -37,6 +37,7 @@ class Profile(XCCDFEntity, SelectionHandler): KEYS = dict( description=lambda: "", extends=lambda: "", + hidden=lambda: "", metadata=lambda: None, reference=lambda: None, selections=lambda: list(), @@ -112,6 +113,9 @@ def _should_have_version(self): 'version'] is not None def to_xml_element(self): + if self.hidden: + return ET.Comment('Default Profile') + element = ET.Element('{%s}Profile' % XCCDF12_NS) element.set("id", OSCAP_PROFILE + self.id_) if self._should_have_version(): @@ -207,6 +211,7 @@ def validate_variables(self, variables): def validate_rules(self, rules, groups): existing_rule_ids = [r.id_ for r in rules] +# prodtype_conflicting_rules = [r.id_ for r in rules if 'prodtype' in r.conflicts] rule_selectors = self.get_rule_selectors() for id_ in rule_selectors: if id_ in groups: @@ -226,6 +231,8 @@ def validate_rules(self, rules, groups): .format(rule_id=id_, profile_id=self.id_) ) raise ValueError(msg) +# if id_ in prodtype_conflicting_rules: +# print(f"We have a prodtype conflict in profile {self.id_}, rule: {id_}") def _find_empty_groups(self, group, profile_rules): is_empty = True @@ -262,6 +269,7 @@ def __sub__(self, other): profile.extends = self.extends profile.platforms = self.platforms profile.platform = self.platform + profile.hidden = self.hidden profile.selected = list(set(self.selected) - set(other.selected)) profile.selected.sort() profile.unselected = list(set(self.unselected) - set(other.unselected)) diff --git a/utils/convert_prodtype_ds_check.py b/utils/convert_prodtype_ds_check.py new file mode 100755 index 000000000000..7e31d2121117 --- /dev/null +++ b/utils/convert_prodtype_ds_check.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +import os +from xml.etree import ElementTree + +NAMESPACES = dict( + xccdf_ns="http://scap.nist.gov/schema/scap/source/1.2", + profile_ns="http://checklists.nist.gov/xccdf/1.2", +) + +SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +def fname_to_etree(fname): + input_tree = ElementTree.parse(fname) + return input_tree + + +def get_profiles_from_etree(tree): + xpath_expr = ".//{%s}Profile" % NAMESPACES["profile_ns"] + xccdfs = tree.findall(xpath_expr) + return xccdfs + + +def get_selections_from_etree(tree): + xpath_expr = ".//{%s}select" % NAMESPACES["profile_ns"] + xccdfs = tree.findall(xpath_expr) + return xccdfs + + +def get_rules_from_etree(tree): + xpath_expr = ".//{%s}Rule" % NAMESPACES["profile_ns"] + xccdfs = tree.findall(xpath_expr) + return xccdfs + + +def extract_tree_from_file(fname): + return fname_to_etree(fname) + + +def get_ds_stats(filename): + tree = extract_tree_from_file(filename) + profiles = sorted(get_profiles_from_etree(tree), key=lambda x: x.attrib["id"]) + rules = sorted(get_rules_from_etree(tree), key=lambda x: x.attrib["id"]) + yield f"Found {len(profiles)} profiles, {len(rules)} rules\n" + rules_selections = {} + for p in profiles: + p_id = p.attrib["id"].removeprefix("xccdf_org.ssgproject.content_") + selections = sorted(get_selections_from_etree(p), key=lambda x: x.attrib["idref"]) + yield f"{p_id} (selections: {len(selections)})\n" + for sel in selections: + r_id = sel.attrib["idref"].removeprefix("xccdf_org.ssgproject.content_") + r_selected = sel.attrib["selected"].lower() == "true" + yield f" {'+' if r_selected else '-'}{r_id}\n" + r_stats = rules_selections.get(r_id, {"selected": 0, "unselected": 0}) + r_stats["selected" if r_selected else "unselected"] += 1 + rules_selections[r_id] = r_stats + for r in rules: + r_id = r.attrib["id"].removeprefix("xccdf_org.ssgproject.content_") + r_selected = r.attrib["selected"].lower() == "true" + in_profiles = f"selected: {rules_selections[r_id]['selected']}, unselected: {rules_selections[r_id]['unselected']}" if r_id in rules_selections else "absent" + yield f"{'+' if r_selected else '-'}{r_id} (profiles: {in_profiles})\n" + + +if __name__ == "__main__": + for d in os.listdir(f"{SSG_ROOT}/products"): + fn = f"{SSG_ROOT}/build/ssg-{d}-ds.xml" + if os.path.isfile(fn): + stats = get_ds_stats(fn) + with open(f"{SSG_ROOT}/build/ssg-{d}-ds.prof-stats", "w") as f: + print(f"Writing stats for {d}") + f.writelines(stats) diff --git a/utils/convert_prodtype_into_default_profile_entry.py b/utils/convert_prodtype_into_default_profile_entry.py new file mode 100755 index 000000000000..d81a8cd14032 --- /dev/null +++ b/utils/convert_prodtype_into_default_profile_entry.py @@ -0,0 +1,197 @@ +#!/usr/bin/python3 + +import sys +import os +import argparse +import json + +import ssg.rules +import ssg.utils +import ssg.products +import ssg.rule_yaml +import ssg.build_yaml +import ssg.build_profile +import ssg.entities.profile_base +import ssg.controls +import ssg.build_cpe +import ssg.yaml + +SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +GUIDE_RULES = {} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--product", type=str, action="store", + help="Product (defaults to all if not set)") + return parser.parse_args() + + +def collect_rule_ids_and_dirs(rules_dir): + for rule_dir in sorted(ssg.rules.find_rule_dirs(rules_dir)): + yield ssg.rules.get_rule_dir_id(rule_dir), rule_dir + + +def handle_rule_yaml(env, rule_dir): + rule_file = ssg.rules.get_rule_dir_yaml(rule_dir) + rule_yaml = ssg.build_yaml.Rule.from_yaml(rule_file, env) + return rule_file, rule_yaml + + +def get_rules(product, product_path, env): + guide_path = os.path.abspath(os.path.join(product_path, product['benchmark_root'])) + if guide_path not in GUIDE_RULES: + print(f"Loading rules from '{guide_path}'...", end=""), + GUIDE_RULES[guide_path] = {} + for rule_id, rule_dir in collect_rule_ids_and_dirs(guide_path): + try: + rule_file, rule_yaml = handle_rule_yaml(env, rule_dir) + GUIDE_RULES[guide_path][rule_yaml.id_] = rule_yaml + except ssg.yaml.DocumentationNotComplete: + # Happens on non-debug build when a rule is "documentation-incomplete" + continue + print(len(GUIDE_RULES[guide_path])) + return GUIDE_RULES[guide_path] + + +def get_products_and_rules(root_path): + linux_products, other_products = ssg.products.get_all(root_path) + all_products = linux_products.union(other_products) + for product_id in all_products: + product_path = ssg.products.product_yaml_path(root_path, product_id) + product_props_path = os.path.join(root_path, "product_properties") + product = ssg.products.load_product_yaml(product_path) + product.read_properties_from_directory(product_props_path) + env = dict(product) + rules = get_rules(product, os.path.join(root_path, "products", product_id), env) + yield product_id, product, env, rules + + +def unselect_rules_in_profile(product, profile, profile_path, prof_unsel): + if len(prof_unsel) == 0: + return + + comment = f"# Following rules once had a prodtype incompatible with the {product['product']} product" + with open(profile_path, 'r') as f: + lines = f.readlines() + comment_line = 0 + indent = 0 + sel_start = 0 + sel_end = 0 + already_unselected = [] + for i, line in enumerate(lines): + strip_line = line.strip() + if strip_line.startswith("- '!") or strip_line.startswith('- "!'): + already_unselected.append(strip_line[4:-1]) + continue + if comment in line: + comment_line = i + continue + if strip_line.startswith("#") or not strip_line.strip(): + continue + if line.startswith("selections:"): + sel_start = i + sel_end = len(lines) - 1 + continue + if sel_start != 0: + if not strip_line.startswith("-"): + sel_end = i - 1 + elif indent == 0: + indent = line.find("-") + + for unsel in prof_unsel: + if unsel in already_unselected: + continue + lines.insert(sel_end+1, f"{indent * ' '}- '!{unsel}'\n") + if comment_line == 0: + lines.insert(sel_end+1, f"{indent * ' '}{comment}\n") + + with open(profile_path, 'w') as f: + f.writelines(lines) + + +def select_rules_in_default_profile(product, profiles_root, prof_def_sel): + if len(prof_def_sel) == 0: + return + + prefix = (f"documentation_complete: true\n" + f"\n" + f"hidden: true\n" + f"\n" + f"title: Default Profile for {product['full_name']}\n" + f"\n" + f"description: |-\n" + f" This profile contains all the rules that once belonged to the\n" + f" {product['product']} product via 'prodtype'. This profile won't\n" + f" be rendered into an XCCDF Profile entity, nor it will select any\n" + f" of these rules by default. The only purpose of this profile\n" + f" is to keep a rule in the product's XCCDF Benchmark.\n" + f"\n" + f"selections:\n") + + path = os.path.join(SSG_ROOT, "products", product['product'], profiles_root) + with open(f"{path}/default.profile", "w") as f: + f.write(prefix) + for sel in prof_def_sel: + f.write(f" - {sel}\n") + + +def main(): + args = parse_args() + for product_id, product, env, all_rules in get_products_and_rules(SSG_ROOT): + if args.product: + if product_id != args.product: + continue + print(f"Product '{product_id}', profiles:") + + product_cpes = ssg.build_cpe.ProductCPEs() + product_cpes.load_product_cpes(env) + + controls_path = os.path.join(SSG_ROOT, "controls") + controls_manager = ssg.controls.ControlsManager(controls_path, env) + controls_manager.load() + + profile_paths = ssg.products.get_profile_files_from_root(env, product) + all_sels = set() + for profile_path in profile_paths: + try: + profile = ssg.build_yaml.ProfileWithInlinePolicies.from_yaml(profile_path, env, product_cpes) + except ssg.yaml.DocumentationNotComplete: + continue + if profile.id_ == 'default': + # Whatever is there — we are going to overwrite it + continue + profile.resolve_controls(controls_manager) + sels = sorted(profile.selected) + unsels = sorted(profile.unselected) + extends = profile.extends if profile.extends else "no" + print(f" {profile.id_} (extends: {extends}, selections: {len(sels)}/{len(unsels)})", end="") + prof_unsel = set() + for sel in sels: + if sel in all_rules: + if all_rules[sel].prodtype == '' or all_rules[sel].prodtype is None: + raise Exception("Should not happen!") + if product_id not in all_rules[sel].prodtype and all_rules[sel].prodtype != 'all': + if sel not in unsels: + #print(f' - {sel}') + prof_unsel.add(sel) + print(f' -{len(prof_unsel)}') + all_sels.update(sels) + all_sels.difference_update(unsels) + + unselect_rules_in_profile(product, profile, profile_path, prof_unsel) + + prof_def_sel = set() + print(' default (hidden)', end="") + for rul_id, rul in all_rules.items(): + if product_id in rul.prodtype or rul.prodtype == 'all': + if rul_id not in all_sels: + #print(f' + {rul_id}') + prof_def_sel.add(rul_id) + print(f' +{len(prof_def_sel)}') + + select_rules_in_default_profile(product, env['profiles_root'], prof_def_sel) + + +if __name__ == "__main__": + main() diff --git a/utils/convert_prodtype_remove_prodtype.sh b/utils/convert_prodtype_remove_prodtype.sh new file mode 100755 index 000000000000..c3ec87e58158 --- /dev/null +++ b/utils/convert_prodtype_remove_prodtype.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +guides="linux_os/guide \ +products/chromium/guide \ +applications \ +products/firefox/guide \ +apple_os" + +script=$(realpath "$0") +root=$(dirname "$script") + +for path in $guides; do + find "$root/../$path" -type f -print0 | xargs -0 sed -i "/prodtype:/d" +done