From a4b775036de744cac4cc218aead39aa548c9a2fd Mon Sep 17 00:00:00 2001 From: Mustafa Kemal Gilor Date: Tue, 25 Jun 2024 18:56:25 +0300 Subject: [PATCH] requires/types: introduce expression type (expr.py) `expression` is a new requirement type that allows user to express a requirement in human-readable, SQL-like syntax. It has a proper grammar to allow user to write checks as easily digestable manner, which will improve the QOL for scenario writers and code reviewers. The grammar is constructed using `pyparsing` library. The current grammar supports the following constructs: - Keywords (True, False, None) - Constants (Integer, Float, String literal) - Runtime variables (Python properties) - Functions(len, not, file, systemd, read_ini, read_cert) - Arithmetic operators(sign, mul, div, add, sub, exp) - Comparison operators(<, <=, >, >=, ==, !=, in) - Logical operators(and, or, not) - Comments('#', '//', '/*...*/') Also, the following changes have been made: - Updated requirement_types.rst to document the new Expression requirement type. - Moved property resolver logic from YPropertyBase to PythonEntityResolver class. - Added `pyparsing` as a dependency. - Added unit tests for the new code, the coverage rate is 100 pct for the expr.py This patch rewrites some of the scenario checks in the new expression syntax to demonstrate the difference and provide examples. Signed-off-by: Mustafa Kemal Gilor --- .../property_ref/requirement_types.rst | 620 ++++++++++++ hotsos/core/exceptions.py | 8 + hotsos/core/plugins/kernel/memory.py | 6 +- .../core/ycheck/engine/properties/checks.py | 14 +- .../core/ycheck/engine/properties/common.py | 93 +- .../engine/properties/requires/requires.py | 4 +- .../engine/properties/requires/types/expr.py | 686 +++++++++++++ .../scenarios/juju/jujud_machine_checks.yaml | 6 +- .../defs/scenarios/kernel/amd_iommu_pt.yaml | 18 +- .../scenarios/kernel/kernlog_calltrace.yaml | 38 +- hotsos/defs/scenarios/kernel/memory.yaml | 32 +- .../defs/scenarios/kernel/network/misc.yaml | 6 +- .../scenarios/kernel/network/netlink.yaml | 4 +- hotsos/defs/scenarios/kernel/network/tcp.yaml | 163 ++-- hotsos/defs/scenarios/kernel/network/udp.yaml | 76 +- .../kubernetes/system_cpufreq_mode.yaml | 13 +- hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml | 12 +- hotsos/defs/scenarios/openstack/eol.yaml | 6 +- .../openstack_apache2_certificates.yaml | 14 +- .../openstack/openstack_charm_conflicts.yaml | 14 +- .../pkgs_from_mixed_releases_found.yaml | 8 +- .../openstack/system_cpufreq_mode.yaml | 17 +- .../openstack/systemd_masked_services.yaml | 8 +- .../ovn/ovn_central_certs_logs.yaml | 39 +- .../openvswitch/ovn/ovn_certs_valid.yaml | 34 +- requirements.txt | 1 + tests/unit/test_yexpr_parser.py | 431 +++++++++ tests/unit/test_yexpr_token.py | 902 ++++++++++++++++++ 28 files changed, 2929 insertions(+), 344 deletions(-) create mode 100644 hotsos/core/ycheck/engine/properties/requires/types/expr.py create mode 100644 tests/unit/test_yexpr_parser.py create mode 100644 tests/unit/test_yexpr_token.py diff --git a/doc/source/contrib/language_ref/property_ref/requirement_types.rst b/doc/source/contrib/language_ref/property_ref/requirement_types.rst index 0139cb251..4f58b7ea8 100644 --- a/doc/source/contrib/language_ref/property_ref/requirement_types.rst +++ b/doc/source/contrib/language_ref/property_ref/requirement_types.rst @@ -337,3 +337,623 @@ Cache keys: * input_value: value of the variable used as input * ops: str representation of ops list + +Expression +---------- + +Expressions allow the user to express a requirement in a human-readable domain specific language. Its purpose +is to provide a simplified, clear and concise expression grammar for all requirement types, and to eliminate +redundancies. + +Usage: + +.. code-block:: yaml + + checks: + checkmyvar: + # Explicit expression + expression: | + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + @hotsos.core.plugins.plugin3.class.property_2) + +Explicit expression form can be combined with other requirement types: + +.. code-block:: yaml + + checks: + checkmyvar: + # Explicit expression + expression: | + # 'test' should not be present + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + # Property value should exceed the threshold + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + # Property is True + @hotsos.core.plugins.plugin3.class.property_2) + systemd: test-service + +Expressions can be declared implicitly when assigned as a literal to a check: + +.. code-block:: yaml + + checks: + # Implicit expression + checkmyvar: | + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + @hotsos.core.plugins.plugin3.class.property_2) + +Expression grammar consist of the following constructs to allow user to express a requirement: + +Keywords +******** + +Keywords consist of a special set of words that has a purpose in the grammar. The current keywords are +[:ref:`None keyword` | :ref:`True keyword` | :ref:`False keyword`] + +None keyword +============== + +Case-insensitive. Equal to Python `None`. + +**Examples** + +.. code-block:: yaml + + expression: | + NONE + NoNE + +True keyword +============== + +Case-insensitive. Equal to Python `True`. + +**Examples** + +.. code-block:: yaml + + expression: | + True + TrUe + +False keyword +=============== + +Case-insensitive. Equal to Python `False`. + +**Examples** + +.. code-block:: yaml + + expression: | + FALSE + False + +Constants +********* + +Constants are invariable values in the grammar. The current constants are [:ref:`float constant` | :ref:`integer constant` | :ref:`String literal`] + +float constant +================ + +Floating point constants. `[0-9]+\.[0-9]+` + +**Examples** + +.. code-block:: yaml + + expression: | + 12.34 + -56.78 + +integer constant +================== + +Integer constants. `[0-9]+` + +**Examples** + +.. code-block:: yaml + + expression: | + 1234 + -4567 + +String literal +============== + +String literal. Declared with single quotes `''` + +**Examples** + +.. code-block:: yaml + + expression: | + 'this is a string literal' + 'this is @nother 1' + '1234' + '1.2' + 'True' + +Functions +********* + +Functions are defined as CaselessKeyword, followed by a `(` args `)`. + +`caseless_keyword(expr1, ... exprN)`. + +len(expr...) +============ + +Function to retrieve length of an expression. The expression argument should evaluate +to a construct that has a `length` property. + +**Examples** + +.. code-block:: yaml + + expression: | + len('literal') # would return 7 + len('lit' + 'eral') # would return 7 + len(@class.property) # would return the length of the property value + +not(expr...) +============ + +Function to negate an expression's boolean vlaue. The expression argument should evaluate +to a construct that is `boolable`. + +**Examples** + +.. code-block:: yaml + + expression: | + not(True) # would return False + not(None) # would return True + not(3 == 4) # Would return True + +file('path-to-file', 'property-name[optional]') +=============================================== + +Function to retrieve a file's properties. Returns True or False when called with one argument depending +whether the file exists or not. Returns the property's value when called with two arguments. Property names +are all the properties that `hotsos.core.host_helpers.filestat.FileObj` has. + +** Examples ** + +.. code-block:: yaml + + expression: | + # Returns True or False, depending on whether the `/path/to/file` exists or not. + file('/path/to/file') + +.. code-block:: yaml + + expression: | + # Returns file's mtime property when file exists, YExprNotFound when the + # file is absent. Raises an exception when the given property name is not + # present on the service object. + file('/path/to/file', 'mtime') + +systemd('service-name', 'property-name[optional]') +================================================== + +Function to retrieve a systemd service's properties. Returns True or False when called with one argument depending +whether the service exists or not. Returns the property's value when called with two arguments. + +Property names are all the properties that `hotsos.core.host_helpers.systemd.SystemdService` has. + +**Examples** + +.. code-block:: yaml + + expression: | + # Returns True or False, depending on whether the `systemd-resolve` service + # exists or not. + systemd('systemd-resolved') + +.. code-block:: yaml + + expression: | + # Returns the service's `start_time_secs` property value when the service file + # exists, YExprNotFound when the service is absent. Raises an exception when the + # given property name is not present on the service object. + systemd('systemd-resolved', 'start_time_secs') + +read_ini('path-to-ini', 'key', 'section[optional]', 'default[optional]') +======================================================================== + +Function to read values from an ini file. Returns the first matching key's value when called with two arguments. Returns +the key's value in the specific section when called with three arguments. Returns the value specified in `default` when +the `default` argument is provided and given key in given section is not found. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return the value of the first encountered key 'foo' from any section. + # would return YExprNotFound when the key is not found. + read_ini('/etc/ini-file', 'foo') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from section 'bar. + # would return YExprNotFound when the key is not found. + read_ini('/etc/ini-file', 'foo', 'bar') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from section 'bar'. + # would return the string literal 'abcd' when the key is not found. + read_ini('/etc/ini-file', 'foo', 'bar', 'abcd') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from any section. + # would return the string literal 'abcd' when the key is not found. + read_ini('/etc/ini-file', 'foo', None, 'abcd') + +read_cert('path-to-cert', 'property-name[optional]') +==================================================== + +Function to read values from a X509 certificate (e.g. a typical SSL certificate). Returns True or False depending whether +the certificate file 'path-to-cert' exist when called with a single argument. With two arguments, Returns the value of the +`property-name` when the certificate file `path-to-cert` exist, YExprNotFound otherwise. + +Property names are all the properties that `hotsos.core.host_helpers.ssl.SSLCertificate` has. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return True or False depending on whether '/etc/ssl/cert' exist or not. + read_cert('/etc/ssl/cert') + +.. code-block:: yaml + + expression: | + # Would return the certificate's expiry_date property when the certificate + # file is present. + # Raises an exception when property name cannot be found on SSLCertificate object. + #read_cert('/etc/ssl/cert', 'expiry_date') + +Runtime variables +***************** + +Python property `@module.class.property_name` +============================================= + +Retrieve the value of a runtime Python property. Raises an exception when a property with a given name is not found. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return the value of the property + # Raises exception when property is absent. + @hotsos.plugins.plugin.class_name.property_name + + +Operands +******** + +:ref:`Keywords` | :ref:`Constants` | :ref:`Functions` | :ref:`Runtime Variables` + +Arithmetic operators +******************** + +plus sign (`+`) +=============== + +Indicate the sign of an Integer or Float. + +**Examples** + +.. code-block:: yaml + + expression: | + # plus six + +6 + # plus six + +(+6) + +minus sign (`-`) +================ + +Indicate the sign of an Integer or Float. + +**Examples** + +.. code-block:: yaml + + expression: | + # plus six + -(-6) + # minus six + -6 + +multiplication (`*`) +==================== + +Multiply the given operands for the operands that are multiplicable with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 0.6 + 1 * 0.6 + +division(`/`) +============= + +Perform a division with the given operands for the operands that supports division with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 2.5 + 1 / 0.4 + +addition(`+`) +============= + +Add two things with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 8 + 3 + 5 # 8 + # result is 'aabb' + 'aa' + 'bb' + +subtraction(`-`) +================ + +Subtract two things from each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is -9 + 3 - 5 - 7 + +exponent (`\*\*`) +================= + +Calculate the exponent of a number. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 8 + 2**3 + # result is 729 + 9**3 + +Comparison operators +******************** + +Comparison operators allow users to compare values within expressions. These operators can be used to evaluate whether a particular condition is met. + +less than (`<`, `LT`) +===================== + +Evaluates whether the left operand is less than the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 < 5 + # result is False + 3.81 > 3.80 + +less than or equal (`<=`, `LE`) +=============================== + +Evaluates whether the left operand is less than or equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 5 <= 5 + # result is False + 3.81 LE 3.80 + +greater than (`>`, `GT`) +======================== + +Evaluates whether the left operand is greater than the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is False + 3 > 5 + # result is True + 3.81 > (2.80 + 1) + +greater than or equal (`>=`, `GE`) +================================== + +Evaluates whether the left operand is greater than or equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is False + 3 >= 5 + # result is True + 3.81 GE (3.80 + 0.01) + +equal (`==`, `EQ`) +================== + +Evaluates whether the left operand is equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 == 3 + # result is False + True == False + # result is True + 'test' EQ 'test' + +not equal (`!=`, `NE`, `<>`) +============================ + +Evaluates whether the left operand is not equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 != 5 + # result is False + 3.81 <> (3.80 + 0.01) + # result is True + 'test' NE None + +in (`IN`) +========= + +Evaluates whether the left operand is contained within the right operand (e.g., within a string or list). + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 'a' in 'ab' + # result is True (assuming list of ints = [1,2,3]) + 1 in @module.prop.list_of_ints + +Logical operators +***************** + +Logical operators allow users to combine multiple expressions. These operators evaluate the truthiness of expressions to determine the overall outcome. + +logical and (`and`) +=================== + +Returns True if both operands are True. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + True and (5 < 3 or 'test' in 'testament') + # result is False + (3.3 <= 2) or (5 > 3 and 'rest' in 'testament') + +logical or (`or`) +================= + +Returns True if at least one of the operands is True. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + True or False + # result is False + False or not True + +logical not (`not`) +=================== + +Returns the opposite boolean value of the operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + not None + # result is True + not False + +Comments +******** + +Comments are used to annotate the code and are ignored during execution. This can be useful for adding explanations or notes within the expression grammar. + +Python-style comments (`#`) +=========================== + +Single-line comments that begin with `#`. + +**Examples** + +.. code-block:: yaml + + expression: | + # This is a comment explaining what the expression does. + not None + + +C-style comments (`//`, `/*...*/`) +================================== + +Single-line comments using `//` or multi-line comments enclosed in `/*...*/`. + +**Examples** + +.. code-block:: yaml + + expression: | + // This is a single-line comment + +.. code-block:: yaml + + expression: | + /* This is a + multi-line comment */ + +Expression grammar +****************** + +- :ref:`Operands` | :ref:`Arithmetic operators` | :ref:`Comparison operators` | :ref:`Logical operators` + +Note that comments are not the part of the expression and ignored by the parser. diff --git a/hotsos/core/exceptions.py b/hotsos/core/exceptions.py index fd68f9e12..7c51fb271 100644 --- a/hotsos/core/exceptions.py +++ b/hotsos/core/exceptions.py @@ -35,6 +35,10 @@ class NotEnoughParametersError(Exception): """Raised when an operation did not get enough parameters to operate.""" +class TooManyParametersError(Exception): + """Raised when an operation did get more parameters than expected.""" + + class MissingRequiredParameterError(Exception): """Raised when an operation did not get a parameter required for the operation.""" @@ -59,3 +63,7 @@ class PreconditionError(Exception): class ExpectationNotMetError(Exception): """Raised when an operation's expectation is not met.""" + + +class NoSuchPropertyError(Exception): + """Raised when an object does not have a property where it should.""" diff --git a/hotsos/core/plugins/kernel/memory.py b/hotsos/core/plugins/kernel/memory.py index b934c51af..66cc6bf6f 100644 --- a/hotsos/core/plugins/kernel/memory.py +++ b/hotsos/core/plugins/kernel/memory.py @@ -106,11 +106,13 @@ def huge_pages_enabled(self): @property def hugetlb_to_mem_total_percentage(self): - return round((self.Hugetlb * 100) / self.MemTotal) + return (self.MemTotal and round( + (self.Hugetlb * 100) / self.MemTotal)) @property def mem_avail_to_mem_total_percentage(self): - return round((self.MemAvailable * 100) / self.MemTotal) + return (self.MemTotal and round( + (self.MemAvailable * 100) / self.MemTotal)) @property def hugep_used_to_hugep_total_percentage(self): diff --git a/hotsos/core/ycheck/engine/properties/checks.py b/hotsos/core/ycheck/engine/properties/checks.py index 560920c1a..f06a072e8 100644 --- a/hotsos/core/ycheck/engine/properties/checks.py +++ b/hotsos/core/ycheck/engine/properties/checks.py @@ -1,6 +1,9 @@ from functools import cached_property -from propertree.propertree2 import PTreeLogicalGrouping +from propertree.propertree2 import ( + PTreeLogicalGrouping, + PTreeOverrideLiteralType +) from hotsos.core.log import log from hotsos.core.ycheck.engine.properties.common import ( YPropertyOverrideBase, @@ -11,6 +14,9 @@ from hotsos.core.ycheck.engine.properties.requires.requires import ( YPropertyRequires ) +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + YPropertyExpr +) from hotsos.core.ycheck.engine.properties.search import ( YPropertySearch, ) @@ -162,6 +168,12 @@ def result(self): stop_executon = False for member in self.members: for item in member: + # Allow implicit declaration of expression. + if isinstance(item, PTreeOverrideLiteralType): + item = YPropertyExpr(item.root, "expression", + item.content, item.override_path, + context=self.context) + # Ignore these here as they are used by search properties. if isinstance(item, YPropertyInput): continue diff --git a/hotsos/core/ycheck/engine/properties/common.py b/hotsos/core/ycheck/engine/properties/common.py index cd6041e21..b18348f6d 100644 --- a/hotsos/core/ycheck/engine/properties/common.py +++ b/hotsos/core/ycheck/engine/properties/common.py @@ -284,44 +284,19 @@ def __getattr__(self, key): return self.data.get(key) -class YPropertyBase(PTreeOverrideBase): - """ - Base class used for all YAML property objects. Implements and extends - PTreeOverrideBase to provide frequently used methods and helpers to - property implementations. - """ - def __init__(self, *args, **kwargs): - self._cache = PropertyCache() - super().__init__(*args, **kwargs) - - def resolve_var(self, name): - """ - Resolve variable with name to value. This can be used speculatively and - will return the name as value if it can't be resolved. - """ - if not name.startswith('$'): - return name - - if hasattr(self, 'context'): - if self.context.vars: - _name = name.partition('$')[2] - return self.context.vars.resolve(_name) +class PythonEntityResolver: + """A class to resolve Python entities (e.g. variable, property) + by their import path.""" - log.warning("could not resolve var '%s' - vars not found in " - "context", name) + # Class-level cache for Python module imports for future use - return name + import_cache = None - @property - def cache(self): - """ - All properties get their own cache object that they can use as they - wish. - """ - return self._cache + def __init__(self, context): + self.context = context def _load_from_import_cache(self, key): - """ Retrieve from global context if one exists. + """ Retrieve from global cache if one exists. @param key: key to retrieve """ @@ -374,16 +349,14 @@ def _add_to_import_cache(self, key, value): @param key: key to save @param value: value to save """ - if self.context is None: - log.info("context not available - cannot save '%s'", key) + + if not self.import_cache: + self.import_cache = {key: value} + log.debug("import cache initialized with initial key `%s`", key) return - c = getattr(self.context, 'import_cache') - if c: - c[key] = value - else: - c = {key: value} - setattr(self.context, 'import_cache', c) + self.import_cache[key] = value + log.debug("import cache updated for key %s", key) def get_cls(self, import_str): """ Import and instantiate Python class. @@ -535,6 +508,44 @@ def get_import(self, import_str): return self.get_attribute(import_str) +class YPropertyBase(PTreeOverrideBase, PythonEntityResolver): + """ + Base class used for all YAML property objects. Implements and extends + PTreeOverrideBase to provide frequently used methods and helpers to + property implementations. + """ + + def __init__(self, *args, **kwargs): + self._cache = PropertyCache() + super().__init__(*args, **kwargs) + + def resolve_var(self, name): + """ + Resolve variable with name to value. This can be used speculatively and + will return the name as value if it can't be resolved. + """ + if not name.startswith('$'): + return name + + if hasattr(self, 'context'): + if self.context.vars: + _name = name.partition('$')[2] + return self.context.vars.resolve(_name) + + log.warning("could not resolve var '%s' - vars not found in " + "context", name) + + return name + + @property + def cache(self): + """ + All properties get their own cache object that they can use as they + wish. + """ + return self._cache + + class YPropertyOverrideBase(YPropertyBase, PTreeOverrideBase): """ Base class for all simple/flat property objects. """ diff --git a/hotsos/core/ycheck/engine/properties/requires/requires.py b/hotsos/core/ycheck/engine/properties/requires/requires.py index 9e06313aa..b96de6fb6 100644 --- a/hotsos/core/ycheck/engine/properties/requires/requires.py +++ b/hotsos/core/ycheck/engine/properties/requires/requires.py @@ -13,6 +13,7 @@ property as rproperty, path, varops, + expr, ) CACHE_CHECK_KEY = '__PREVIOUSLY_CACHED_PROPERTY_TYPE' @@ -94,7 +95,8 @@ class YPropertyRequires(YPropertyMappedOverrideBase): systemd.YRequirementTypeSystemd, rproperty.YRequirementTypeProperty, path.YRequirementTypePath, - varops.YPropertyVarOps] + varops.YPropertyVarOps, + expr.YPropertyExpr,] # We want to be able to use this property both on its own and as a member # of other mapping properties e.g. Checks. The following setting enables # this. diff --git a/hotsos/core/ycheck/engine/properties/requires/types/expr.py b/hotsos/core/ycheck/engine/properties/requires/types/expr.py new file mode 100644 index 000000000..b153bfafa --- /dev/null +++ b/hotsos/core/ycheck/engine/properties/requires/types/expr.py @@ -0,0 +1,686 @@ +import os +from abc import abstractmethod +from typing import Callable, Iterable + +import pyparsing as pp +from hotsos.core.exceptions import ( + NotEnoughParametersError, + TooManyParametersError, + NoSuchPropertyError, + UnexpectedParameterError, +) +from hotsos.core.ycheck.engine.properties.common import PythonEntityResolver +from hotsos.core.ycheck.engine.properties.requires import ( + intercept_exception, + YRequirementTypeWithOpsBase, +) +from hotsos.core.config import HotSOSConfig +from hotsos.core.host_helpers.filestat import FileObj +from hotsos.core.host_helpers.systemd import SystemdHelper +from hotsos.core.host_helpers.config import IniConfigBase +from hotsos.core.host_helpers.ssl import SSLCertificate + +# Inspired from the pyparsing examples: +# https://github.com/pyparsing/pyparsing/blob/master/examples/eval_arith.py +# https://github.com/pyparsing/pyparsing/blob/master/examples/simpleBool.py +# Enables "packrat" parsing, which adds memoizing to the parsing logic. +# pylint: disable-next=no-value-for-parameter +pp.ParserElement.enablePackrat() + +# ___________________________________________________________________________ # + + +class YExprToken: + """Expression token.""" + + def __init__(self, tokens): + self.tokens = tokens + + def token(self, index): + return self.tokens[index] + + @abstractmethod + def eval(self): + """Inheriting class must implement this.""" + + @staticmethod + def operator_operands(tokenlist): + """generator to extract operator operands in pairs.""" + it = iter(tokenlist) + while 1: + try: + yield (next(it), next(it)) + except StopIteration: + break + + +# ___________________________________________________________________________ # + + +class YExprNotFound: + """Type for indicating a thing is not found. + Allows short-circuiting boolean functions to False, + e.g. systemd('svc-name', 'start_time_secs') > 123 would evaluate to + False if there's no such service named `svc-name`.""" + + def __init__(self, desc): + self.desc = desc + + def __repr__(self): + return f"<`{self.desc}` not found>" + + +class YExprInvalidArgumentException(pp.ParseFatalException): + """Exception to raise when a expression parsing error is occured.""" + + def __init__(self, s, loc, msg): + super().__init__(s, loc, f"invalid argument '{msg}'") + + +# ___________________________________________________________________________ # + + +class YExprLogicalOpBase(YExprToken): + """Base class for logical operators.""" + + logical_fn: Callable[[Iterable[bool]], bool] = lambda _: False + + def eval(self): + # Yield odd operands 'True' 'and' 'False' 'and' 'False' 'and' 'True' + # would yield 'True', 'False', 'False', 'True' + eval_exprs = (t.eval() for t in self.token(0)[::2]) + + return self.logical_fn(eval_exprs) + + +class YExprLogicalAnd(YExprLogicalOpBase): + """And operator. Binary. Supports chaining.""" + + logical_fn = all + + def __repr__(self): + return " and ".join([str(t.eval()) for t in self.token(0)[::2]]) + + +class YExprLogicalOr(YExprLogicalOpBase): + """Or operator. Binary. Supports chaining.""" + + logical_fn = any + + def __repr__(self): + return " or ".join([str(t.eval()) for t in self.token(0)[::2]]) + + +class YExprLogicalNot(YExprToken): + """Not operator. Unary.)""" + + def eval(self): + return not self.token(0)[1].eval() + + +# ___________________________________________________________________________ # + + +# pylint: disable-next=abstract-method +class YExprFnBase(YExprToken): + """Common base class for all function implementations.""" + + def arg(self, index): + args = self.tokens[1:][0] + if not len(args) >= (index + 1): + return None + return args[index] + + +class YExprFnLen(YExprFnBase): + """len(expr) function implementation.""" + + def eval(self): + v = self.arg(0).eval() + return len(v) if v else 0 + + +class YExprFnNot(YExprFnBase): + """not(expr) function implementation.""" + + def eval(self): + return not self.arg(0).eval() + + +class YExprFnFile(YExprFnBase): + """fstat(fname, prop[optional]) function implementation.""" + + def eval(self): + file_name = self.arg(0).eval() + fobj = FileObj(file_name) + + if fobj.exists: + if not self.arg(1): + return True + else: + return YExprNotFound(f"{file_name}") + + property_name = self.arg(1).eval() + + if hasattr(fobj, property_name): + return getattr(fobj, property_name) + + raise NoSuchPropertyError(f"Unknown file property {property_name}") + + +class YExprFnSystemd(YExprFnBase): + """systemd(unit_name, ...property) function implementation.""" + + def eval(self): + if not self.arg(0): + raise NotEnoughParametersError( + "systemd(...) function expects at least one argument." + ) + if self.arg(2): + raise TooManyParametersError( + "systemd(...) function expects at most two arguments." + ) + + service_name = self.arg(0).eval() + service_obj = SystemdHelper([service_name]).services.get(service_name) + + if service_obj: + if not self.arg(1): + return True + else: + return YExprNotFound(f"{service_name}") + + property_name = self.arg(1).eval() + + if hasattr(service_obj, property_name): + return getattr(service_obj, property_name) + + raise NoSuchPropertyError( + f"systemd service `{service_name}` object " + f"has no such property {property_name}" + ) + + +class YExprFnReadIni(YExprFnBase): + """read_ini funciton implementation.""" + + def eval(self): + + if not self.arg(1): + raise NotEnoughParametersError( + "read_ini(...) function expects at least two arguments." + ) + if self.arg(4): + raise TooManyParametersError( + "read_ini(...) function expects at most four arguments." + ) + + ini_path = self.arg(0).eval() + path = os.path.join(HotSOSConfig.data_root, ini_path) + ini_file = IniConfigBase(path) + if not ini_file.exists: + return YExprNotFound(f"{path}") + + key = self.arg(1).eval() + section = self.arg(2).eval() if self.arg(2) else None + value = ini_file.get(key, section, expand_to_list=False) + + if self.arg(3) and value is None: + return self.arg(3).eval() + + return value + + +class YExprFnReadCert(YExprFnBase): + """read_cert function implementation.""" + + def eval(self): + if not self.arg(0): + raise NotEnoughParametersError( + "cert(...) function expects at least on argument." + ) + if self.arg(2): + raise TooManyParametersError( + "cert(...) function expects at most two arguments." + ) + + cert_path = self.arg(0).eval() + try: + cert = SSLCertificate(cert_path) + except OSError: + return YExprNotFound(f"{cert_path}") + + # If no property is specified, then return True/False + # indicating that whether the cert file exist or not. + if not self.arg(1): + return cert is not None + + property_name = self.arg(1).eval() + + if hasattr(cert, property_name): + return getattr(cert, property_name) + + raise NoSuchPropertyError( + f"certificate `{cert_path}` object has no" + " such property {property_name}" + ) + + +# ___________________________________________________________________________ # + + +class YExprArgBoolean(YExprToken): + """Boolean argument type. Triggered by True and False + keywords (case-insensitive).""" + + def eval(self): + if self.token(0).lower() == "true": + return True + + if self.token(0).lower() == "false": + return False + + raise ValueError(f"Non-boolean string: {self.token(0)}") + + +class YExprArgNone(YExprToken): + """Boolean argument type. Triggered by None + keyword (case-insensitive).""" + + def eval(self): + return None + + +class YExprArgStringLiteral(YExprToken): + """String literal 'foo'. Triggered by single quotation mark ''.""" + + def eval(self): + return self.token(0) + + +class YExprArgInteger(YExprToken): + """Integer argument type. Triggered by [0-9]+""" + + def eval(self): + return int(self.token(0)) + + +class YExprArgFloat(YExprToken): + """Integer argument type. Triggered by [0-9]+.[0-9]+""" + + def eval(self): + return float(self.token(0)) + + +class YExprArgRuntimeVariable(YExprToken, PythonEntityResolver): + """Runtime variable argument type. Triggered by '@' symbol followed by + any non-whitespace character.""" + + def __init__(self, tokens, context): + YExprToken.__init__(self, tokens=tokens) + PythonEntityResolver.__init__(self, context=context) + + def eval(self): + # use PythonEntityResolver to retrieve value associated with + # the given name. + v = self.get_property(self.token(0)[1:]) + return v + + +# ___________________________________________________________________________ # + + +class YExprSignOp(YExprToken): + "Class to evaluate expressions with a leading + or - sign" + + def __init__(self, tokens): + super().__init__(tokens) + self.sign = self.token(0)[0] + + def eval(self): + mult = {"+": 1, "-": -1}[self.sign] + return mult * self.token(0)[1].eval() + + +class YExprPowerOp(YExprToken): + "Class to evaluate power expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Power operation expects at least 3 tokens.") + + if len(self.token(0)) % 2 == 0: + raise TooManyParametersError( + "Power requires odd amount of tokens.") + + result = self.token(0)[-1].eval() + for val in self.token(0)[-3::-2]: + operand = val.eval() + result = operand**result + return result + + +class YExprMulDivOp(YExprToken): + "Class to evaluate multiplication and division expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Mul/div operation expects at least 3 tokens." + ) + + if len(self.token(0)) % 2 == 0: + raise TooManyParametersError( + "Mul/div requires odd amount of tokens.") + + prod = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + if op == "*": + prod *= val.eval() + elif op == "/": + prod /= val.eval() + else: + raise NameError(f"Unrecognized operation {op}") + + return prod + + +class YExprAddSubOp(YExprToken): + "Class to evaluate addition and subtraction expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Add/sub operation expects at least 3 tokens." + ) + + if len(self.token(0)) % 2 == 0: + raise UnexpectedParameterError( + "Add/sub requires odd amount of tokens.") + + sum_v = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + if op == "+": + sum_v += val.eval() + elif op == "-": + sum_v -= val.eval() + else: + raise NameError(f"Unrecognized operation {op}") + return sum_v + + +class YExprComparisonOp(YExprToken): + "Class to evaluate comparison expressions" + + ops = { + "<": lambda lhs, rhs: lhs < rhs, + "<=": lambda lhs, rhs: lhs <= rhs, + ">": lambda lhs, rhs: lhs > rhs, + ">=": lambda lhs, rhs: lhs >= rhs, + "!=": lambda lhs, rhs: lhs != rhs, + "==": lambda lhs, rhs: lhs == rhs, + # pylint: disable=unnecessary-lambda + "LT": lambda lhs, rhs: YExprComparisonOp.ops["<"](lhs, rhs), + "LE": lambda lhs, rhs: YExprComparisonOp.ops["<="](lhs, rhs), + "GT": lambda lhs, rhs: YExprComparisonOp.ops[">"](lhs, rhs), + "GE": lambda lhs, rhs: YExprComparisonOp.ops[">="](lhs, rhs), + "NE": lambda lhs, rhs: YExprComparisonOp.ops["!="](lhs, rhs), + "EQ": lambda lhs, rhs: YExprComparisonOp.ops["=="](lhs, rhs), + "<>": lambda lhs, rhs: YExprComparisonOp.ops["!="](lhs, rhs), + # pylint:enable=unnecessary-lambda + "IN": lambda lhs, rhs: lhs in rhs, + } + + def eval(self): + lhs = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + op_fn = self.ops[op] + rhs = val.eval() + + # if either of the operands is not found, return False. + if any(isinstance(x, YExprNotFound) for x in [lhs, rhs]): + return False + + if not op_fn(lhs, rhs): + break + lhs = rhs + else: + return True + return False + + +# ___________________________________________________________________________ # + + +def _tok_error(exception_type): + """Parser matcher type for raising parse errors.""" + + def raise_exception(s, loc, typ): + raise exception_type(s, loc, typ[0]) + + return pp.Word(pp.printables).setParseAction(raise_exception) + + +def _tok_boolean_kw(): + # Define True & False as their corresponding bool values + # Example: [True, False, TRUE, FALSE, TrUe, FaLsE] + kw = pp.CaselessKeyword("True") | pp.CaselessKeyword("False") + kw.setParseAction(lambda s, loc, tokens: YExprArgBoolean(tokens)) + return kw + + +def _tok_none_kw(): + # Define `None` as keyword for None + kw = pp.CaselessKeyword("None") + kw.setParseAction(lambda s, loc, tokens: YExprArgNone(tokens)) + return kw + + +def _tok_string_literal(): + # Declare syntax for string literals + # Example: ['this is a test'] + kw = pp.QuotedString("'") + kw.setParseAction(lambda s, loc, tokens: YExprArgStringLiteral(tokens)) + return kw + + +def _tok_integer(): + # Declare syntax for integers + # example: [123, 1, 1234] + kw = pp.Word(pp.nums) + kw.setParseAction(lambda s, loc, tokens: YExprArgInteger(tokens)) + return kw + + +def _tok_real(): + # Declare syntax for real numbers (float) + # example. [1.3, 1.23] + kw = pp.Combine(pp.Word(pp.nums) + "." + pp.Word(pp.nums)) + kw.setParseAction(lambda s, loc, tokens: YExprArgFloat(tokens)) + return kw + + +def _tok_python_property(context): + # Declare syntax for Python runtime properties. + # Properties start with `@` symbol and can contain alphanumeric + '.', '_'` + # example. [@hotsos.module.class.property_1] + kw = pp.Combine("@" + pp.Word(pp.alphanums + "._-:/")) + kw.setParseAction( + lambda s, loc, tokens: YExprArgRuntimeVariable(tokens, context)) + return kw + + +def _make_fn_token(expr, name, parser): + lpar, rpar = map(pp.Suppress, "()") + function_call_tail = pp.Group( + lpar + pp.Optional(pp.delimited_list(expr)) + rpar) + fn = pp.CaselessKeyword(name) + function_call_tail + fn.setParseAction(lambda s, loc, tokens: parser(tokens)) + return fn + + +def _tok_functions(expr): + return ( + _make_fn_token(expr, "len", YExprFnLen) + | _make_fn_token(expr, "not", YExprFnNot) + | _make_fn_token(expr, "file", YExprFnFile) + | _make_fn_token(expr, "systemd", YExprFnSystemd) + | _make_fn_token(expr, "read_ini", YExprFnReadIni) + | _make_fn_token(expr, "read_cert", YExprFnReadCert) + ) + + +def _tok_arith_expr(base_expr): + # Declare arithmetic operations + signop = pp.one_of("+ -") + multop = pp.one_of("* /") + plusop = pp.one_of("+ -") + expop = pp.Literal("**") + arith_expr = pp.infix_notation( + base_expr, + [ + ( + signop, + 1, + pp.OpAssoc.RIGHT, + lambda s, loc, tokens: YExprSignOp(tokens), + ), + ( + expop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprPowerOp(tokens), + ), + ( + multop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprMulDivOp(tokens), + ), + ( + plusop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprAddSubOp(tokens), + ), + ], + ) + return arith_expr + + +def _tok_comp_expr(base_expr): + # Declare comparison/boolean operations + comparisonop = pp.one_of( + " ".join(YExprComparisonOp.ops.keys()), caseless=True) + comp_expr = pp.infix_notation( + base_expr, + [ + ( + comparisonop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprComparisonOp(tokens), + ), + ], + ) + return comp_expr + + +def _tok_logical_expr(base_expr): + logical_expr = pp.infix_notation( + base_expr, + [ + ( + pp.CaselessKeyword("not"), + 1, + pp.OpAssoc.RIGHT, + lambda s, loc, tokens: YExprLogicalNot(tokens), + ), + ( + pp.CaselessKeyword("and"), + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprLogicalAnd(tokens), + ), + ( + pp.CaselessKeyword("or"), + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprLogicalOr(tokens), + ), + ], + ) + return logical_expr + + +def init_parser(context=None): + """Initialize parser for check expressions. + + The grammar currently supports the following constructs: + + Keywords: + None, True, False + built-ins: + integer, float, string literal + runtime: + python properties + functions: + len(...) + not(...) + arithmetic: + - sign + - plus/minus + - exponent + - mul/div + boolean: + - gt/ge + - lt/le + - eq/ne + - in/and/or/not + """ + + # This is a forward declaration because functions can take an expression as + # an argument.. + expr = pp.Forward() + + # The order matters. + keywords = _tok_boolean_kw() | _tok_none_kw() + constants = _tok_real() | _tok_integer() | _tok_string_literal() + operand = ( + _tok_functions(expr) | keywords | + constants | _tok_python_property(context) + ) + + arith_expr = _tok_arith_expr(operand) + comp_expr = _tok_comp_expr(arith_expr) + logical_expr = _tok_logical_expr(comp_expr) + + # Append all of them to "expr". Anything that does not match + # to the comp_expr is an error. + expr <<= logical_expr | _tok_error(YExprInvalidArgumentException) + # Ignore comments. + expr.ignore(pp.python_style_comment) + expr.ignore(pp.c_style_comment) + return expr + + +class YPropertyExpr(YRequirementTypeWithOpsBase): + """Expression requirement property type.""" + + _override_keys = ["expression"] + _overrride_autoregister = True + + @property + def input(self): + return self.content + + @property + @intercept_exception + def _result(self): + parser = init_parser(context=self.context) + parsed_expr = parser.parse_string(self.input) + result = parsed_expr[0].eval() + if isinstance(result, YExprNotFound): + return False + return result diff --git a/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml b/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml index 55972f407..f9078f203 100644 --- a/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml +++ b/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml @@ -1,8 +1,6 @@ checks: - jujud_not_found: - property: - path: hotsos.core.plugins.juju.JujuChecks.systemd_processes - ops: [[contains, jujud], [not_]] + jujud_not_found: | + NOT 'jujud' IN @hotsos.core.plugins.juju.JujuChecks.systemd_processes conclusions: jujud-not-found: decision: jujud_not_found diff --git a/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml b/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml index 3861eedac..51b0443ac 100644 --- a/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml +++ b/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml @@ -1,20 +1,12 @@ -vars: - virt_type: '@hotsos.core.plugins.system.SystemBase.virtualisation_type' - cpu_vendor: '@hotsos.core.plugins.kernel.sysfs.CPU.vendor' - kernel_cmd_line: '@hotsos.core.plugins.kernel.KernelBase.boot_parameters' checks: - is_phy_host: - varops: [[$virt_type], [not_]] - cpu_vendor_is_amd: - varops: [[$cpu_vendor], [eq, 'authenticamd']] - iommu_not_pt: - varops: [[$kernel_cmd_line], [contains, 'iommu=pt'], [not_]] + not_using_iommu_passthrough_for_suitable_amd: | + (@hotsos.core.plugins.kernel.sysfs.CPU.vendor == 'authenticamd') AND + (@hotsos.core.plugins.system.SystemBase.virtualisation_type == None) AND + NOT('iommu=pt' IN @hotsos.core.plugins.kernel.KernelBase.boot_parameters) conclusions: mixed-pkg-releases: decision: - - is_phy_host - - cpu_vendor_is_amd - - iommu_not_pt + - not_using_iommu_passthrough_for_suitable_amd raises: type: SystemWarning message: >- diff --git a/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml b/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml index df3c596ee..cdb9f885e 100644 --- a/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml +++ b/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml @@ -1,24 +1,14 @@ checks: - has_stacktraces: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype - ops: [[length_hint]] - has_oom_killer_invoked: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.oom_killer - ops: [[length_hint]] - has_bcache_deadlock_invoked: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace-bcache - ops: [[length_hint]] - has_hungtasks: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask - ops: [[length_hint]] - has_fanotify_hang: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify - ops: [[length_hint]] + has_stacktraces: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype) > 0 + has_oom_killer_invoked: | + len(@hotsos.core.plugins.kernel.CallTraceManager.oom_killer) > 0 + has_bcache_deadlock_invoked: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace-bcache) > 0 + has_hungtasks: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask) > 0 + has_fanotify_hang: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify) > 0 conclusions: stacktraces: # Give this one lowest priority so that if any other call trace types match they @@ -30,7 +20,7 @@ conclusions: message: >- {numreports} reports of stacktraces in kern.log - please check. format-dict: - numreports: '@checks.has_stacktraces.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype:len oom-killer-invoked: priority: 2 decision: has_oom_killer_invoked @@ -39,7 +29,7 @@ conclusions: message: >- {numreports} reports of oom-killer invoked in kern.log - please check. format-dict: - numreports: '@checks.has_oom_killer_invoked.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.oom_killer:len bcache-deadlock-invoked: priority: 2 decision: has_bcache_deadlock_invoked @@ -60,7 +50,7 @@ conclusions: message: >- {numreports} reports of hung tasks in kern.log - please check. format-dict: - numreports: '@checks.has_hungtasks.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask:len fanotify_hangs: priority: 2 decision: has_fanotify_hang @@ -70,4 +60,4 @@ conclusions: {numreports} reports of fanotify related hangs in kern.log. This may be related to antivirus software running in the system. format-dict: - numreports: '@checks.has_fanotify_hang.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify:len diff --git a/hotsos/defs/scenarios/kernel/memory.yaml b/hotsos/defs/scenarios/kernel/memory.yaml index 49ef9ac6b..3df9dd1ae 100644 --- a/hotsos/defs/scenarios/kernel/memory.yaml +++ b/hotsos/defs/scenarios/kernel/memory.yaml @@ -4,11 +4,6 @@ vars: compact_success: '@hotsos.core.plugins.kernel.memory.VMStat.compact_success' compaction_failures_percent: '@hotsos.core.plugins.kernel.memory.VMStat.compaction_failures_percent' slab_major_consumers: '@hotsos.core.plugins.kernel.memory.SlabInfo.major_consumers' - # We use an arbitrary threshold of 10k to suggest that a lot of - # compaction has occurred but noting that this is a rolling counter - # and is not necessarily representative of current state. - min_compaction_success: 10000 - max_compaction_failures_pcent: 10 hugetlb_to_mem_total_percentage: '@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_to_mem_total_percentage' mem_avail_to_mem_total_percentage: @@ -18,20 +13,21 @@ vars: mem_total_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.mem_total_gb' mem_available_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.mem_available_gb' hugetlb_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_gb' - # Arbitrary thresholds set for the memory allocated for the huge - # pages to total memory and memory available to total memory. - hugetlb_to_mem_total_threshold_percent: 80 - mem_available_to_mem_total_thershold_percent: 3 checks: - low_free_high_order_mem_blocks: - varops: [[$nodes_with_limited_high_order_memory], [length_hint]] - high_compaction_failures: - - varops: [[$compact_success], [gt, $min_compaction_success]] - - varops: [[$compaction_failures_percent], [gt, $max_compaction_failures_pcent]] - too_many_free_hugepages: - - property: hotsos.core.plugins.kernel.memory.MemInfo.huge_pages_enabled - - varops: [[$hugetlb_to_mem_total_percentage], [gt, $hugetlb_to_mem_total_threshold_percent]] - - varops: [[$mem_avail_to_mem_total_percentage], [lt, $mem_available_to_mem_total_thershold_percent]] + low_free_high_order_mem_blocks: | + len(@hotsos.core.plugins.kernel.memory.MemoryChecks.nodes_with_limited_high_order_memory) > 0 + high_compaction_failures: | + # We use an arbitrary threshold of 10k to suggest that a lot of + # compaction has occurred but noting that this is a rolling counter + # and is not necessarily representative of current state. + (@hotsos.core.plugins.kernel.memory.VMStat.compact_success > 10000) AND + (@hotsos.core.plugins.kernel.memory.VMStat.compaction_failures_percent > 10) + too_many_free_hugepages: | + (@hotsos.core.plugins.kernel.memory.MemInfo.huge_pages_enabled == True) AND + # Arbitrary thresholds set for the memory allocated for the huge + # pages to total memory and memory available to total memory. + (@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_to_mem_total_percentage > 80) AND + (@hotsos.core.plugins.kernel.memory.MemInfo.mem_avail_to_mem_total_percentage < 3) conclusions: low_free_high_order_mem_blocks: decision: low_free_high_order_mem_blocks diff --git a/hotsos/defs/scenarios/kernel/network/misc.yaml b/hotsos/defs/scenarios/kernel/network/misc.yaml index 1243c624d..a69389011 100644 --- a/hotsos/defs/scenarios/kernel/network/misc.yaml +++ b/hotsos/defs/scenarios/kernel/network/misc.yaml @@ -7,8 +7,8 @@ checks: # or # "Jun 08 10:48:13 compute4 kernel:" expr: '(\w{3,5}\s+\d{1,2}\s+[\d:]+)\S+.+ nf_conntrack: table full, dropping packet' - has_over_mtu_dropped_packets: - property: hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets + has_over_mtu_dropped_packets: | + len(@hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets) > 0 conclusions: nf-conntrack-full: decision: has_nf_conntrack_full @@ -27,4 +27,4 @@ conclusions: This host is reporting over-mtu dropped packets for ({num_ifaces}) interfaces. See kern.log for full details. format-dict: - num_ifaces: '@checks.has_over_mtu_dropped_packets.requires.value_actual:len' + num_ifaces: hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets:len diff --git a/hotsos/defs/scenarios/kernel/network/netlink.yaml b/hotsos/defs/scenarios/kernel/network/netlink.yaml index e67907faa..688252fed 100644 --- a/hotsos/defs/scenarios/kernel/network/netlink.yaml +++ b/hotsos/defs/scenarios/kernel/network/netlink.yaml @@ -1,6 +1,6 @@ checks: - has_socks_with_drops: - property: hotsos.core.plugins.kernel.net.NetLink.all_with_drops + has_socks_with_drops: | + len(@hotsos.core.plugins.kernel.net.NetLink.all_with_drops) > 0 conclusions: netlink-socks-with-drops: decision: has_socks_with_drops diff --git a/hotsos/defs/scenarios/kernel/network/tcp.yaml b/hotsos/defs/scenarios/kernel/network/tcp.yaml index c57c7c2d9..635bff764 100644 --- a/hotsos/defs/scenarios/kernel/network/tcp.yaml +++ b/hotsos/defs/scenarios/kernel/network/tcp.yaml @@ -1,71 +1,42 @@ -vars: - incsumerr: '@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors' - incsumrate_pcent: '@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs' - outsegs: '@hotsos.core.plugins.kernel.net.SNMPTcp.OutSegs' - retrans: '@hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegs' - outretrans_pcent: '@hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs' - spurrtx: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueues' - spurrtx_pcent: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs' - prunec: '@hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled' - rcvcoll: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvCollapsed' - rcvpr: '@hotsos.core.plugins.kernel.net.NetStatTCP.RcvPruned' - ofopr: '@hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned' - backlogd: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop' - rpfilterd: '@hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter' - ldrop: '@hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops' - pfmemd: '@hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop' - minttld: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop' - listenovf: '@hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows' - ofod: '@hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned' - zwind: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop' - rcvqd: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop' - rcvqd_pcent: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs' - rqfulld: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop' - rqfullcook: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies' - memusage_pages_inuse: '@hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages' - memusage_pages_max: '@hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax' - memusage_pct: '@hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct' checks: - incsumerr_high: - or: - - varops: [[$incsumerr], [gt, 500]] - - varops: [[$incsumrate_pcent], [gt, 1]] - retrans_out_rate_gt_1pcent: - varops: [[$outretrans_pcent], [gt, 1]] - incsumrate_gt_1pcent: - varops: [[$incsumrate_pcent], [gt, 1]] - mem_pressure_prunec: - varops: [[$prunec], [gt, 500]] - backlogd: - varops: [[$backlogd], [gt, 500]] - rpfilterd: - varops: [[$rpfilterd], [gt, 500]] - ldrop: - varops: [[$ldrop], [gt, 500]] - spurrtx_gt_1pcent: - varops: [[$spurrtx_pcent], [gt, 1]] - pfmemd: - varops: [[$pfmemd], [gt, 500]] - minttld: - varops: [[$minttld], [gt, 500]] - listenovf: - varops: [[$listenovf], [gt, 500]] - ofod: - varops: [[$ofod], [gt, 500]] - zwind: - varops: [[$zwind], [gt, 500]] - rcvqd: - or: - - varops: [[$rcvqd], [gt, 500]] - - varops: [[$rcvqd_pcent], [gt, 1]] - rqfulld: - varops: [[$rqfulld], [gt, 500]] - rqfullcook: - varops: [[$rqfullcook], [gt, 500]] - memusage_pct_high: - varops: [[$memusage_pct], [gt, 85]] - memusage_pct_exhausted: - varops: [[$memusage_pct], [ge, 100]] + incsumerr_high: | + (@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors > 500) OR + (@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs > 1) + retrans_out_rate_gt_1pcent: | + @hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs > 1 + incsumrate_gt_1pcent: | + @hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs > 1 + mem_pressure_prunec: | + @hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled > 500 + backlogd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop > 500 + rpfilterd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter > 500 + ldrop: | + @hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops > 500 + spurrtx_gt_1pcent: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs > 1 + pfmemd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop > 500 + minttld: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop > 500 + listenovf: | + @hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows > 500 + ofod: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPOFODrop > 500 + zwind: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop > 500 + rcvqd: | + (@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop > 500) OR + (@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs > 1) + rqfulld: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop > 500 + rqfullcook: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies > 500 + memusage_pct_high: | + @hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct > 85 + memusage_pct_exhausted: | + @hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct >= 100 conclusions: # retransmissions incsumerr_high: @@ -76,23 +47,23 @@ conclusions: tcp ingress checksum errors are at {count}. This could mean that one or more interfaces are experiencing hardware errors. format-dict: - count: $incsumerr + count: hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors retrans_out_rate_gt_1pcent: decision: retrans_out_rate_gt_1pcent raises: type: KernelWarning message: tcp retransmissions ({retrans}) are at {rate}% of tx segments ({outsegs}). format-dict: - outsegs: $outsegs - retrans: $retrans - rate: $outretrans_pcent + outsegs: hotsos.core.plugins.kernel.net.SNMPTcp.OutSegs + retrans: hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegs + rate: hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs incsumrate_gt_1pcent: decision: incsumrate_gt_1pcent raises: type: KernelWarning message: tcp ingress checksum error rate is at {rate}% of rx segments. format-dict: - rate: $incsumrate_pcent + rate: hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs # mem pressure mem_pressure_prunec: decision: mem_pressure_prunec @@ -102,10 +73,10 @@ conclusions: tcp pruned {prunec} times: collapsed={rcvcoll} recv={rcvpr} ofo={ofopr}. format-dict: - prunec: $prunec - rcvcoll: $rcvcoll - rcvpr: $rcvpr - ofopr: $ofopr + prunec: hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled + rcvcoll: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvCollapsed + rcvpr: hotsos.core.plugins.kernel.net.NetStatTCP.RcvPruned + ofopr: hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned # misc drops backlogd: decision: backlogd @@ -113,64 +84,64 @@ conclusions: type: KernelWarning message: tcp socket backlog queue full (drops={count}). format-dict: - count: $backlogd + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop pfmemd: decision: pfmemd raises: type: KernelWarning message: PFMEMALLOC skb to non-MEMALLOC socket (drops={count}). format-dict: - count: $pfmemd + count: hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop minttld: decision: minttld raises: type: KernelWarning message: IP TTL below minimum (drops={count}). format-dict: - count: $minttld + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop rpfilterd: decision: rpfilterd raises: type: KernelWarning message: Failed reverse path filter test (drops={count}). format-dict: - count: $rpfilterd + count: hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter listenovf: decision: listenovf raises: type: KernelWarning message: tcp accept queue overflow (drops={count}). format-dict: - count: $listenovf + count: hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows ldrop: decision: ldrop raises: type: KernelWarning message: tcp incoming connect request catch-all (drops={count}). format-dict: - count: $ldrop + count: hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops ofod: decision: ofod raises: type: KernelWarning message: tcp no rmem adding to OFO recv queue (drops={count}). format-dict: - count: $ofod + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPOFODrop zwind: decision: zwind raises: type: KernelWarning message: tcp receive window full (drops={count}). format-dict: - count: $zwind + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop rcvqd: decision: rcvqd raises: type: KernelWarning message: tcp no rmem adding to recv queue (drops={count}, {pcent})% of total rx). format-dict: - count: $rcvqd - pcent: $rcvqd_pcent + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop + pcent: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs # synflood rqfulld: decision: rqfulld @@ -178,14 +149,14 @@ conclusions: type: KernelWarning message: tcp request queue full, syncookies off (drops={count}). format-dict: - count: $rqfulld + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop rqfullcook: decision: rqfullcook raises: type: KernelWarning message: tcp request queue full, syncookies on (cookies={count}). format-dict: - count: $rqfullcook + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies # misc spurrtx_gt_1pcent: decision: spurrtx_gt_1pcent @@ -194,8 +165,8 @@ conclusions: message: >- this host has {num} tcp retransmissions ({pcent}% of total) with original still queued. format-dict: - num: $spurrtx - pcent: $spurrtx_pcent + num: hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueues + pcent: hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs memusage_pct_high: decision: - memusage_pct_high @@ -206,9 +177,9 @@ conclusions: TCP memory page usage is high ({pct}%), ({pa} out of {pm} mem pages are in use). Kernel will start to drop TCP frames if all pages are exhausted. format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct memusage_pct_exhausted: decision: memusage_pct_exhausted raises: @@ -217,6 +188,6 @@ conclusions: All TCP memory pages are exhausted! ({pa} out of {pm} mem pages are in use). Kernel will drop TCP packets, expect packet losses on ALL TCP transport! format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct diff --git a/hotsos/defs/scenarios/kernel/network/udp.yaml b/hotsos/defs/scenarios/kernel/network/udp.yaml index d970562ba..08b84fae2 100644 --- a/hotsos/defs/scenarios/kernel/network/udp.yaml +++ b/hotsos/defs/scenarios/kernel/network/udp.yaml @@ -1,36 +1,20 @@ -vars: - inerrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.InErrors' - inerrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams' - rcvbuferrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors' - rcvbuferrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams' - sndbuferrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors' - sndbuferrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams' - incsumerrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors' - incsumerrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams' - memusage_pages_inuse: '@hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages' - memusage_pages_max: '@hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax' - memusage_pct: '@hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct' checks: - rcvbuferrors_high: - or: - - varops: [[$rcvbuferrors], [gt, 500]] - - varops: [[$rcvbuferrors_pcent], [gt, 1]] - sndbuferrors_high: - or: - - varops: [[$sndbuferrors], [gt, 500]] - - varops: [[$sndbuferrors_pcent], [gt, 1]] - inerrs_high_pcent_or_above_limit: - or: - - varops: [[$inerrors], [gt, 500]] - - varops: [[$inerrors_pcent], [gt, 1]] - incsumerrs_high_or_above_limit: - or: - - varops: [[$incsumerrors], [gt, 500]] - - varops: [[$incsumerrors_pcent], [gt, 1]] - memusage_pct_high: - varops: [[$memusage_pct], [gt, 85]] - memusage_pct_exhausted: - varops: [[$memusage_pct], [ge, 100]] + rcvbuferrors_high: | + @hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams > 1 + sndbuferrors_high: | + @hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams > 1 + inerrs_high_pcent_or_above_limit: | + @hotsos.core.plugins.kernel.net.SNMPUdp.InErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams > 1 + incsumerrs_high_or_above_limit: | + @hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams > 1 + memusage_pct_high: | + @hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct > 85 + memusage_pct_exhausted: | + @hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct >= 100 conclusions: inerrs_high_pcent_or_above_limit: decision: inerrs_high_pcent_or_above_limit @@ -39,8 +23,8 @@ conclusions: message: >- UDP ingress errors are at {count} ({pcent}% of total datagrams). format-dict: - count: $inerrors - pcent: $inerrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.InErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams incsumerrs_high_or_above_limit: decision: incsumerrs_high_or_above_limit raises: @@ -49,8 +33,8 @@ conclusions: UDP ingress checksum errors are at {count} ({pcent}% of total datagrams). This could mean that one or more interfaces are experiencing hardware errors. format-dict: - count: $incsumerrors - pcent: $incsumerrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams rcvbuferrors_high: decision: rcvbuferrors_high raises: @@ -58,8 +42,8 @@ conclusions: message: >- UDP receive buffer errors are at {count} ({pcent}% of total rx). format-dict: - count: $rcvbuferrors - pcent: $rcvbuferrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams sndbuferrors_high: decision: sndbuferrors_high raises: @@ -67,8 +51,8 @@ conclusions: message: >- UDP send buffer errors are at {count} ({pcent}% of total tx). format-dict: - count: $sndbuferrors - pcent: $sndbuferrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams memusage_pct_high: decision: - memusage_pct_high @@ -79,9 +63,9 @@ conclusions: UDP memory page usage is high ({pct}%), ({pa} out of {pm} mem pages are in use). Kernel will start to drop UDP frames if all pages are exhausted. format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct memusage_pct_exhausted: decision: memusage_pct_exhausted raises: @@ -90,6 +74,6 @@ conclusions: All UDP memory pages are exhausted! ({pa} out of {pm} mem pages are in use). Kernel will drop UDP packets, expect packet losses on ALL UDP transport! format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct diff --git a/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml b/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml index 5959e2d1b..b67d05544 100644 --- a/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml +++ b/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml @@ -11,17 +11,14 @@ vars: order for changes to persist. scaling_governor: '@hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all' checks: - cpufreq_governor_not_performance: - # can we actually see the setting - - varops: [[$scaling_governor], [ne, unknown]] - # does it have the expected value - - varops: [[$scaling_governor], [ne, performance]] + cpufreq_governor_not_performance: | + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'unknown' AND + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'performance' kubernetes_installed: - snap: kubelet # ignore if not running on metal - - property: - path: hotsos.core.plugins.system.system.SystemBase.virtualisation_type - ops: [[eq, null]] + - expression: | + @hotsos.core.plugins.system.system.SystemBase.virtualisation_type == None ondemand_installed_and_enabled: systemd: ondemand: enabled diff --git a/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml b/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml index fc4f7cfa6..c5e49aa30 100644 --- a/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml +++ b/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml @@ -1,12 +1,8 @@ checks: - is_not_a_lxc_container: - property: - path: hotsos.core.plugins.system.SystemBase.virtualisation_type - ops: [[ne, 'lxc']] - has_lxc_containers: - property: - path: hotsos.core.plugins.lxd.LXD.instances - ops: [[length_hint], [gt, 0]] + is_not_a_lxc_container: | + @hotsos.core.plugins.system.SystemBase.virtualisation_type != 'lxc' + has_lxc_containers: | + len(@hotsos.core.plugins.lxd.LXD.instances) > 0 has_lxd_version_5_9: snap: lxd: diff --git a/hotsos/defs/scenarios/openstack/eol.yaml b/hotsos/defs/scenarios/openstack/eol.yaml index 4293982df..34a118e5a 100644 --- a/hotsos/defs/scenarios/openstack/eol.yaml +++ b/hotsos/defs/scenarios/openstack/eol.yaml @@ -1,8 +1,6 @@ checks: - is_eol: - property: - path: hotsos.core.plugins.openstack.OpenstackBase.days_to_eol - ops: [[le, 0]] + is_eol: | + @hotsos.core.plugins.openstack.OpenstackBase.days_to_eol <= 0 conclusions: is-eol: decision: is_eol diff --git a/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml b/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml index 480b09b3a..389c1ccb7 100644 --- a/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml +++ b/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml @@ -1,10 +1,8 @@ checks: - ssl_enabled: - property: hotsos.core.plugins.openstack.OpenstackBase.ssl_enabled - apache2_certificate_expiring: - property: - path: hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring - ops: [[ne, []]] + ssl_enabled: | + @hotsos.core.plugins.openstack.OpenstackBase.ssl_enabled == True + apache2_certificate_expiring: | + len(@hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring) > 0 conclusions: need-certificate-renewal: decision: @@ -16,5 +14,5 @@ conclusions: The following certificates will expire in less than {apache2-certificates-days-to-expire} days: {apache2-certificates-path} format-dict: - apache2-certificates-path: '@checks.apache2_certificate_expiring.requires.value_actual:comma_join' - apache2-certificates-days-to-expire: 'hotsos.core.plugins.openstack.OpenstackBase.certificate_expire_days' + apache2-certificates-path: hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring:comma_join + apache2-certificates-days-to-expire: hotsos.core.plugins.openstack.OpenstackBase.certificate_expire_days diff --git a/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml b/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml index 63c2d26b5..ae4d32eb1 100644 --- a/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml +++ b/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml @@ -1,12 +1,10 @@ -vars: - local_charms: '@hotsos.core.plugins.juju.JujuChecks.charms' checks: - neutron_conflicts: - - varops: [[$local_charms], [contains, neutron-api]] - - varops: [[$local_charms], [contains, neutron-gateway]] - nova_conflicts: - - varops: [[$local_charms], [contains, nova-cloud-controller]] - - varops: [[$local_charms], [contains, nova-compute]] + neutron_conflicts: | + 'neutron-api' IN @hotsos.core.plugins.juju.JujuChecks.charms AND + 'neutron-gateway' IN @hotsos.core.plugins.juju.JujuChecks.charms + nova_conflicts: | + 'nova-cloud-controller' IN @hotsos.core.plugins.juju.JujuChecks.charms and + 'nova-compute' IN @hotsos.core.plugins.juju.JujuChecks.charms conclusions: neutron_charm_conflicts: decision: neutron_conflicts diff --git a/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml b/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml index 03f95edf6..7d0936f51 100644 --- a/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml +++ b/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml @@ -1,8 +1,6 @@ checks: - has_mixed_pkg_releases: - property: - path: hotsos.core.plugins.openstack.OpenstackBase.installed_pkg_release_names - ops: [[length_hint], [gt, 1]] + has_mixed_pkg_releases: | + len(@hotsos.core.plugins.openstack.OpenstackBase.installed_pkg_release_names) > 1 conclusions: mixed-pkg-releases: decision: has_mixed_pkg_releases @@ -11,4 +9,4 @@ conclusions: message: >- Openstack packages from a more than one release identified: {releases}. format-dict: - releases: '@checks.has_mixed_pkg_releases.requires.value_actual:comma_join' + releases: hotsos.core.plugins.openstack.OpenstackBase.installed_pkg_release_names:comma_join diff --git a/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml b/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml index e83c12db6..ed8a1c000 100644 --- a/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml +++ b/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml @@ -12,13 +12,10 @@ vars: msg_ondemand: >- You will also need to stop and disable the ondemand systemd service in order for changes to persist. - scaling_governor: '@hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all' checks: - cpufreq_governor_not_performance: - # can we actually see the setting - - varops: [[$scaling_governor], [ne, unknown]] - # does it have the expected value - - varops: [[$scaling_governor], [ne, performance]] + cpufreq_governor_not_performance: | + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'unknown' and + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'performance' is_neutron_gateway: # i.e. neutron-l3-agent but no nova == neutron gateway - not: @@ -42,7 +39,7 @@ conclusions: format-dict: msg_pt1: $message_pt1_nova_compute msg_pt2: $message_pt2 - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-n-gw: priority: 1 decision: @@ -55,7 +52,7 @@ conclusions: format-dict: msg_pt1: $message_pt1_neutron_gateway msg_pt2: $message_pt2 - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-with-ondemand-n-cpu: priority: 2 decision: @@ -70,7 +67,7 @@ conclusions: msg_pt1: $message_pt1_nova_compute msg_pt2: $message_pt2 msg_ondemand: $msg_ondemand - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-with-ondemand-n-gw: priority: 2 decision: @@ -85,4 +82,4 @@ conclusions: msg_pt1: $message_pt1_neutron_gateway msg_pt2: $message_pt2 msg_ondemand: $msg_ondemand - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all diff --git a/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml b/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml index 7ba57f773..b898ab1b7 100644 --- a/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml +++ b/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml @@ -1,8 +1,6 @@ checks: - has_unexpected_masked: - property: - path: hotsos.core.plugins.openstack.OpenstackBase.unexpected_masked_services - ops: [[ne, []]] + has_unexpected_masked: | + len(@hotsos.core.plugins.openstack.OpenstackBase.unexpected_masked_services) > 0 conclusions: has-unexpected-masked: decision: has_unexpected_masked @@ -13,4 +11,4 @@ conclusions: ensure that this is intended otherwise these services may be unavailable. format-dict: - masked: '@checks.has_unexpected_masked.requires.value_actual:comma_join' + masked: hotsos.core.plugins.openstack.OpenstackBase.unexpected_masked_services:comma_join diff --git a/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml b/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml index 5c7af947e..75497ae3d 100644 --- a/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml +++ b/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml @@ -7,21 +7,30 @@ vars: cert_expired_expr: '([\d-]+)T([\d:]+)\.\d+Z\|\S+\|stream_ssl\|WARN\|SSL_accept: error:\S+:SSL routines:ssl3_read_bytes:sslv3 alert certificate expired' cert_invalid_expr: '([\d-]+)T([\d:]+)\.\d+Z\|\S+\|stream_ssl\|WARN\|SSL_accept: error:\S+:SSL routines:tls_process_client_certificate:certificate verify failed' checks: - northd_not_restarted_after_cert_update: - - systemd: ovn-northd - - varops: [[$northd_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $northd_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $northd_start_time]] - nbdb_not_restarted_after_cert_update: - - systemd: ovn-ovsdb-server-nb - - varops: [[$ovsdb_nb_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $ovsdb_nb_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_nb_start_time]] - sbdb_not_restarted_after_cert_update: - - systemd: ovn-ovsdb-server-sb - - varops: [[$ovsdb_sb_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $ovsdb_sb_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_sb_start_time]] + northd_not_restarted_after_cert_update: | + systemd('ovn-northd') AND systemd('ovn-northd', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-northd', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-northd', 'start_time_secs') + # - systemd: ovn-northd + # - varops: [[$northd_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $northd_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $northd_start_time]] + nbdb_not_restarted_after_cert_update: | + systemd('ovn-ovsdb-server-nb') AND systemd('ovn-ovsdb-server-nb', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-ovsdb-server-nb', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-ovsdb-server-nb', 'start_time_secs') + # - systemd: ovn-ovsdb-server-nb + # - varops: [[$ovsdb_nb_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $ovsdb_nb_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_nb_start_time]] + sbdb_not_restarted_after_cert_update: | + systemd('ovn-ovsdb-server-sb') AND systemd('ovn-ovsdb-server-sb', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-ovsdb-server-sb', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-ovsdb-server-sb', 'start_time_secs') + # - systemd: ovn-ovsdb-server-sb + # - varops: [[$ovsdb_sb_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $ovsdb_sb_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_sb_start_time]] northd_certs_invalid_logs: input: var/log/ovn/ovn-northd.log expr: $cert_invalid_expr diff --git a/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml b/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml index 9c830acfc..3240f33e1 100644 --- a/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml +++ b/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml @@ -1,30 +1,20 @@ -vars: - ml2_mechanism_driver: '@hotsos.core.plugins.openstack.neutron.Config.mechanism_drivers:plugins/ml2/ml2_conf.ini' - data_root_is_sosreport: '@hotsos.core.plugins.sosreport.SOSReportChecks.data_root_is_sosreport' - ovn_cert_host_exists: '@hotsos.core.host_helpers.filestat.FileFactory.exists:etc/ovn/cert_host' - ovn_cert_host_days: '@hotsos.core.host_helpers.ssl.SSLCertificatesFactory.days_to_expire:etc/ovn/cert_host' - neutron_ml2_cert_host_exists: - '@hotsos.core.host_helpers.filestat.FileFactory.exists:etc/neutron/plugins/ml2/cert_host' - neutron_ml2_cert_host_days: - '@hotsos.core.host_helpers.ssl.SSLCertificatesFactory.days_to_expire:etc/neutron/plugins/ml2/cert_host' checks: - is_not_sosreport_data_root: - varops: [[$data_root_is_sosreport], [ne, true]] + is_not_sosreport_data_root: | + @hotsos.core.plugins.sosreport.SOSReportChecks.data_root_is_sosreport != True is_ovn_central: apt: ovn-central - neutron_ml2_ovn_enabled: - - varops: [[$ml2_mechanism_driver], [ne, null]] - - varops: [[$ml2_mechanism_driver], [contains, 'ovn']] - central_cert_exists: - varops: [[$ovn_cert_host_exists], [truth]] - central_cert_expiring: - varops: [[$ovn_cert_host_days], [lt, 30]] + neutron_ml2_ovn_enabled: | + 'ovn' IN read_ini('etc/neutron/plugins/ml2/ml2_conf.ini', 'mechanism_drivers') + central_cert_exists: | + file('etc/ovn/cert_host', 'exists') + central_cert_expiring: | + read_cert('etc/ovn/cert_host', 'days_to_expire') < 30 is_neutron_server: apt: neutron-server - neutron_cert_exists: - varops: [[$neutron_ml2_cert_host_exists], [truth]] - neutron_cert_expiring: - varops: [[$neutron_ml2_cert_host_days], [lt, 30]] + neutron_cert_exists: | + file('etc/neutron/plugins/ml2/cert_host', 'exists') + neutron_cert_expiring: | + read_cert('etc/neutron/plugins/ml2/cert_host', 'days_to_expire') < 30 conclusions: central_missing_cert: decision: diff --git a/requirements.txt b/requirements.txt index aa597ba41..37d659746 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ simplejson sphinx>=4.3.2 python-dateutil pytz +pyparsing diff --git a/tests/unit/test_yexpr_parser.py b/tests/unit/test_yexpr_parser.py new file mode 100644 index 000000000..7df3f30e5 --- /dev/null +++ b/tests/unit/test_yexpr_parser.py @@ -0,0 +1,431 @@ +from unittest import mock + +from pyparsing import pyparsing_test as ppt +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + init_parser, + YExprInvalidArgumentException, + YExprNotFound +) + +from . import utils + + +class PropertyMock(mock.Mock): + """Property mock for testing.""" + + def __get__(self, obj, obj_type=None): + return self(obj, obj_type) + +# ___________________________________________________________________________# + + +# pylint: disable-next=too-many-public-methods +class TestYExprParser(ppt.TestParseResultsAsserts, utils.BaseTestCase): + """Unit tests for init_parser function.""" + + parser = init_parser() + + def parse_eval(self, expr_str): + return self.parser.parse_string(expr_str)[0].eval() + + def expect_true(self, expr_str): + self.assertTrue(self.parse_eval(expr_str)) + + def expect_false(self, expr_str): + self.assertFalse(self.parse_eval(expr_str)) + + def expect_isinstance(self, expr_str, type_x): + + self.assertTrue(isinstance(self.parse_eval(expr_str), type_x)) + + def test_expr_garbage(self): + input_v = ["\ttest", "g@rb@g€", "$!?/"] + for v in input_v: + with self.assertRaises(YExprInvalidArgumentException): + self.parser.parse_string(v) + + def test_and_expr_true(self): + self.expect_true("TrUe and True and True") + + def test_and_expr_false(self): + self.expect_false("TrUe and False and True") + + def test_and_expr_arg_is_expr(self): + self.expect_true("len('aaaa') == 4 and not False") + + def test_and_expr_not_a_bool(self): + with self.assertRaises(YExprInvalidArgumentException): + self.expect_false("Frue and Talse") + + def test_or_expr_true(self): + self.expect_true("False or True") + + def test_or_expr_false(self): + self.expect_false("False or False") + + def test_or_expr_chain(self): + self.expect_true("False or False or True or False") + + def test_and_or_chain_2(self): + self.expect_false("False or True and True and False") + + def test_not(self): + self.expect_true("not false") + self.expect_false("not true") + self.expect_false("not not not true") + self.expect_true("not None") + + def test_not_expr(self): + self.expect_false("not len('aaa') == 3") + + def test_not_and_or_precedence(self): + self.expect_true("not False and True") + self.expect_true("not False or False") + + def test_fn_len_string_literal(self): + self.assertEqual(self.parse_eval("len('aaaa')"), 4) + + def test_fn_len_none(self): + self.assertEqual(self.parse_eval("len(None)"), 0) + + def test_fn_len_int(self): + with self.assertRaises(TypeError): + self.parse_eval("len(1)") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=["this", "is", "a", "list"]) + def test_fn_len_property(self, _): + self.assertEqual(self.parse_eval("len(@dummy_prop)"), 4) + + def test_fn_not(self): + self.expect_true("False or not(True and False)") + + def test_fn_not_none(self): + self.expect_true("not(None)") + + def test_fn_not_2(self): + self.expect_false("False or not((True and False) or (False or True))") + + def test_fn_not_3(self): + self.expect_true("not(3 == 2)") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_fn_file_prop_exists(self, _, __): + self.expect_true("file('/etc/dummy', 'mtime') == 42") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_fn_file_prop_does_not_exist(self, _): + # Should raise exception + with self.assertRaises(Exception): + self.parse_eval("file('/etc/dummy', 'mtime') == 42") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_exists(self, _): + self.expect_true("systemd('dummy-service')") + + def test_fn_systemd_service_does_not_exist(self): + self.expect_isinstance("systemd('dummy-service')", YExprNotFound) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_and_property_exists(self, mock_systemd_helper): + # pylint: disable=duplicate-code + class DummyMock(mock.MagicMock): + """For testing purposes.""" + + # Create a mock for the services attribute + mock_services = DummyMock() + # Set the return value of get to another mock + mock_service = DummyMock() + mock_services.get.return_value = mock_service + # Set test_prop to a PropertyMock that returns 42 + mock_test_prop = PropertyMock(return_value="this is the value") + type(mock_service).test_prop = mock_test_prop + mock_systemd_helper.return_value.services = mock_services + self.assertEqual(self.parse_eval("systemd('dummy', 'test_prop')"), + "this is the value") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_exists_no_prop(self, mock_systemd_helper): + # Create a mock for the services attribute + mock_services = mock.MagicMock() + # Set the return value of get to another mock + mock_service = object() + mock_services.get.return_value = mock_service + mock_systemd_helper.return_value.services = mock_services + with self.assertRaises(Exception): + self.parse_eval("systemd('dummy', 'test_prop')") + + def test_fn_read_ini_does_not_exist(self): + self.expect_isinstance("read_ini('/etc/no-such.ini', 'field')", + YExprNotFound) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=42) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field')"), + 42) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=56) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key_section(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field', 'sec')"), + 56) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key_default(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field', None, 55)"), + 55) + + def test_fn_read_cert_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + self.expect_true("read_cert('/etc/dummy-cert')") + + def test_fn_read_cert_does_not_exist(self): + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.expect_isinstance("read_cert('/etc/dummy-cert')", + YExprNotFound) + self.assertIn("Unable to read SSL certificate file" + " /etc/dummy-cert: [Errno 2] No such " + "file or directory: '/etc/dummy-cert'", + log.output[0]) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SSLCertificate.expiry_date', + new_callable=PropertyMock, + return_value=123456) + def test_fn_read_cert_prop(self, _): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + self.assertEqual( + self.parse_eval("read_cert('/etc/dummy-cert','expiry_date')"), + 123456) + + def test_fn_read_cert_prop_does_not_exist(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + with self.assertRaises(Exception): + self.parse_eval("read_cert('/etc/dummy-cert','expiry_date')") + + def test_none(self): + self.assertEqual(self.parse_eval("None"), None) + + def test_string_literal(self): + self.assertEqual(self.parse_eval("'string literal'"), 'string literal') + + def test_integer_positive(self): + self.assertEqual(self.parse_eval("1234"), 1234) + + def test_integer_negative(self): + self.assertEqual(self.parse_eval("-1234"), -1234) + + def test_float_positive(self): + self.assertEqual(self.parse_eval("1234.56"), 1234.56) + + def test_float_negative(self): + self.assertEqual(self.parse_eval("-1234.56"), -1234.56) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=42) + def test_runtime_variable_exists(self, _): + self.expect_true("@prop.stuff == 42") + + def test_runtime_variable_does_not_exist(self): + with self.assertRaises(Exception): + with self.assertLogs(logger='hotsos', level='ERROR') as log: + self.assertIn("No module named 'prop'", log.output[0]) + self.expect_true("@prop.stuff == 42") + self.parse_eval("@prop.stuff.mod.x == 42") + + def test_sign_op_neg(self): + self.assertEqual(self.parse_eval("-(+1)"), -1) + + def test_sign_op_pos(self): + self.assertEqual(self.parse_eval("+(-1)"), -1) + + def test_power_op(self): + self.assertEqual(self.parse_eval("3**3**2"), 19683) + + def test_mul_op(self): + self.assertEqual(self.parse_eval("5*3"), 15) + + def test_mul_op_2(self): + self.assertEqual(self.parse_eval("-5*3"), -15) + + def test_div_op(self): + self.assertEqual(self.parse_eval("10/2"), 5) + + def test_div_op_2(self): + self.assertEqual(self.parse_eval("-10/2"), -5) + + def test_add_op(self): + self.assertEqual(self.parse_eval("5+3"), 8) + + def test_add_op_string(self): + self.assertEqual(self.parse_eval("'aa'+'bb'"), 'aabb') + + def test_add_op_2(self): + self.assertEqual(self.parse_eval("-5+3"), -2) + + def test_sub_op(self): + self.assertEqual(self.parse_eval("10-2"), 8) + + def test_sub_op_2(self): + self.assertEqual(self.parse_eval("1-2"), -1) + + def test_comparison_lt_true(self): + self.expect_true("-3 < -2") + + def test_comparison_lt_false(self): + self.expect_false("-3 < -4") + + def test_comparison_lte_true(self): + self.expect_true("-3 <= -3") + + def test_comparison_lte_false(self): + self.expect_false("-3 <= -4") + + def test_comparison_gt_true(self): + self.expect_true("-1 > -2") + + def test_comparison_gt_false(self): + self.expect_false("-5 > -4") + + def test_comparison_gte_true(self): + self.expect_true("-1 >= -1") + + def test_comparison_gte_false(self): + self.expect_false("-5 >= -4") + + def test_comparison_eq_true(self): + self.expect_true("-1 == (-2+1)") + + def test_comparison_eq_false(self): + self.expect_false("-5 == -4") + + def test_comparison_ne_true(self): + self.expect_true("(4+5) != (-2+1)") + + def test_comparison_ne_false(self): + self.expect_false("(-3-2) != (5+-10)") + + def test_comparison_in_string_true(self): + self.expect_true("'abcd' in '01234abcdefg'") + + def test_comparison_in_string_false(self): + self.expect_false("'abd' IN '01234abcdefg'") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=[42, 24]) + def test_comparison_in_prop_true(self, _): + self.expect_true("24 in @test.prop") + + def test_expr_complex_0(self): + self.expect_true( + """not((not True or True and False or True) + AND NOT(1 > 0) OR len('abcd') > 3 AND + 'ab' in 'bcabdef' and -5 == -4) AND TRUE + """ + ) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_1(self, _): + expr = """len('abc') > 2 and not(@hotsos.module.class.property_1 < 5.5) + or 3 * (4 + 4) / 2 == 12""" + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + def test_complex_expression_2(self, _, __, ___): + expr = """file('path/to/file', 'mtime') == 42 + and read_ini('config.ini', 'key') + or systemd('service')""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + def test_complex_expression_3(self): + self.expect_false( + """(len('abc') + 5 * 2 ** 3) / (3 - 1) > 10 + and (not(True) or False)""") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_4(self, _): + expr = """@hotsos.module.class.property_1 + 123 == 456 + or read_cert('cert.pem') and True""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_5(self, _): + expr = """not not(len('test') > 3 or @module.class.property != None) + or file('/path/to/file')""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value="aaaaaa") + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value='value') + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_complex_expression_6(self, mock_systemd_helper, _, __): + mock_systemd_helper.services = mock.MagicMock() + mock_systemd_helper.services.get.return_value = None + expr = """# this is a python style comment + len(@hotsos.module.class.property_1) * 3 >= 10 and + /* and this is a c-style comment*/ + systemd('aaa') or + # another python comment + read_ini('config.ini', 'key') == 'value'""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) diff --git a/tests/unit/test_yexpr_token.py b/tests/unit/test_yexpr_token.py new file mode 100644 index 000000000..89e375842 --- /dev/null +++ b/tests/unit/test_yexpr_token.py @@ -0,0 +1,902 @@ +from unittest import mock + +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + YExprToken, + YExprNotFound, + YExprLogicalOpBase, + YExprLogicalAnd, + YExprLogicalOr, + YExprLogicalNot, + YExprFnLen, + YExprFnNot, + YExprFnFile, + YExprFnSystemd, + YExprFnReadIni, + YExprFnReadCert, + YExprArgBoolean, + YExprArgNone, + YExprArgStringLiteral, + YExprArgInteger, + YExprArgFloat, + YExprArgRuntimeVariable, + YExprSignOp, + YExprPowerOp, + YExprMulDivOp, + YExprAddSubOp, + YExprComparisonOp, +) +from hotsos.core.exceptions import ( + NotEnoughParametersError, + TooManyParametersError +) + +from . import utils + +# ___________________________________________________________________________ # + + +class MockToken(YExprToken): + """Mock token for testing.""" + def __init__(self, v): + super().__init__(None) + self.v = v + + def eval(self): + return self.v + + +class PropertyMock(mock.Mock): + """Property mock for testing.""" + + def __get__(self, obj, obj_type=None): + return self(obj, obj_type) + +# ___________________________________________________________________________ # + + +class TestYExprToken(utils.BaseTestCase): + """Unit tests for YExprToken class.""" + + def test_operator_operands(self): + input_v = ("a", "b", "c", "d", "e", "f") + expected = [("a", "b"), ("c", "d"), ("e", "f")] + for i, operands in enumerate(YExprToken.operator_operands(input_v)): + self.assertEqual(operands, expected[i]) + + def test_operator_operands_not_even(self): + input_v = ("a", "b", "c", "d", "e") + expected = [("a", "b"), ("c", "d")] + for i, operands in enumerate(YExprToken.operator_operands(input_v)): + self.assertEqual(operands, expected[i]) + + +# ___________________________________________________________________________ # + + +class TestYExprNotFound(utils.BaseTestCase): + """Unit tests for YExprNotFound class.""" + + def test_not_found(self): + obj = YExprNotFound("dummy") + self.assertEqual(str(obj), "<`dummy` not found>") + + +# ___________________________________________________________________________ # + + +class TestYExprLogicalAnd(utils.BaseTestCase): + """Unit tests for YExprLogicalAnd class.""" + + def test_base_default_fn(self): + self.assertFalse(YExprLogicalOpBase.logical_fn([True])) + + def test_true(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_true_multiple(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), None, + MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_false_multiple(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_repr(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + + self.assertEqual(repr(uut), "True and True and False") + + +# ___________________________________________________________________________ # + + +class TestYExprLogicalOr(utils.BaseTestCase): + """Unit tests for YExprLogicalOr class.""" + + def test_true(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_true_multiple(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False), None, + MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_false_multiple(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False), + None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_repr(self): + uut = YExprLogicalOr( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + + self.assertEqual(repr(uut), "True or True or False") + + +# ___________________________________________________________________________ # + +class TestYExprLogicalNot(utils.BaseTestCase): + """Unit tests for YExprLogicalNot class.""" + + def test_true(self): + uut = YExprLogicalNot( + [[None, MockToken(False)]] + ) + + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalNot( + [[None, MockToken(True)]] + ) + + self.assertFalse(uut.eval()) + +# ___________________________________________________________________________ # + + +class TestYExprFnLen(utils.BaseTestCase): + """Unit tests for YExprFnLen class.""" + + def test_len_str(self): + input_v = "This is a string." + uut = YExprFnLen( + [None, [MockToken(input_v)]] + ) + self.assertEqual(uut.eval(), len(input_v)) + + def test_len_list(self): + input_v = ['this', 'is', 'a', 'string.'] + uut = YExprFnLen( + [None, [MockToken(input_v)]] + ) + self.assertEqual(uut.eval(), len(input_v)) + +# ___________________________________________________________________________ # + + +class TestYExprFnNot(utils.BaseTestCase): + """Unit tests for YExprFnNot class.""" + + def test_true(self): + uut = YExprFnNot( + [None, [MockToken(False)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprFnNot( + [None, [MockToken(True)]] + ) + self.assertFalse(uut.eval()) + +# ___________________________________________________________________________ # + + +class TestYExprFnFile(utils.BaseTestCase): + """Unit tests for YExprFnFile class.""" + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_prop_exists(self, _, mock_property): + uut = YExprFnFile( + [None, [MockToken('does_not_matter'), MockToken('mtime')]] + ) + self.assertEqual(uut.eval(), 42) + mock_property.assert_called() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_exists_noprop(self, _): + uut = YExprFnFile( + [None, [MockToken('does_not_matter')]] + ) + self.assertEqual(uut.eval(), True) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_prop_does_not_exists(self, _): + uut = YExprFnFile( + [None, [MockToken('does_not_matter'), MockToken('not_exists')]] + ) + + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________ # + + +class TestYExprFnSystemd(utils.BaseTestCase): + """Unit tests for YExprFnSystemd class.""" + + def test_no_arg(self): + uut = YExprFnSystemd( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnSystemd( + [None, [1, 2, 3]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_exists(self, mock_systemd_helper): + mock_systemd_helper.services = mock.MagicMock() + mock_systemd_helper.services.get.return_value = object() + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter')]] + ) + self.assertTrue(uut.eval()) + +# ___________________________________________________________________________# + + def test_service_does_not_exists(self): + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter')]] + ) + self.assertEqual(str(uut.eval()), + str(YExprNotFound('does_not_matter'))) + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_prop_exists(self, mock_systemd_helper): + # pylint: disable=duplicate-code + class DummyMock(mock.MagicMock): + """ For testing. """ + + # Create a mock for the services attribute + mock_services = DummyMock() + # Set the return value of get to another mock + mock_service = DummyMock() + mock_services.get.return_value = mock_service + # Set test_prop to a PropertyMock that returns 42 + mock_test_prop = PropertyMock(return_value="this is the value") + type(mock_service).test_prop = mock_test_prop + mock_systemd_helper.return_value.services = mock_services + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter'), MockToken('test_prop')]] + ) + self.assertEqual(uut.eval(), "this is the value") + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_prop_does_not_exists(self, mock_systemd_helper): + # Create a mock for the services attribute + mock_services = mock.MagicMock() + # Set the return value of get to another mock + mock_service = object() + mock_services.get.return_value = mock_service + mock_systemd_helper.return_value.services = mock_services + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter'), MockToken('test_prop')]] + ) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprFnReadIni(utils.BaseTestCase): + """Unit tests for YExprFnReadIni class.""" + + def test_no_arg(self): + uut = YExprFnReadIni( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnReadIni( + [None, [1, 2, 3, 4, 5]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=42) + @mock.patch('builtins.open') + def test_ini_exists(self, _, __, ___): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter")]] + ) + + self.assertEqual(uut.eval(), 42) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=False) + def test_ini_does_not_exists(self, _): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter")]] + ) + self.assertTrue(isinstance(uut.eval(), YExprNotFound)) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + def test_ini_does_not_exists_w_section(self, mock_get, __): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter"), + MockToken("the_section")]] + ) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertEqual(uut.eval(), None) + self.assertIn("cannot parse config file", log.output[0]) + mock_get.assert_called_once_with("does_not_matter", + "the_section", + expand_to_list=False) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + def test_ini_does_not_exists_default(self, _, __): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter"), + MockToken(None), + MockToken("default")]] + ) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertEqual(uut.eval(), "default") + self.assertIn("cannot parse config file", log.output[0]) + + +# ___________________________________________________________________________# + + +class TestYExprFnReadCert(utils.BaseTestCase): + """Unit tests for YExprFnReadCert class.""" + + def test_no_arg(self): + uut = YExprFnReadCert( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnReadCert( + [None, [1, 2, 3]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + + def test_cert_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter")]]) + self.assertTrue(uut.eval()) + mock_file.assert_called_once() + + def test_cert_does_not_exists(self): + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + mock_file.side_effect = OSError() + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter")]]) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertTrue(isinstance(uut.eval(), YExprNotFound)) + self.assertIn("Unable to read SSL certificate", log.output[0]) + mock_file.assert_called_once() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SSLCertificate.expiry_date', + new_callable=PropertyMock, + return_value=123456) + def test_cert_read_prop_exists(self, _): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter"), + MockToken("expiry_date")]]) + self.assertEqual(uut.eval(), 123456) + mock_file.assert_called_once() + + def test_cert_read_prop_does_not_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter"), + MockToken("not_a_prop")]]) + with self.assertRaises(Exception): + self.assertEqual(uut.eval(), 123456) + mock_file.assert_called_once() + + +# ___________________________________________________________________________# + + +class TestYExprArgBoolean(utils.BaseTestCase): + """Unit tests for tYExprArgBoolean class.""" + + def test_yexpr_boolean_true(self): + uut = YExprArgBoolean(["TrUe"]) + self.assertTrue(uut.eval()) + + def test_yexpr_boolean_false(self): + uut = YExprArgBoolean(["fAlSe"]) + self.assertFalse(uut.eval()) + + def test_yexpr_boolean_garbage(self): + uut = YExprArgBoolean(["not-a-boolean"]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprArgNone(utils.BaseTestCase): + """Unit tests for YExprArgNone class.""" + + def test_yexpr_none(self): + uut = YExprArgNone(["does-not-matter"]) + self.assertEqual(uut.eval(), None) + +# ___________________________________________________________________________# + + +class TestYExprStringLiteral(utils.BaseTestCase): + """Unit tests for YExprArgStringLiteral class.""" + + def test_yexpr_string_literal(self): + uut = YExprArgStringLiteral(["does-not-matter"]) + self.assertEqual(uut.eval(), "does-not-matter") + +# ___________________________________________________________________________# + + +class TestYExprArgInteger(utils.BaseTestCase): + """Unit tests for YExprArgInteger class.""" + + def test_yexpr_integer_from_int_str(self): + uut = YExprArgInteger(["1"]) + self.assertEqual(uut.eval(), 1) + + def test_yexpr_integer_not_an_int(self): + uut = YExprArgInteger(["garbage"]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_integer_not_an_int_float(self): + uut = YExprArgInteger(["1.1"]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprArgFloat(utils.BaseTestCase): + """Unit tests for YExprArgFloat class.""" + + def test_yexpr_float_from_int_str(self): + uut = YExprArgFloat(["1.1"]) + self.assertEqual(uut.eval(), 1.1) + + def test_yexpr_float_not_an_int(self): + uut = YExprArgFloat(["garbage"]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_float_not_an_int_float(self): + uut = YExprArgFloat(["1"]) + self.assertEqual(uut.eval(), 1.0) + + +# ___________________________________________________________________________# + + +class TestYExprArgRuntimeVariable(utils.BaseTestCase): + """Unit tests for YExprArgRuntimeVariable class.""" + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=1.1) + def test_yexpr_runtime_variable_exists(self, _): + # (mgilor): 'thisx' was 'this' at first, but it triggers + # a Python easter egg which prints out a poem-like text to the console. + uut = YExprArgRuntimeVariable(["@thisx.is.a.test"], context=None) + self.assertEqual(uut.eval(), float(1.1)) + + def test_yexpr_runtime_variable_does_not_exist(self): + uut = YExprArgRuntimeVariable(["@thisx.is.a.test"], context=None) + with self.assertRaises(Exception): + with self.assertLogs(logger='hotsos', level='ERROR') as log: + uut.eval() + self.assertIn("failed to import class a from thisx.is", + log.output[0]) + +# ___________________________________________________________________________# + + +class TestYExprSignOp(utils.BaseTestCase): + """Unit tests for YExprSignOp class.""" + + def test_yexpr_signop_plus_negative(self): + uut = YExprSignOp([["+", MockToken(-1)]]) + self.assertEqual(uut.eval(), -1) + + def test_yexpr_signop_plus_positive(self): + uut = YExprSignOp([["+", MockToken(1)]]) + self.assertEqual(uut.eval(), 1) + + def test_yexpr_signop_minus_positive(self): + uut = YExprSignOp([["-", MockToken(1)]]) + self.assertEqual(uut.eval(), -1) + + def test_yexpr_signop_minus_negative(self): + uut = YExprSignOp([["-", MockToken(-1)]]) + self.assertEqual(uut.eval(), 1) + +# ___________________________________________________________________________# + + +class TestYExprPowerOp(utils.BaseTestCase): + """Unit tests for YExprPowerOp class.""" + + def test_yexpr_power_op_non_even_args(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(2), "test"]]) + with self.assertRaises(TooManyParametersError): + uut.eval() + + def test_yexpr_power_op_positive_positive(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), 9) + + def test_yexpr_power_op_positive_negative(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(-2)]]) + self.assertEqual(round(uut.eval(), 5), 0.11111) + + def test_yexpr_power_op_negative_positive(self): + uut = YExprPowerOp([[MockToken(-3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), 9) + + def test_yexpr_power_op_negative_negative(self): + uut = YExprPowerOp([[MockToken(-3), "**", MockToken(-2)]]) + self.assertEqual(round(uut.eval(), 5), 0.11111) + + def test_yexpr_power_op_chain_multiple(self): + uut = YExprPowerOp([[MockToken(-3), "**", + MockToken(-3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), -19683) + + def test_yexpr_power_op_not_enough_tokens(self): + uut = YExprPowerOp([[MockToken(-3), "**"]]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprMulDivOp(utils.BaseTestCase): + """Unit tests for YExprMulDivOp class.""" + + def test_yexpr_mult_op_positive_positive(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_mult_op_positive_negative(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(-2)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_mult_op_negative_positive(self): + uut = YExprMulDivOp([[MockToken(-3), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_mult_op_negative_negative(self): + uut = YExprMulDivOp([[MockToken(-3), "*", MockToken(-2)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_mult_op_chain_multiple(self): + uut = YExprMulDivOp([[MockToken(3), "*", + MockToken(2), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), 12) + + def test_yexpr_mult_op_int_float(self): + uut = YExprMulDivOp([[MockToken(3.6), "*", + MockToken(2)]]) + self.assertEqual(uut.eval(), 7.2) + + def test_yexpr_mult_not_enough_args(self): + uut = YExprMulDivOp([[MockToken(3), "*"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_mult_not_odd_num_of_args(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(3), "*"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_div_op_positive_positive(self): + uut = YExprMulDivOp([[MockToken(6), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), 3) + + def test_yexpr_div_op_positive_negative(self): + uut = YExprMulDivOp([[MockToken(6), "/", MockToken(-2)]]) + self.assertEqual(uut.eval(), -3) + + def test_yexpr_div_op_negative_positive(self): + uut = YExprMulDivOp([[MockToken(-6), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), -3) + + def test_yexpr_div_op_negative_negative(self): + uut = YExprMulDivOp([[MockToken(-6), "/", MockToken(-2)]]) + self.assertEqual(uut.eval(), 3) + + def test_yexpr_div_op_chain_multiple(self): + uut = YExprMulDivOp([[MockToken(-12), "/", + MockToken(-3), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), 2) + + def test_yexpr_div_not_enough_args(self): + uut = YExprMulDivOp([[MockToken(3), "/"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_div_not_odd_num_of_args(self): + uut = YExprMulDivOp([[MockToken(3), "/", MockToken(3), "/"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_muldiv_not_a_valid_op_token(self): + uut = YExprMulDivOp([[MockToken(3), "f", MockToken(3)]]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprAddSubOp(utils.BaseTestCase): + """Unit tests for YExprAddSubOp class.""" + + def test_yexpr_unrecognized_op(self): + uut = YExprAddSubOp([[MockToken(3), "?", MockToken(3)]]) + with self.assertRaisesRegex(NameError, r"Unrecognized operation \?"): + uut.eval() + + def test_yexpr_add_positive_positive(self): + uut = YExprAddSubOp([[MockToken(3), "+", MockToken(3)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_add_positive_negative(self): + uut = YExprAddSubOp([[MockToken(3), "+", MockToken(-3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_negative_positive(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_negative_negative(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(-3)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_add_chain_multiple(self): + uut = YExprAddSubOp([[MockToken(-3), "+", + MockToken(-3), "+", MockToken(6)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_not_enough_args(self): + uut = YExprAddSubOp([[MockToken(-3), "+"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_add_not_odd_number_of_args(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(-3), "+"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_sub_positive_positive(self): + uut = YExprAddSubOp([[MockToken(3), "-", MockToken(3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_sub_positive_negative(self): + uut = YExprAddSubOp([[MockToken(3), "-", MockToken(-3)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_sub_negative_positive(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(3)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_sub_negative_negative(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(-3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_sub_chain_multiple(self): + uut = YExprAddSubOp([[MockToken(-3), "-", + MockToken(-3), "+", MockToken(6)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_sub_not_enough_args(self): + uut = YExprAddSubOp([[MockToken(-3), "-"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_sub_not_odd_number_of_args(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(-3), "-"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_addsub_not_a_valid_op_token(self): + uut = YExprMulDivOp([[MockToken(3), "f", MockToken(3)]]) + with self.assertRaises(Exception): + uut.eval() + + +# ___________________________________________________________________________# + + +class TestYExprComparisonOp(utils.BaseTestCase): + """Unit tests for YExprComparisonOp class.""" + + def test_yexpr_comp_op_lt_true(self): + for alt in ["<", "LT"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_lt_false(self): + for alt in ["<", "LT"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_lte_true(self): + for alt in ["<=", "LE"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_lte_false(self): + for alt in ["<=", "LE"]: + uut = YExprComparisonOp([[MockToken(5), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_gt_true(self): + for alt in [">", "GT"]: + uut = YExprComparisonOp([[MockToken(5), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_gt_false(self): + for alt in [">", "GT"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_gte_true(self): + for alt in [">=", "GE"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_gte_false(self): + for alt in [">=", "GE"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_ne_true(self): + for alt in ["!=", "NE", "<>"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_ne_false(self): + for alt in ["!=", "NE", "<>"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(3)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_eq_true(self): + for alt in ["==", "EQ"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(3)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_eq_false(self): + for alt in ["==", "EQ"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(3)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_in_list_true(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken(3), alt, + MockToken([1, 2, 3])]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_in_list_false(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken(3), alt, + MockToken([1, 2, 4])]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_in_string_true(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken("a"), alt, + MockToken("this is a test")]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_in_string_false(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken("a"), alt, + MockToken("this is b test")]]) + self.assertFalse(uut.eval())