diff --git a/doc/source/contrib/language_ref/property_ref/requirement_types.rst b/doc/source/contrib/language_ref/property_ref/requirement_types.rst index 0139cb251..4f58b7ea8 100644 --- a/doc/source/contrib/language_ref/property_ref/requirement_types.rst +++ b/doc/source/contrib/language_ref/property_ref/requirement_types.rst @@ -337,3 +337,623 @@ Cache keys: * input_value: value of the variable used as input * ops: str representation of ops list + +Expression +---------- + +Expressions allow the user to express a requirement in a human-readable domain specific language. Its purpose +is to provide a simplified, clear and concise expression grammar for all requirement types, and to eliminate +redundancies. + +Usage: + +.. code-block:: yaml + + checks: + checkmyvar: + # Explicit expression + expression: | + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + @hotsos.core.plugins.plugin3.class.property_2) + +Explicit expression form can be combined with other requirement types: + +.. code-block:: yaml + + checks: + checkmyvar: + # Explicit expression + expression: | + # 'test' should not be present + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + # Property value should exceed the threshold + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + # Property is True + @hotsos.core.plugins.plugin3.class.property_2) + systemd: test-service + +Expressions can be declared implicitly when assigned as a literal to a check: + +.. code-block:: yaml + + checks: + # Implicit expression + checkmyvar: | + NOT 'test' IN @hotsos.core.plugins.plugin.class.property AND + (@hotsos.core.plugins.plugin2.class.property_1 >= 500 OR + @hotsos.core.plugins.plugin3.class.property_2) + +Expression grammar consist of the following constructs to allow user to express a requirement: + +Keywords +******** + +Keywords consist of a special set of words that has a purpose in the grammar. The current keywords are +[:ref:`None keyword` | :ref:`True keyword` | :ref:`False keyword`] + +None keyword +============== + +Case-insensitive. Equal to Python `None`. + +**Examples** + +.. code-block:: yaml + + expression: | + NONE + NoNE + +True keyword +============== + +Case-insensitive. Equal to Python `True`. + +**Examples** + +.. code-block:: yaml + + expression: | + True + TrUe + +False keyword +=============== + +Case-insensitive. Equal to Python `False`. + +**Examples** + +.. code-block:: yaml + + expression: | + FALSE + False + +Constants +********* + +Constants are invariable values in the grammar. The current constants are [:ref:`float constant` | :ref:`integer constant` | :ref:`String literal`] + +float constant +================ + +Floating point constants. `[0-9]+\.[0-9]+` + +**Examples** + +.. code-block:: yaml + + expression: | + 12.34 + -56.78 + +integer constant +================== + +Integer constants. `[0-9]+` + +**Examples** + +.. code-block:: yaml + + expression: | + 1234 + -4567 + +String literal +============== + +String literal. Declared with single quotes `''` + +**Examples** + +.. code-block:: yaml + + expression: | + 'this is a string literal' + 'this is @nother 1' + '1234' + '1.2' + 'True' + +Functions +********* + +Functions are defined as CaselessKeyword, followed by a `(` args `)`. + +`caseless_keyword(expr1, ... exprN)`. + +len(expr...) +============ + +Function to retrieve length of an expression. The expression argument should evaluate +to a construct that has a `length` property. + +**Examples** + +.. code-block:: yaml + + expression: | + len('literal') # would return 7 + len('lit' + 'eral') # would return 7 + len(@class.property) # would return the length of the property value + +not(expr...) +============ + +Function to negate an expression's boolean vlaue. The expression argument should evaluate +to a construct that is `boolable`. + +**Examples** + +.. code-block:: yaml + + expression: | + not(True) # would return False + not(None) # would return True + not(3 == 4) # Would return True + +file('path-to-file', 'property-name[optional]') +=============================================== + +Function to retrieve a file's properties. Returns True or False when called with one argument depending +whether the file exists or not. Returns the property's value when called with two arguments. Property names +are all the properties that `hotsos.core.host_helpers.filestat.FileObj` has. + +** Examples ** + +.. code-block:: yaml + + expression: | + # Returns True or False, depending on whether the `/path/to/file` exists or not. + file('/path/to/file') + +.. code-block:: yaml + + expression: | + # Returns file's mtime property when file exists, YExprNotFound when the + # file is absent. Raises an exception when the given property name is not + # present on the service object. + file('/path/to/file', 'mtime') + +systemd('service-name', 'property-name[optional]') +================================================== + +Function to retrieve a systemd service's properties. Returns True or False when called with one argument depending +whether the service exists or not. Returns the property's value when called with two arguments. + +Property names are all the properties that `hotsos.core.host_helpers.systemd.SystemdService` has. + +**Examples** + +.. code-block:: yaml + + expression: | + # Returns True or False, depending on whether the `systemd-resolve` service + # exists or not. + systemd('systemd-resolved') + +.. code-block:: yaml + + expression: | + # Returns the service's `start_time_secs` property value when the service file + # exists, YExprNotFound when the service is absent. Raises an exception when the + # given property name is not present on the service object. + systemd('systemd-resolved', 'start_time_secs') + +read_ini('path-to-ini', 'key', 'section[optional]', 'default[optional]') +======================================================================== + +Function to read values from an ini file. Returns the first matching key's value when called with two arguments. Returns +the key's value in the specific section when called with three arguments. Returns the value specified in `default` when +the `default` argument is provided and given key in given section is not found. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return the value of the first encountered key 'foo' from any section. + # would return YExprNotFound when the key is not found. + read_ini('/etc/ini-file', 'foo') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from section 'bar. + # would return YExprNotFound when the key is not found. + read_ini('/etc/ini-file', 'foo', 'bar') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from section 'bar'. + # would return the string literal 'abcd' when the key is not found. + read_ini('/etc/ini-file', 'foo', 'bar', 'abcd') + +.. code-block:: yaml + + expression: | + # would return the value of the key 'foo' from any section. + # would return the string literal 'abcd' when the key is not found. + read_ini('/etc/ini-file', 'foo', None, 'abcd') + +read_cert('path-to-cert', 'property-name[optional]') +==================================================== + +Function to read values from a X509 certificate (e.g. a typical SSL certificate). Returns True or False depending whether +the certificate file 'path-to-cert' exist when called with a single argument. With two arguments, Returns the value of the +`property-name` when the certificate file `path-to-cert` exist, YExprNotFound otherwise. + +Property names are all the properties that `hotsos.core.host_helpers.ssl.SSLCertificate` has. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return True or False depending on whether '/etc/ssl/cert' exist or not. + read_cert('/etc/ssl/cert') + +.. code-block:: yaml + + expression: | + # Would return the certificate's expiry_date property when the certificate + # file is present. + # Raises an exception when property name cannot be found on SSLCertificate object. + #read_cert('/etc/ssl/cert', 'expiry_date') + +Runtime variables +***************** + +Python property `@module.class.property_name` +============================================= + +Retrieve the value of a runtime Python property. Raises an exception when a property with a given name is not found. + +**Examples** + +.. code-block:: yaml + + expression: | + # would return the value of the property + # Raises exception when property is absent. + @hotsos.plugins.plugin.class_name.property_name + + +Operands +******** + +:ref:`Keywords` | :ref:`Constants` | :ref:`Functions` | :ref:`Runtime Variables` + +Arithmetic operators +******************** + +plus sign (`+`) +=============== + +Indicate the sign of an Integer or Float. + +**Examples** + +.. code-block:: yaml + + expression: | + # plus six + +6 + # plus six + +(+6) + +minus sign (`-`) +================ + +Indicate the sign of an Integer or Float. + +**Examples** + +.. code-block:: yaml + + expression: | + # plus six + -(-6) + # minus six + -6 + +multiplication (`*`) +==================== + +Multiply the given operands for the operands that are multiplicable with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 0.6 + 1 * 0.6 + +division(`/`) +============= + +Perform a division with the given operands for the operands that supports division with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 2.5 + 1 / 0.4 + +addition(`+`) +============= + +Add two things with each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 8 + 3 + 5 # 8 + # result is 'aabb' + 'aa' + 'bb' + +subtraction(`-`) +================ + +Subtract two things from each other. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is -9 + 3 - 5 - 7 + +exponent (`\*\*`) +================= + +Calculate the exponent of a number. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is 8 + 2**3 + # result is 729 + 9**3 + +Comparison operators +******************** + +Comparison operators allow users to compare values within expressions. These operators can be used to evaluate whether a particular condition is met. + +less than (`<`, `LT`) +===================== + +Evaluates whether the left operand is less than the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 < 5 + # result is False + 3.81 > 3.80 + +less than or equal (`<=`, `LE`) +=============================== + +Evaluates whether the left operand is less than or equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 5 <= 5 + # result is False + 3.81 LE 3.80 + +greater than (`>`, `GT`) +======================== + +Evaluates whether the left operand is greater than the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is False + 3 > 5 + # result is True + 3.81 > (2.80 + 1) + +greater than or equal (`>=`, `GE`) +================================== + +Evaluates whether the left operand is greater than or equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is False + 3 >= 5 + # result is True + 3.81 GE (3.80 + 0.01) + +equal (`==`, `EQ`) +================== + +Evaluates whether the left operand is equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 == 3 + # result is False + True == False + # result is True + 'test' EQ 'test' + +not equal (`!=`, `NE`, `<>`) +============================ + +Evaluates whether the left operand is not equal to the right operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 3 != 5 + # result is False + 3.81 <> (3.80 + 0.01) + # result is True + 'test' NE None + +in (`IN`) +========= + +Evaluates whether the left operand is contained within the right operand (e.g., within a string or list). + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + 'a' in 'ab' + # result is True (assuming list of ints = [1,2,3]) + 1 in @module.prop.list_of_ints + +Logical operators +***************** + +Logical operators allow users to combine multiple expressions. These operators evaluate the truthiness of expressions to determine the overall outcome. + +logical and (`and`) +=================== + +Returns True if both operands are True. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + True and (5 < 3 or 'test' in 'testament') + # result is False + (3.3 <= 2) or (5 > 3 and 'rest' in 'testament') + +logical or (`or`) +================= + +Returns True if at least one of the operands is True. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + True or False + # result is False + False or not True + +logical not (`not`) +=================== + +Returns the opposite boolean value of the operand. + +**Examples** + +.. code-block:: yaml + + expression: | + # result is True + not None + # result is True + not False + +Comments +******** + +Comments are used to annotate the code and are ignored during execution. This can be useful for adding explanations or notes within the expression grammar. + +Python-style comments (`#`) +=========================== + +Single-line comments that begin with `#`. + +**Examples** + +.. code-block:: yaml + + expression: | + # This is a comment explaining what the expression does. + not None + + +C-style comments (`//`, `/*...*/`) +================================== + +Single-line comments using `//` or multi-line comments enclosed in `/*...*/`. + +**Examples** + +.. code-block:: yaml + + expression: | + // This is a single-line comment + +.. code-block:: yaml + + expression: | + /* This is a + multi-line comment */ + +Expression grammar +****************** + +- :ref:`Operands` | :ref:`Arithmetic operators` | :ref:`Comparison operators` | :ref:`Logical operators` + +Note that comments are not the part of the expression and ignored by the parser. diff --git a/hotsos/core/alias.py b/hotsos/core/alias.py new file mode 100644 index 000000000..2b423578c --- /dev/null +++ b/hotsos/core/alias.py @@ -0,0 +1,105 @@ +"""Aliasing utilities.""" + +from hotsos.core.log import log + + +class AliasAlreadyInUseError(Exception): + """Raised when an alias is already in use.""" + + def __init__(self, name): + self.message = f"Alias '{name}` already in use!" + + def __str__(self): + return self.message + + +class AliasForbiddenError(Exception): + """Raised when an alias is forbidden to use.""" + + def __init__(self, name): + self.message = f"Alias '{name}` is forbidden!" + + def __str__(self): + return self.message + + +class AliasRegistry: + """ + A class that provides a registry for aliasing Python things. + """ + + # A class-level dictionary to store registered aliases. + registry = {} + + @staticmethod + def register(name, decoratee): + """ + Register a function, method, or property under an alias. + + This method handles different types of Python objects and creates + appropriate wrappers or registrations based on the object type. + + Args: + name (str): The alias under which to register the decoratee. + decoratee (callable or property): The Python object to be + registered. + + Raises: + AliasAlreadyInUseError: If the alias name is already registered. + AliasForbiddenError: If the alias name starts with "hotsos." + """ + isprop = isinstance(decoratee, property) + target = decoratee.fget if isprop else decoratee + + if name.startswith("hotsos."): + raise AliasForbiddenError(name) + + if name in AliasRegistry.registry: + log.debug("alias registration failed -- already in use(`%s`)", + name) + raise AliasAlreadyInUseError(name) + + import_path = f"{target.__module__}.{target.__qualname__}" + log.debug("registering alias `%s` --> {%s}", name, import_path) + # Register full import path. + AliasRegistry.registry[name] = import_path + + @staticmethod + def resolve(the_alias, default=None): + """ + Retrieve a registered alias. + + Args: + the_alias (str): The alias to retrieve. + + Returns: + callable: The function or wrapper associated with the alias. + + Raises: + NoSuchAliasError: No such alias in the registry. + """ + + if the_alias not in AliasRegistry.registry: + log.debug( + "alias `%s` not found in the registry, " + "returning the default value", + the_alias, + ) + return default + + value = AliasRegistry.registry[the_alias] + log.debug("alias %s resolved to %s", the_alias, value) + return value + + +def alias(argument): + """Create an alias for a property, function or a thing.""" + + def real_decorator(func): + """We're not wrapping the func as we don't want + to do anything at runtime. We just want to alias + `func` to some user-defined name and call it on-demand.""" + AliasRegistry.register(argument, func) + return func + + return real_decorator diff --git a/hotsos/core/exceptions.py b/hotsos/core/exceptions.py index fd68f9e12..7c51fb271 100644 --- a/hotsos/core/exceptions.py +++ b/hotsos/core/exceptions.py @@ -35,6 +35,10 @@ class NotEnoughParametersError(Exception): """Raised when an operation did not get enough parameters to operate.""" +class TooManyParametersError(Exception): + """Raised when an operation did get more parameters than expected.""" + + class MissingRequiredParameterError(Exception): """Raised when an operation did not get a parameter required for the operation.""" @@ -59,3 +63,7 @@ class PreconditionError(Exception): class ExpectationNotMetError(Exception): """Raised when an operation's expectation is not met.""" + + +class NoSuchPropertyError(Exception): + """Raised when an object does not have a property where it should.""" diff --git a/hotsos/core/plugins/kernel/memory.py b/hotsos/core/plugins/kernel/memory.py index b934c51af..66cc6bf6f 100644 --- a/hotsos/core/plugins/kernel/memory.py +++ b/hotsos/core/plugins/kernel/memory.py @@ -106,11 +106,13 @@ def huge_pages_enabled(self): @property def hugetlb_to_mem_total_percentage(self): - return round((self.Hugetlb * 100) / self.MemTotal) + return (self.MemTotal and round( + (self.Hugetlb * 100) / self.MemTotal)) @property def mem_avail_to_mem_total_percentage(self): - return round((self.MemAvailable * 100) / self.MemTotal) + return (self.MemTotal and round( + (self.MemAvailable * 100) / self.MemTotal)) @property def hugep_used_to_hugep_total_percentage(self): diff --git a/hotsos/core/ycheck/engine/properties/checks.py b/hotsos/core/ycheck/engine/properties/checks.py index 560920c1a..f06a072e8 100644 --- a/hotsos/core/ycheck/engine/properties/checks.py +++ b/hotsos/core/ycheck/engine/properties/checks.py @@ -1,6 +1,9 @@ from functools import cached_property -from propertree.propertree2 import PTreeLogicalGrouping +from propertree.propertree2 import ( + PTreeLogicalGrouping, + PTreeOverrideLiteralType +) from hotsos.core.log import log from hotsos.core.ycheck.engine.properties.common import ( YPropertyOverrideBase, @@ -11,6 +14,9 @@ from hotsos.core.ycheck.engine.properties.requires.requires import ( YPropertyRequires ) +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + YPropertyExpr +) from hotsos.core.ycheck.engine.properties.search import ( YPropertySearch, ) @@ -162,6 +168,12 @@ def result(self): stop_executon = False for member in self.members: for item in member: + # Allow implicit declaration of expression. + if isinstance(item, PTreeOverrideLiteralType): + item = YPropertyExpr(item.root, "expression", + item.content, item.override_path, + context=self.context) + # Ignore these here as they are used by search properties. if isinstance(item, YPropertyInput): continue diff --git a/hotsos/core/ycheck/engine/properties/common.py b/hotsos/core/ycheck/engine/properties/common.py index cd6041e21..b18348f6d 100644 --- a/hotsos/core/ycheck/engine/properties/common.py +++ b/hotsos/core/ycheck/engine/properties/common.py @@ -284,44 +284,19 @@ def __getattr__(self, key): return self.data.get(key) -class YPropertyBase(PTreeOverrideBase): - """ - Base class used for all YAML property objects. Implements and extends - PTreeOverrideBase to provide frequently used methods and helpers to - property implementations. - """ - def __init__(self, *args, **kwargs): - self._cache = PropertyCache() - super().__init__(*args, **kwargs) - - def resolve_var(self, name): - """ - Resolve variable with name to value. This can be used speculatively and - will return the name as value if it can't be resolved. - """ - if not name.startswith('$'): - return name - - if hasattr(self, 'context'): - if self.context.vars: - _name = name.partition('$')[2] - return self.context.vars.resolve(_name) +class PythonEntityResolver: + """A class to resolve Python entities (e.g. variable, property) + by their import path.""" - log.warning("could not resolve var '%s' - vars not found in " - "context", name) + # Class-level cache for Python module imports for future use - return name + import_cache = None - @property - def cache(self): - """ - All properties get their own cache object that they can use as they - wish. - """ - return self._cache + def __init__(self, context): + self.context = context def _load_from_import_cache(self, key): - """ Retrieve from global context if one exists. + """ Retrieve from global cache if one exists. @param key: key to retrieve """ @@ -374,16 +349,14 @@ def _add_to_import_cache(self, key, value): @param key: key to save @param value: value to save """ - if self.context is None: - log.info("context not available - cannot save '%s'", key) + + if not self.import_cache: + self.import_cache = {key: value} + log.debug("import cache initialized with initial key `%s`", key) return - c = getattr(self.context, 'import_cache') - if c: - c[key] = value - else: - c = {key: value} - setattr(self.context, 'import_cache', c) + self.import_cache[key] = value + log.debug("import cache updated for key %s", key) def get_cls(self, import_str): """ Import and instantiate Python class. @@ -535,6 +508,44 @@ def get_import(self, import_str): return self.get_attribute(import_str) +class YPropertyBase(PTreeOverrideBase, PythonEntityResolver): + """ + Base class used for all YAML property objects. Implements and extends + PTreeOverrideBase to provide frequently used methods and helpers to + property implementations. + """ + + def __init__(self, *args, **kwargs): + self._cache = PropertyCache() + super().__init__(*args, **kwargs) + + def resolve_var(self, name): + """ + Resolve variable with name to value. This can be used speculatively and + will return the name as value if it can't be resolved. + """ + if not name.startswith('$'): + return name + + if hasattr(self, 'context'): + if self.context.vars: + _name = name.partition('$')[2] + return self.context.vars.resolve(_name) + + log.warning("could not resolve var '%s' - vars not found in " + "context", name) + + return name + + @property + def cache(self): + """ + All properties get their own cache object that they can use as they + wish. + """ + return self._cache + + class YPropertyOverrideBase(YPropertyBase, PTreeOverrideBase): """ Base class for all simple/flat property objects. """ diff --git a/hotsos/core/ycheck/engine/properties/requires/requires.py b/hotsos/core/ycheck/engine/properties/requires/requires.py index 9e06313aa..b96de6fb6 100644 --- a/hotsos/core/ycheck/engine/properties/requires/requires.py +++ b/hotsos/core/ycheck/engine/properties/requires/requires.py @@ -13,6 +13,7 @@ property as rproperty, path, varops, + expr, ) CACHE_CHECK_KEY = '__PREVIOUSLY_CACHED_PROPERTY_TYPE' @@ -94,7 +95,8 @@ class YPropertyRequires(YPropertyMappedOverrideBase): systemd.YRequirementTypeSystemd, rproperty.YRequirementTypeProperty, path.YRequirementTypePath, - varops.YPropertyVarOps] + varops.YPropertyVarOps, + expr.YPropertyExpr,] # We want to be able to use this property both on its own and as a member # of other mapping properties e.g. Checks. The following setting enables # this. diff --git a/hotsos/core/ycheck/engine/properties/requires/types/expr.py b/hotsos/core/ycheck/engine/properties/requires/types/expr.py new file mode 100644 index 000000000..03f01e0aa --- /dev/null +++ b/hotsos/core/ycheck/engine/properties/requires/types/expr.py @@ -0,0 +1,672 @@ +import os +from abc import abstractmethod +from typing import Callable, Iterable + +import pyparsing as pp +from hotsos.core.exceptions import ( + NotEnoughParametersError, + TooManyParametersError, + NoSuchPropertyError, + UnexpectedParameterError +) +from hotsos.core.ycheck.engine.properties.common import PythonEntityResolver +from hotsos.core.ycheck.engine.properties.requires import ( + intercept_exception, + YRequirementTypeWithOpsBase, +) +from hotsos.core.config import HotSOSConfig +from hotsos.core.host_helpers.filestat import FileObj +from hotsos.core.host_helpers.systemd import SystemdHelper +from hotsos.core.host_helpers.config import IniConfigBase +from hotsos.core.host_helpers.ssl import SSLCertificate + +# Inspired from the pyparsing examples: +# https://github.com/pyparsing/pyparsing/blob/master/examples/eval_arith.py +# https://github.com/pyparsing/pyparsing/blob/master/examples/simpleBool.py +# Enables "packrat" parsing, which adds memoizing to the parsing logic. +# pylint: disable-next=no-value-for-parameter +pp.ParserElement.enablePackrat() + +# ___________________________________________________________________________ # + + +class YExprToken: + """Expression token.""" + + def __init__(self, tokens): + self.tokens = tokens + + def token(self, index): + return self.tokens[index] + + @abstractmethod + def eval(self): + """Inheriting class must implement this.""" + + @staticmethod + def operator_operands(tokenlist): + """generator to extract operator operands in pairs.""" + it = iter(tokenlist) + while 1: + try: + yield (next(it), next(it)) + except StopIteration: + break + + +# ___________________________________________________________________________ # + + +class YExprNotFound: + """Type for indicating a thing is not found. + Allows short-circuiting boolean functions to False, + e.g. systemd('svc-name', 'start_time_secs') > 123 would evaluate to + False if there's no such service named `svc-name`.""" + + def __init__(self, desc): + self.desc = desc + + def __repr__(self): + return f"<`{self.desc}` not found>" + + +class YExprInvalidArgumentException(pp.ParseFatalException): + """Exception to raise when a expression parsing error is occured.""" + + def __init__(self, s, loc, msg): + super().__init__(s, loc, f"invalid argument '{msg}'") + + +# ___________________________________________________________________________ # + + +class YExprLogicalOpBase(YExprToken): + """Base class for logical operators.""" + + logical_fn: Callable[[Iterable[bool]], bool] = lambda _: False + + def eval(self): + # Yield odd operands 'True' 'and' 'False' 'and' 'False' 'and' 'True' + # would yield 'True', 'False', 'False', 'True' + eval_exprs = (t.eval() for t in self.token(0)[::2]) + + return self.logical_fn(eval_exprs) + + +class YExprLogicalAnd(YExprLogicalOpBase): + """And operator. Binary. Supports chaining.""" + + logical_fn = all + + def __repr__(self): + return " and ".join([str(t.eval()) for t in self.token(0)[::2]]) + + +class YExprLogicalOr(YExprLogicalOpBase): + """Or operator. Binary. Supports chaining.""" + + logical_fn = any + + def __repr__(self): + return " or ".join([str(t.eval()) for t in self.token(0)[::2]]) + + +class YExprLogicalNot(YExprToken): + """Not operator. Unary.)""" + + def eval(self): + return not self.token(0)[1].eval() + + +# ___________________________________________________________________________ # + +# pylint: disable-next=abstract-method +class YExprFnBase(YExprToken): + """Common base class for all function implementations.""" + + def arg(self, index): + args = self.tokens[1:][0] + if not len(args) >= (index + 1): + return None + return args[index] + + +class YExprFnLen(YExprFnBase): + """len(expr) function implementation.""" + + def eval(self): + v = self.arg(0).eval() + return len(v) if v else 0 + + +class YExprFnNot(YExprFnBase): + """not(expr) function implementation.""" + + def eval(self): + return not self.arg(0).eval() + + +class YExprFnFile(YExprFnBase): + """fstat(fname, prop[optional]) function implementation.""" + + def eval(self): + file_name = self.arg(0).eval() + fobj = FileObj(file_name) + + if fobj.exists: + if not self.arg(1): + return True + else: + return YExprNotFound(f"{file_name}") + + property_name = self.arg(1).eval() + + if hasattr(fobj, property_name): + return getattr(fobj, property_name) + + raise NoSuchPropertyError(f"Unknown file property {property_name}") + + +class YExprFnSystemd(YExprFnBase): + """systemd(unit_name, ...property) function implementation.""" + + def eval(self): + if not self.arg(0): + raise NotEnoughParametersError("systemd(...) function expects " + "at least one argument.") + if self.arg(2): + raise TooManyParametersError("systemd(...) function expects " + "at most two arguments.") + + service_name = self.arg(0).eval() + service_obj = SystemdHelper([service_name]).services.get(service_name) + + if service_obj: + if not self.arg(1): + return True + else: + return YExprNotFound(f"{service_name}") + + property_name = self.arg(1).eval() + + if hasattr(service_obj, property_name): + return getattr(service_obj, property_name) + + raise NoSuchPropertyError( + f"systemd service `{service_name}` object " + f"has no such property {property_name}" + ) + + +class YExprFnReadIni(YExprFnBase): + """read_ini funciton implementation.""" + + def eval(self): + + if not self.arg(1): + raise NotEnoughParametersError("read_ini(...) function expects " + "at least two arguments.") + if self.arg(4): + raise TooManyParametersError("read_ini(...) function expects " + "at most four arguments.") + + ini_path = self.arg(0).eval() + path = os.path.join(HotSOSConfig.data_root, ini_path) + ini_file = IniConfigBase(path) + if not ini_file.exists: + return YExprNotFound(f"{path}") + + key = self.arg(1).eval() + section = self.arg(2).eval() if self.arg(2) else None + value = ini_file.get(key, section, expand_to_list=False) + + if self.arg(3) and value is None: + return self.arg(3).eval() + + return value + + +class YExprFnReadCert(YExprFnBase): + """read_cert function implementation.""" + + def eval(self): + if not self.arg(0): + raise NotEnoughParametersError("cert(...) function expects " + "at least on argument.") + if self.arg(2): + raise TooManyParametersError("cert(...) function expects " + "at most two arguments.") + + cert_path = self.arg(0).eval() + try: + cert = SSLCertificate(cert_path) + except OSError: + return YExprNotFound(f"{cert_path}") + + # If no property is specified, then return True/False + # indicating that whether the cert file exist or not. + if not self.arg(1): + return cert is not None + + property_name = self.arg(1).eval() + + if hasattr(cert, property_name): + return getattr(cert, property_name) + + raise NoSuchPropertyError( + f"certificate `{cert_path}` object " + f"has no such property {property_name}" + ) + + +# ___________________________________________________________________________ # + + +# pylint: disable-next=abstract-method +class YExprArgBase(YExprToken): + """Base class for all argument types.""" + + +class YExprArgBoolean(YExprArgBase): + """Boolean argument type. Triggered by True and False + keywords (case-insensitive).""" + + def eval(self): + if self.token(0).lower() == "true": + return True + + if self.token(0).lower() == "false": + return False + + raise ValueError(f"Non-boolean string: {self.token(0)}") + + +class YExprArgNone(YExprArgBase): + """Boolean argument type. Triggered by None + keyword (case-insensitive).""" + + def eval(self): + return None + + +class YExprArgStringLiteral(YExprArgBase): + """String literal 'foo'. Triggered by single quotation mark ''.""" + + def eval(self): + return self.token(0) + + +class YExprArgInteger(YExprArgBase): + """Integer argument type. Triggered by [0-9]+""" + + def eval(self): + return int(self.token(0)) + + +class YExprArgFloat(YExprArgBase): + """Integer argument type. Triggered by [0-9]+.[0-9]+""" + + def eval(self): + return float(self.token(0)) + + +class YExprArgRuntimeVariable(YExprArgBase, PythonEntityResolver): + """Runtime variable argument type. Triggered by '@' symbol followed by + any non-whitespace character.""" + + def __init__(self, tokens, context): + YExprArgBase.__init__(self, tokens=tokens) + PythonEntityResolver.__init__(self, context=context) + + def eval(self): + # use PythonEntityResolver to retrieve value associated with + # the given name. + v = self.get_property(self.token(0)[1:]) + return v + + +# ___________________________________________________________________________ # + + +class YExprSignOp(YExprToken): + "Class to evaluate expressions with a leading + or - sign" + + def __init__(self, tokens): + super().__init__(tokens) + self.sign = self.token(0)[0] + + def eval(self): + mult = {"+": 1, "-": -1}[self.sign] + return mult * self.token(0)[1].eval() + + +class YExprPowerOp(YExprToken): + "Class to evaluate power expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Power operation expects at least 3 tokens.") + + if len(self.token(0)) % 2 == 0: + raise TooManyParametersError( + "Power requires odd amount of tokens.") + + result = self.token(0)[-1].eval() + for val in self.token(0)[-3::-2]: + operand = val.eval() + result = operand**result + return result + + +class YExprMulDivOp(YExprToken): + "Class to evaluate multiplication and division expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Mul/div operation expects at least 3 tokens.") + + if len(self.token(0)) % 2 == 0: + raise TooManyParametersError( + "Mul/div requires odd amount of tokens.") + + prod = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + if op == "*": + prod *= val.eval() + elif op == "/": + prod /= val.eval() + else: + raise NameError(f"Unrecognized operation {op}") + + return prod + + +class YExprAddSubOp(YExprToken): + "Class to evaluate addition and subtraction expressions" + + def eval(self): + if len(self.token(0)) < 3: + raise NotEnoughParametersError( + "Add/sub operation expects at least 3 tokens.") + + if len(self.token(0)) % 2 == 0: + raise UnexpectedParameterError( + "Add/sub requires odd amount of tokens.") + + sum_v = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + if op == "+": + sum_v += val.eval() + elif op == "-": + sum_v -= val.eval() + else: + raise NameError(f"Unrecognized operation {op}") + return sum_v + + +class YExprComparisonOp(YExprToken): + "Class to evaluate comparison expressions" + + ops = { + "<": lambda lhs, rhs: lhs < rhs, + "<=": lambda lhs, rhs: lhs <= rhs, + ">": lambda lhs, rhs: lhs > rhs, + ">=": lambda lhs, rhs: lhs >= rhs, + "!=": lambda lhs, rhs: lhs != rhs, + "==": lambda lhs, rhs: lhs == rhs, + # pylint: disable=unnecessary-lambda + "LT": lambda lhs, rhs: YExprComparisonOp.ops["<"](lhs, rhs), + "LE": lambda lhs, rhs: YExprComparisonOp.ops["<="](lhs, rhs), + "GT": lambda lhs, rhs: YExprComparisonOp.ops[">"](lhs, rhs), + "GE": lambda lhs, rhs: YExprComparisonOp.ops[">="](lhs, rhs), + "NE": lambda lhs, rhs: YExprComparisonOp.ops["!="](lhs, rhs), + "EQ": lambda lhs, rhs: YExprComparisonOp.ops["=="](lhs, rhs), + "<>": lambda lhs, rhs: YExprComparisonOp.ops["!="](lhs, rhs), + # pylint:enable=unnecessary-lambda + "IN": lambda lhs, rhs: lhs in rhs, + } + + def eval(self): + lhs = self.token(0)[0].eval() + for op, val in self.operator_operands(self.token(0)[1:]): + op_fn = self.ops[op] + rhs = val.eval() + + # if either of the operands is not found, return False. + if any(isinstance(x, YExprNotFound) for x in [lhs, rhs]): + return False + + if not op_fn(lhs, rhs): + break + lhs = rhs + else: + return True + return False + + +# ___________________________________________________________________________ # + + +def error(exception_type): + """Parser matcher type for raising parse errors.""" + + def raise_exception(s, loc, typ): + raise exception_type(s, loc, typ[0]) + + return pp.Word(pp.printables).setParseAction(raise_exception) + + +def init_parser(context=None): + """Initialize parser for check expressions. + + The grammar currently supports the following constructs: + + Keywords: + None, True, False + built-ins: + integer, float, string literal + runtime: + python properties + functions: + len(...) + not(...) + arithmetic: + - sign + - plus/minus + - exponent + - mul/div + boolean: + - gt/ge + - lt/le + - eq/ne + - in/and/or/not + """ + + # This is a forward declaration because functions can take an expression as + # an argument.. + expr = pp.Forward() + # Define `None` as keyword for None + none_keyword = pp.CaselessKeyword("None") + none_keyword.setParseAction( + lambda s, loc, tokens: YExprArgNone(tokens)) + # Define True & False as their corresponding bool values + # Example: [True, False, TRUE, FALSE, TrUe, FaLsE] + boolean_keywords = pp.CaselessKeyword("True") | pp.CaselessKeyword("False") + boolean_keywords.setParseAction( + lambda s, loc, tokens: YExprArgBoolean(tokens) + ) + # Declare syntax for string literals + # Example: ['this is a test'] + string_literal = pp.QuotedString("'") + string_literal.setParseAction( + lambda s, loc, tokens: YExprArgStringLiteral(tokens) + ) + # Declare syntax for integers + # example: [123, 1, 1234] + integer = pp.Word(pp.nums) + integer.setParseAction( + lambda s, loc, tokens: YExprArgInteger(tokens)) + # Declare syntax for real numbers (float) + # example. [1.3, 1.23] + real = pp.Combine(pp.Word(pp.nums) + "." + pp.Word(pp.nums)) + real.setParseAction( + lambda s, loc, tokens: YExprArgFloat(tokens)) + # Declare syntax for Python runtime properties. + # Properties start with `@` symbol and can contain alphanumeric + '.', '_'` + # example. [@hotsos.module.class.property_1] + python_property = pp.Combine("@" + pp.Word(pp.alphanums + "._-:/")) + python_property.setParseAction( + lambda s, loc, tokens: YExprArgRuntimeVariable(tokens, context) + ) + + # Functions + lpar, rpar = map(pp.Suppress, "()") + function_call_tail = pp.Group( + lpar + pp.Optional(pp.delimited_list(expr)) + rpar) + # Len + len_function = pp.CaselessKeyword("len") + function_call_tail + len_function.setParseAction( + lambda s, loc, tokens: YExprFnLen(tokens)) + # Not + not_function = pp.CaselessKeyword("not") + function_call_tail + not_function.setParseAction( + lambda s, loc, tokens: YExprFnNot(tokens)) + # file(..) + file_function = pp.CaselessKeyword("file") + function_call_tail + file_function.setParseAction( + lambda s, loc, tokens: YExprFnFile(tokens)) + # systemd(...) + systemd_function = pp.CaselessKeyword("systemd") + function_call_tail + systemd_function.setParseAction( + lambda s, loc, tokens: YExprFnSystemd(tokens) + ) + # read_ini(...) + read_ini_function = pp.CaselessKeyword("read_ini") + function_call_tail + read_ini_function.setParseAction( + lambda s, loc, tokens: YExprFnReadIni(tokens) + ) + # cert(...) + read_cert_function = pp.CaselessKeyword("read_cert") + function_call_tail + read_cert_function.setParseAction( + lambda s, loc, tokens: YExprFnReadCert(tokens) + ) + + functions = ( + len_function + | not_function + | file_function + | systemd_function + | read_ini_function + | read_cert_function + ) + + # The order matters. + keywords = boolean_keywords | none_keyword + constants = real | integer | string_literal + operand = functions | keywords | constants | python_property + # use parse actions to attach EvalXXX constructors to sub-expressions + + # Declare arithmetic operations + signop = pp.one_of("+ -") + multop = pp.one_of("* /") + plusop = pp.one_of("+ -") + expop = pp.Literal("**") + arith_expr = pp.infix_notation( + operand, + [ + ( + signop, + 1, + pp.OpAssoc.RIGHT, + lambda s, loc, tokens: YExprSignOp(tokens), + ), + ( + expop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprPowerOp(tokens), + ), + ( + multop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprMulDivOp(tokens), + ), + ( + plusop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprAddSubOp(tokens), + ), + ], + ) + + # Declare comparison/boolean operations + comparisonop = pp.one_of(" ".join( + YExprComparisonOp.ops.keys()), caseless=True) + comp_expr = pp.infix_notation( + arith_expr, + [ + ( + comparisonop, + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprComparisonOp(tokens), + ), + ], + ) + + logical_ops = pp.infix_notation( + comp_expr, + [ + ( + pp.CaselessKeyword("not"), + 1, + pp.OpAssoc.RIGHT, + lambda s, loc, tokens: YExprLogicalNot(tokens), + ), + ( + pp.CaselessKeyword("and"), + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprLogicalAnd(tokens), + ), + ( + pp.CaselessKeyword("or"), + 2, + pp.OpAssoc.LEFT, + lambda s, loc, tokens: YExprLogicalOr(tokens), + ), + ], + ) + + # Append all of them to "expr". Anything that does not match + # to the comp_expr is an error. + expr <<= logical_ops | error(YExprInvalidArgumentException) + # Ignore comments. + expr.ignore(pp.python_style_comment) + expr.ignore(pp.c_style_comment) + return expr + + +class YPropertyExpr(YRequirementTypeWithOpsBase): + """Expression requirement property type.""" + + _override_keys = ["expression"] + _overrride_autoregister = True + + @property + def input(self): + return self.content + + @property + @intercept_exception + def _result(self): + parser = init_parser(context=self.context) + parsed_expr = parser.parse_string(self.input) + result = parsed_expr[0].eval() + if isinstance(result, YExprNotFound): + return False + return result diff --git a/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml b/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml index 55972f407..f9078f203 100644 --- a/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml +++ b/hotsos/defs/scenarios/juju/jujud_machine_checks.yaml @@ -1,8 +1,6 @@ checks: - jujud_not_found: - property: - path: hotsos.core.plugins.juju.JujuChecks.systemd_processes - ops: [[contains, jujud], [not_]] + jujud_not_found: | + NOT 'jujud' IN @hotsos.core.plugins.juju.JujuChecks.systemd_processes conclusions: jujud-not-found: decision: jujud_not_found diff --git a/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml b/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml index 3861eedac..51b0443ac 100644 --- a/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml +++ b/hotsos/defs/scenarios/kernel/amd_iommu_pt.yaml @@ -1,20 +1,12 @@ -vars: - virt_type: '@hotsos.core.plugins.system.SystemBase.virtualisation_type' - cpu_vendor: '@hotsos.core.plugins.kernel.sysfs.CPU.vendor' - kernel_cmd_line: '@hotsos.core.plugins.kernel.KernelBase.boot_parameters' checks: - is_phy_host: - varops: [[$virt_type], [not_]] - cpu_vendor_is_amd: - varops: [[$cpu_vendor], [eq, 'authenticamd']] - iommu_not_pt: - varops: [[$kernel_cmd_line], [contains, 'iommu=pt'], [not_]] + not_using_iommu_passthrough_for_suitable_amd: | + (@hotsos.core.plugins.kernel.sysfs.CPU.vendor == 'authenticamd') AND + (@hotsos.core.plugins.system.SystemBase.virtualisation_type == None) AND + NOT('iommu=pt' IN @hotsos.core.plugins.kernel.KernelBase.boot_parameters) conclusions: mixed-pkg-releases: decision: - - is_phy_host - - cpu_vendor_is_amd - - iommu_not_pt + - not_using_iommu_passthrough_for_suitable_amd raises: type: SystemWarning message: >- diff --git a/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml b/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml index df3c596ee..cdb9f885e 100644 --- a/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml +++ b/hotsos/defs/scenarios/kernel/kernlog_calltrace.yaml @@ -1,24 +1,14 @@ checks: - has_stacktraces: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype - ops: [[length_hint]] - has_oom_killer_invoked: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.oom_killer - ops: [[length_hint]] - has_bcache_deadlock_invoked: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace-bcache - ops: [[length_hint]] - has_hungtasks: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask - ops: [[length_hint]] - has_fanotify_hang: - property: - path: hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify - ops: [[length_hint]] + has_stacktraces: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype) > 0 + has_oom_killer_invoked: | + len(@hotsos.core.plugins.kernel.CallTraceManager.oom_killer) > 0 + has_bcache_deadlock_invoked: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace-bcache) > 0 + has_hungtasks: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask) > 0 + has_fanotify_hang: | + len(@hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify) > 0 conclusions: stacktraces: # Give this one lowest priority so that if any other call trace types match they @@ -30,7 +20,7 @@ conclusions: message: >- {numreports} reports of stacktraces in kern.log - please check. format-dict: - numreports: '@checks.has_stacktraces.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace_anytype:len oom-killer-invoked: priority: 2 decision: has_oom_killer_invoked @@ -39,7 +29,7 @@ conclusions: message: >- {numreports} reports of oom-killer invoked in kern.log - please check. format-dict: - numreports: '@checks.has_oom_killer_invoked.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.oom_killer:len bcache-deadlock-invoked: priority: 2 decision: has_bcache_deadlock_invoked @@ -60,7 +50,7 @@ conclusions: message: >- {numreports} reports of hung tasks in kern.log - please check. format-dict: - numreports: '@checks.has_hungtasks.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace_hungtask:len fanotify_hangs: priority: 2 decision: has_fanotify_hang @@ -70,4 +60,4 @@ conclusions: {numreports} reports of fanotify related hangs in kern.log. This may be related to antivirus software running in the system. format-dict: - numreports: '@checks.has_fanotify_hang.requires.value_actual:len' + numreports: hotsos.core.plugins.kernel.CallTraceManager.calltrace-fanotify:len diff --git a/hotsos/defs/scenarios/kernel/memory.yaml b/hotsos/defs/scenarios/kernel/memory.yaml index 49ef9ac6b..3df9dd1ae 100644 --- a/hotsos/defs/scenarios/kernel/memory.yaml +++ b/hotsos/defs/scenarios/kernel/memory.yaml @@ -4,11 +4,6 @@ vars: compact_success: '@hotsos.core.plugins.kernel.memory.VMStat.compact_success' compaction_failures_percent: '@hotsos.core.plugins.kernel.memory.VMStat.compaction_failures_percent' slab_major_consumers: '@hotsos.core.plugins.kernel.memory.SlabInfo.major_consumers' - # We use an arbitrary threshold of 10k to suggest that a lot of - # compaction has occurred but noting that this is a rolling counter - # and is not necessarily representative of current state. - min_compaction_success: 10000 - max_compaction_failures_pcent: 10 hugetlb_to_mem_total_percentage: '@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_to_mem_total_percentage' mem_avail_to_mem_total_percentage: @@ -18,20 +13,21 @@ vars: mem_total_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.mem_total_gb' mem_available_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.mem_available_gb' hugetlb_gb: '@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_gb' - # Arbitrary thresholds set for the memory allocated for the huge - # pages to total memory and memory available to total memory. - hugetlb_to_mem_total_threshold_percent: 80 - mem_available_to_mem_total_thershold_percent: 3 checks: - low_free_high_order_mem_blocks: - varops: [[$nodes_with_limited_high_order_memory], [length_hint]] - high_compaction_failures: - - varops: [[$compact_success], [gt, $min_compaction_success]] - - varops: [[$compaction_failures_percent], [gt, $max_compaction_failures_pcent]] - too_many_free_hugepages: - - property: hotsos.core.plugins.kernel.memory.MemInfo.huge_pages_enabled - - varops: [[$hugetlb_to_mem_total_percentage], [gt, $hugetlb_to_mem_total_threshold_percent]] - - varops: [[$mem_avail_to_mem_total_percentage], [lt, $mem_available_to_mem_total_thershold_percent]] + low_free_high_order_mem_blocks: | + len(@hotsos.core.plugins.kernel.memory.MemoryChecks.nodes_with_limited_high_order_memory) > 0 + high_compaction_failures: | + # We use an arbitrary threshold of 10k to suggest that a lot of + # compaction has occurred but noting that this is a rolling counter + # and is not necessarily representative of current state. + (@hotsos.core.plugins.kernel.memory.VMStat.compact_success > 10000) AND + (@hotsos.core.plugins.kernel.memory.VMStat.compaction_failures_percent > 10) + too_many_free_hugepages: | + (@hotsos.core.plugins.kernel.memory.MemInfo.huge_pages_enabled == True) AND + # Arbitrary thresholds set for the memory allocated for the huge + # pages to total memory and memory available to total memory. + (@hotsos.core.plugins.kernel.memory.MemInfo.hugetlb_to_mem_total_percentage > 80) AND + (@hotsos.core.plugins.kernel.memory.MemInfo.mem_avail_to_mem_total_percentage < 3) conclusions: low_free_high_order_mem_blocks: decision: low_free_high_order_mem_blocks diff --git a/hotsos/defs/scenarios/kernel/network/misc.yaml b/hotsos/defs/scenarios/kernel/network/misc.yaml index 1243c624d..a69389011 100644 --- a/hotsos/defs/scenarios/kernel/network/misc.yaml +++ b/hotsos/defs/scenarios/kernel/network/misc.yaml @@ -7,8 +7,8 @@ checks: # or # "Jun 08 10:48:13 compute4 kernel:" expr: '(\w{3,5}\s+\d{1,2}\s+[\d:]+)\S+.+ nf_conntrack: table full, dropping packet' - has_over_mtu_dropped_packets: - property: hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets + has_over_mtu_dropped_packets: | + len(@hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets) > 0 conclusions: nf-conntrack-full: decision: has_nf_conntrack_full @@ -27,4 +27,4 @@ conclusions: This host is reporting over-mtu dropped packets for ({num_ifaces}) interfaces. See kern.log for full details. format-dict: - num_ifaces: '@checks.has_over_mtu_dropped_packets.requires.value_actual:len' + num_ifaces: hotsos.core.plugins.kernel.kernlog.KernLogEvents.over_mtu_dropped_packets:len diff --git a/hotsos/defs/scenarios/kernel/network/netlink.yaml b/hotsos/defs/scenarios/kernel/network/netlink.yaml index e67907faa..688252fed 100644 --- a/hotsos/defs/scenarios/kernel/network/netlink.yaml +++ b/hotsos/defs/scenarios/kernel/network/netlink.yaml @@ -1,6 +1,6 @@ checks: - has_socks_with_drops: - property: hotsos.core.plugins.kernel.net.NetLink.all_with_drops + has_socks_with_drops: | + len(@hotsos.core.plugins.kernel.net.NetLink.all_with_drops) > 0 conclusions: netlink-socks-with-drops: decision: has_socks_with_drops diff --git a/hotsos/defs/scenarios/kernel/network/tcp.yaml b/hotsos/defs/scenarios/kernel/network/tcp.yaml index c57c7c2d9..635bff764 100644 --- a/hotsos/defs/scenarios/kernel/network/tcp.yaml +++ b/hotsos/defs/scenarios/kernel/network/tcp.yaml @@ -1,71 +1,42 @@ -vars: - incsumerr: '@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors' - incsumrate_pcent: '@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs' - outsegs: '@hotsos.core.plugins.kernel.net.SNMPTcp.OutSegs' - retrans: '@hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegs' - outretrans_pcent: '@hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs' - spurrtx: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueues' - spurrtx_pcent: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs' - prunec: '@hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled' - rcvcoll: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvCollapsed' - rcvpr: '@hotsos.core.plugins.kernel.net.NetStatTCP.RcvPruned' - ofopr: '@hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned' - backlogd: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop' - rpfilterd: '@hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter' - ldrop: '@hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops' - pfmemd: '@hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop' - minttld: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop' - listenovf: '@hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows' - ofod: '@hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned' - zwind: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop' - rcvqd: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop' - rcvqd_pcent: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs' - rqfulld: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop' - rqfullcook: '@hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies' - memusage_pages_inuse: '@hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages' - memusage_pages_max: '@hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax' - memusage_pct: '@hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct' checks: - incsumerr_high: - or: - - varops: [[$incsumerr], [gt, 500]] - - varops: [[$incsumrate_pcent], [gt, 1]] - retrans_out_rate_gt_1pcent: - varops: [[$outretrans_pcent], [gt, 1]] - incsumrate_gt_1pcent: - varops: [[$incsumrate_pcent], [gt, 1]] - mem_pressure_prunec: - varops: [[$prunec], [gt, 500]] - backlogd: - varops: [[$backlogd], [gt, 500]] - rpfilterd: - varops: [[$rpfilterd], [gt, 500]] - ldrop: - varops: [[$ldrop], [gt, 500]] - spurrtx_gt_1pcent: - varops: [[$spurrtx_pcent], [gt, 1]] - pfmemd: - varops: [[$pfmemd], [gt, 500]] - minttld: - varops: [[$minttld], [gt, 500]] - listenovf: - varops: [[$listenovf], [gt, 500]] - ofod: - varops: [[$ofod], [gt, 500]] - zwind: - varops: [[$zwind], [gt, 500]] - rcvqd: - or: - - varops: [[$rcvqd], [gt, 500]] - - varops: [[$rcvqd_pcent], [gt, 1]] - rqfulld: - varops: [[$rqfulld], [gt, 500]] - rqfullcook: - varops: [[$rqfullcook], [gt, 500]] - memusage_pct_high: - varops: [[$memusage_pct], [gt, 85]] - memusage_pct_exhausted: - varops: [[$memusage_pct], [ge, 100]] + incsumerr_high: | + (@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors > 500) OR + (@hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs > 1) + retrans_out_rate_gt_1pcent: | + @hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs > 1 + incsumrate_gt_1pcent: | + @hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs > 1 + mem_pressure_prunec: | + @hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled > 500 + backlogd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop > 500 + rpfilterd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter > 500 + ldrop: | + @hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops > 500 + spurrtx_gt_1pcent: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs > 1 + pfmemd: | + @hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop > 500 + minttld: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop > 500 + listenovf: | + @hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows > 500 + ofod: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPOFODrop > 500 + zwind: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop > 500 + rcvqd: | + (@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop > 500) OR + (@hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs > 1) + rqfulld: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop > 500 + rqfullcook: | + @hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies > 500 + memusage_pct_high: | + @hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct > 85 + memusage_pct_exhausted: | + @hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct >= 100 conclusions: # retransmissions incsumerr_high: @@ -76,23 +47,23 @@ conclusions: tcp ingress checksum errors are at {count}. This could mean that one or more interfaces are experiencing hardware errors. format-dict: - count: $incsumerr + count: hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrors retrans_out_rate_gt_1pcent: decision: retrans_out_rate_gt_1pcent raises: type: KernelWarning message: tcp retransmissions ({retrans}) are at {rate}% of tx segments ({outsegs}). format-dict: - outsegs: $outsegs - retrans: $retrans - rate: $outretrans_pcent + outsegs: hotsos.core.plugins.kernel.net.SNMPTcp.OutSegs + retrans: hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegs + rate: hotsos.core.plugins.kernel.net.SNMPTcp.RetransSegsPcentOutSegs incsumrate_gt_1pcent: decision: incsumrate_gt_1pcent raises: type: KernelWarning message: tcp ingress checksum error rate is at {rate}% of rx segments. format-dict: - rate: $incsumrate_pcent + rate: hotsos.core.plugins.kernel.net.SNMPTcp.InCsumErrorsPcentInSegs # mem pressure mem_pressure_prunec: decision: mem_pressure_prunec @@ -102,10 +73,10 @@ conclusions: tcp pruned {prunec} times: collapsed={rcvcoll} recv={rcvpr} ofo={ofopr}. format-dict: - prunec: $prunec - rcvcoll: $rcvcoll - rcvpr: $rcvpr - ofopr: $ofopr + prunec: hotsos.core.plugins.kernel.net.NetStatTCP.PruneCalled + rcvcoll: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvCollapsed + rcvpr: hotsos.core.plugins.kernel.net.NetStatTCP.RcvPruned + ofopr: hotsos.core.plugins.kernel.net.NetStatTCP.OfoPruned # misc drops backlogd: decision: backlogd @@ -113,64 +84,64 @@ conclusions: type: KernelWarning message: tcp socket backlog queue full (drops={count}). format-dict: - count: $backlogd + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPBacklogDrop pfmemd: decision: pfmemd raises: type: KernelWarning message: PFMEMALLOC skb to non-MEMALLOC socket (drops={count}). format-dict: - count: $pfmemd + count: hotsos.core.plugins.kernel.net.NetStatTCP.PFMemallocDrop minttld: decision: minttld raises: type: KernelWarning message: IP TTL below minimum (drops={count}). format-dict: - count: $minttld + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPMinTTLDrop rpfilterd: decision: rpfilterd raises: type: KernelWarning message: Failed reverse path filter test (drops={count}). format-dict: - count: $rpfilterd + count: hotsos.core.plugins.kernel.net.NetStatTCP.IPReversePathFilter listenovf: decision: listenovf raises: type: KernelWarning message: tcp accept queue overflow (drops={count}). format-dict: - count: $listenovf + count: hotsos.core.plugins.kernel.net.NetStatTCP.ListenOverflows ldrop: decision: ldrop raises: type: KernelWarning message: tcp incoming connect request catch-all (drops={count}). format-dict: - count: $ldrop + count: hotsos.core.plugins.kernel.net.NetStatTCP.ListenDrops ofod: decision: ofod raises: type: KernelWarning message: tcp no rmem adding to OFO recv queue (drops={count}). format-dict: - count: $ofod + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPOFODrop zwind: decision: zwind raises: type: KernelWarning message: tcp receive window full (drops={count}). format-dict: - count: $zwind + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPZeroWindowDrop rcvqd: decision: rcvqd raises: type: KernelWarning message: tcp no rmem adding to recv queue (drops={count}, {pcent})% of total rx). format-dict: - count: $rcvqd - pcent: $rcvqd_pcent + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDrop + pcent: hotsos.core.plugins.kernel.net.NetStatTCP.TCPRcvQDropPcentInSegs # synflood rqfulld: decision: rqfulld @@ -178,14 +149,14 @@ conclusions: type: KernelWarning message: tcp request queue full, syncookies off (drops={count}). format-dict: - count: $rqfulld + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDrop rqfullcook: decision: rqfullcook raises: type: KernelWarning message: tcp request queue full, syncookies on (cookies={count}). format-dict: - count: $rqfullcook + count: hotsos.core.plugins.kernel.net.NetStatTCP.TCPReqQFullDoCookies # misc spurrtx_gt_1pcent: decision: spurrtx_gt_1pcent @@ -194,8 +165,8 @@ conclusions: message: >- this host has {num} tcp retransmissions ({pcent}% of total) with original still queued. format-dict: - num: $spurrtx - pcent: $spurrtx_pcent + num: hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueues + pcent: hotsos.core.plugins.kernel.net.NetStatTCP.TCPSpuriousRtxHostQueuesPcentOutSegs memusage_pct_high: decision: - memusage_pct_high @@ -206,9 +177,9 @@ conclusions: TCP memory page usage is high ({pct}%), ({pa} out of {pm} mem pages are in use). Kernel will start to drop TCP frames if all pages are exhausted. format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct memusage_pct_exhausted: decision: memusage_pct_exhausted raises: @@ -217,6 +188,6 @@ conclusions: All TCP memory pages are exhausted! ({pa} out of {pm} mem pages are in use). Kernel will drop TCP packets, expect packet losses on ALL TCP transport! format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobTcpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlTcpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.TCPMemUsagePct diff --git a/hotsos/defs/scenarios/kernel/network/udp.yaml b/hotsos/defs/scenarios/kernel/network/udp.yaml index d970562ba..08b84fae2 100644 --- a/hotsos/defs/scenarios/kernel/network/udp.yaml +++ b/hotsos/defs/scenarios/kernel/network/udp.yaml @@ -1,36 +1,20 @@ -vars: - inerrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.InErrors' - inerrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams' - rcvbuferrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors' - rcvbuferrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams' - sndbuferrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors' - sndbuferrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams' - incsumerrors: '@hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors' - incsumerrors_pcent: '@hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams' - memusage_pages_inuse: '@hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages' - memusage_pages_max: '@hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax' - memusage_pct: '@hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct' checks: - rcvbuferrors_high: - or: - - varops: [[$rcvbuferrors], [gt, 500]] - - varops: [[$rcvbuferrors_pcent], [gt, 1]] - sndbuferrors_high: - or: - - varops: [[$sndbuferrors], [gt, 500]] - - varops: [[$sndbuferrors_pcent], [gt, 1]] - inerrs_high_pcent_or_above_limit: - or: - - varops: [[$inerrors], [gt, 500]] - - varops: [[$inerrors_pcent], [gt, 1]] - incsumerrs_high_or_above_limit: - or: - - varops: [[$incsumerrors], [gt, 500]] - - varops: [[$incsumerrors_pcent], [gt, 1]] - memusage_pct_high: - varops: [[$memusage_pct], [gt, 85]] - memusage_pct_exhausted: - varops: [[$memusage_pct], [ge, 100]] + rcvbuferrors_high: | + @hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams > 1 + sndbuferrors_high: | + @hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams > 1 + inerrs_high_pcent_or_above_limit: | + @hotsos.core.plugins.kernel.net.SNMPUdp.InErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams > 1 + incsumerrs_high_or_above_limit: | + @hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors > 500 OR + @hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams > 1 + memusage_pct_high: | + @hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct > 85 + memusage_pct_exhausted: | + @hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct >= 100 conclusions: inerrs_high_pcent_or_above_limit: decision: inerrs_high_pcent_or_above_limit @@ -39,8 +23,8 @@ conclusions: message: >- UDP ingress errors are at {count} ({pcent}% of total datagrams). format-dict: - count: $inerrors - pcent: $inerrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.InErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.InErrorsPcentInDatagrams incsumerrs_high_or_above_limit: decision: incsumerrs_high_or_above_limit raises: @@ -49,8 +33,8 @@ conclusions: UDP ingress checksum errors are at {count} ({pcent}% of total datagrams). This could mean that one or more interfaces are experiencing hardware errors. format-dict: - count: $incsumerrors - pcent: $incsumerrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.InCsumErrorsPcentInDatagrams rcvbuferrors_high: decision: rcvbuferrors_high raises: @@ -58,8 +42,8 @@ conclusions: message: >- UDP receive buffer errors are at {count} ({pcent}% of total rx). format-dict: - count: $rcvbuferrors - pcent: $rcvbuferrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.RcvbufErrorsPcentInDatagrams sndbuferrors_high: decision: sndbuferrors_high raises: @@ -67,8 +51,8 @@ conclusions: message: >- UDP send buffer errors are at {count} ({pcent}% of total tx). format-dict: - count: $sndbuferrors - pcent: $sndbuferrors_pcent + count: hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrors + pcent: hotsos.core.plugins.kernel.net.SNMPUdp.SndbufErrorsPcentOutDatagrams memusage_pct_high: decision: - memusage_pct_high @@ -79,9 +63,9 @@ conclusions: UDP memory page usage is high ({pct}%), ({pa} out of {pm} mem pages are in use). Kernel will start to drop UDP frames if all pages are exhausted. format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct memusage_pct_exhausted: decision: memusage_pct_exhausted raises: @@ -90,6 +74,6 @@ conclusions: All UDP memory pages are exhausted! ({pa} out of {pm} mem pages are in use). Kernel will drop UDP packets, expect packet losses on ALL UDP transport! format-dict: - pa: $memusage_pages_inuse - pm: $memusage_pages_max - pct: $memusage_pct + pa: hotsos.core.plugins.kernel.net.SockStat.GlobUdpSocksTotalMemPages + pm: hotsos.core.plugins.kernel.net.SockStat.SysctlUdpMemMax + pct: hotsos.core.plugins.kernel.net.SockStat.UDPMemUsagePct diff --git a/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml b/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml index 5959e2d1b..b67d05544 100644 --- a/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml +++ b/hotsos/defs/scenarios/kubernetes/system_cpufreq_mode.yaml @@ -11,17 +11,14 @@ vars: order for changes to persist. scaling_governor: '@hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all' checks: - cpufreq_governor_not_performance: - # can we actually see the setting - - varops: [[$scaling_governor], [ne, unknown]] - # does it have the expected value - - varops: [[$scaling_governor], [ne, performance]] + cpufreq_governor_not_performance: | + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'unknown' AND + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'performance' kubernetes_installed: - snap: kubelet # ignore if not running on metal - - property: - path: hotsos.core.plugins.system.system.SystemBase.virtualisation_type - ops: [[eq, null]] + - expression: | + @hotsos.core.plugins.system.system.SystemBase.virtualisation_type == None ondemand_installed_and_enabled: systemd: ondemand: enabled diff --git a/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml b/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml index fc4f7cfa6..c5e49aa30 100644 --- a/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml +++ b/hotsos/defs/scenarios/lxd/lxcfs_deadlock.yaml @@ -1,12 +1,8 @@ checks: - is_not_a_lxc_container: - property: - path: hotsos.core.plugins.system.SystemBase.virtualisation_type - ops: [[ne, 'lxc']] - has_lxc_containers: - property: - path: hotsos.core.plugins.lxd.LXD.instances - ops: [[length_hint], [gt, 0]] + is_not_a_lxc_container: | + @hotsos.core.plugins.system.SystemBase.virtualisation_type != 'lxc' + has_lxc_containers: | + len(@hotsos.core.plugins.lxd.LXD.instances) > 0 has_lxd_version_5_9: snap: lxd: diff --git a/hotsos/defs/scenarios/openstack/eol.yaml b/hotsos/defs/scenarios/openstack/eol.yaml index 04f85ecbb..1f7d9cef4 100644 --- a/hotsos/defs/scenarios/openstack/eol.yaml +++ b/hotsos/defs/scenarios/openstack/eol.yaml @@ -1,8 +1,6 @@ checks: - is_eol: - property: - path: hotsos.core.plugins.openstack.OpenStackChecks.days_to_eol - ops: [[le, 0]] + is_eol: | + @hotsos.core.plugins.openstack.OpenStackChecks.days_to_eol <= 0 conclusions: is-eol: decision: is_eol diff --git a/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml b/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml index 480b09b3a..389c1ccb7 100644 --- a/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml +++ b/hotsos/defs/scenarios/openstack/openstack_apache2_certificates.yaml @@ -1,10 +1,8 @@ checks: - ssl_enabled: - property: hotsos.core.plugins.openstack.OpenstackBase.ssl_enabled - apache2_certificate_expiring: - property: - path: hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring - ops: [[ne, []]] + ssl_enabled: | + @hotsos.core.plugins.openstack.OpenstackBase.ssl_enabled == True + apache2_certificate_expiring: | + len(@hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring) > 0 conclusions: need-certificate-renewal: decision: @@ -16,5 +14,5 @@ conclusions: The following certificates will expire in less than {apache2-certificates-days-to-expire} days: {apache2-certificates-path} format-dict: - apache2-certificates-path: '@checks.apache2_certificate_expiring.requires.value_actual:comma_join' - apache2-certificates-days-to-expire: 'hotsos.core.plugins.openstack.OpenstackBase.certificate_expire_days' + apache2-certificates-path: hotsos.core.plugins.openstack.OpenstackBase.apache2_certificates_expiring:comma_join + apache2-certificates-days-to-expire: hotsos.core.plugins.openstack.OpenstackBase.certificate_expire_days diff --git a/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml b/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml index 63c2d26b5..ae4d32eb1 100644 --- a/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml +++ b/hotsos/defs/scenarios/openstack/openstack_charm_conflicts.yaml @@ -1,12 +1,10 @@ -vars: - local_charms: '@hotsos.core.plugins.juju.JujuChecks.charms' checks: - neutron_conflicts: - - varops: [[$local_charms], [contains, neutron-api]] - - varops: [[$local_charms], [contains, neutron-gateway]] - nova_conflicts: - - varops: [[$local_charms], [contains, nova-cloud-controller]] - - varops: [[$local_charms], [contains, nova-compute]] + neutron_conflicts: | + 'neutron-api' IN @hotsos.core.plugins.juju.JujuChecks.charms AND + 'neutron-gateway' IN @hotsos.core.plugins.juju.JujuChecks.charms + nova_conflicts: | + 'nova-cloud-controller' IN @hotsos.core.plugins.juju.JujuChecks.charms and + 'nova-compute' IN @hotsos.core.plugins.juju.JujuChecks.charms conclusions: neutron_charm_conflicts: decision: neutron_conflicts diff --git a/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml b/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml index e5cfe0869..7b09e0be3 100644 --- a/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml +++ b/hotsos/defs/scenarios/openstack/pkgs_from_mixed_releases_found.yaml @@ -1,8 +1,6 @@ checks: - has_mixed_pkg_releases: - property: - path: hotsos.core.plugins.openstack.OpenStackChecks.installed_pkg_release_names - ops: [[length_hint], [gt, 1]] + has_mixed_pkg_releases: | + len(@hotsos.core.plugins.openstack.OpenStackChecks.installed_pkg_release_names) > 1 conclusions: mixed-pkg-releases: decision: has_mixed_pkg_releases @@ -11,4 +9,4 @@ conclusions: message: >- Openstack packages from a more than one release identified: {releases}. format-dict: - releases: '@checks.has_mixed_pkg_releases.requires.value_actual:comma_join' + releases: hotsos.core.plugins.openstack.OpenStackChecks.installed_pkg_release_names:comma_join diff --git a/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml b/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml index e83c12db6..ed8a1c000 100644 --- a/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml +++ b/hotsos/defs/scenarios/openstack/system_cpufreq_mode.yaml @@ -12,13 +12,10 @@ vars: msg_ondemand: >- You will also need to stop and disable the ondemand systemd service in order for changes to persist. - scaling_governor: '@hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all' checks: - cpufreq_governor_not_performance: - # can we actually see the setting - - varops: [[$scaling_governor], [ne, unknown]] - # does it have the expected value - - varops: [[$scaling_governor], [ne, performance]] + cpufreq_governor_not_performance: | + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'unknown' and + @hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all != 'performance' is_neutron_gateway: # i.e. neutron-l3-agent but no nova == neutron gateway - not: @@ -42,7 +39,7 @@ conclusions: format-dict: msg_pt1: $message_pt1_nova_compute msg_pt2: $message_pt2 - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-n-gw: priority: 1 decision: @@ -55,7 +52,7 @@ conclusions: format-dict: msg_pt1: $message_pt1_neutron_gateway msg_pt2: $message_pt2 - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-with-ondemand-n-cpu: priority: 2 decision: @@ -70,7 +67,7 @@ conclusions: msg_pt1: $message_pt1_nova_compute msg_pt2: $message_pt2 msg_ondemand: $msg_ondemand - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all cpufreq-not-performance-with-ondemand-n-gw: priority: 2 decision: @@ -85,4 +82,4 @@ conclusions: msg_pt1: $message_pt1_neutron_gateway msg_pt2: $message_pt2 msg_ondemand: $msg_ondemand - governor: $scaling_governor + governor: hotsos.core.plugins.kernel.sysfs.CPU.cpufreq_scaling_governor_all diff --git a/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml b/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml index 2de38f8ab..dadddc955 100644 --- a/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml +++ b/hotsos/defs/scenarios/openstack/systemd_masked_services.yaml @@ -1,8 +1,6 @@ checks: - has_unexpected_masked: - property: - path: hotsos.core.plugins.openstack.OpenStackChecks.unexpected_masked_services - ops: [[ne, []]] + has_unexpected_masked: | + len(@hotsos.core.plugins.openstack.OpenStackChecks.unexpected_masked_services) > 0 conclusions: has-unexpected-masked: decision: has_unexpected_masked @@ -13,4 +11,4 @@ conclusions: ensure that this is intended otherwise these services may be unavailable. format-dict: - masked: '@checks.has_unexpected_masked.requires.value_actual:comma_join' + masked: hotsos.core.plugins.openstack.OpenStackChecks.unexpected_masked_services:comma_join diff --git a/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml b/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml index 5c7af947e..75497ae3d 100644 --- a/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml +++ b/hotsos/defs/scenarios/openvswitch/ovn/ovn_central_certs_logs.yaml @@ -7,21 +7,30 @@ vars: cert_expired_expr: '([\d-]+)T([\d:]+)\.\d+Z\|\S+\|stream_ssl\|WARN\|SSL_accept: error:\S+:SSL routines:ssl3_read_bytes:sslv3 alert certificate expired' cert_invalid_expr: '([\d-]+)T([\d:]+)\.\d+Z\|\S+\|stream_ssl\|WARN\|SSL_accept: error:\S+:SSL routines:tls_process_client_certificate:certificate verify failed' checks: - northd_not_restarted_after_cert_update: - - systemd: ovn-northd - - varops: [[$northd_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $northd_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $northd_start_time]] - nbdb_not_restarted_after_cert_update: - - systemd: ovn-ovsdb-server-nb - - varops: [[$ovsdb_nb_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $ovsdb_nb_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_nb_start_time]] - sbdb_not_restarted_after_cert_update: - - systemd: ovn-ovsdb-server-sb - - varops: [[$ovsdb_sb_start_time], [gt, 0]] - - varops: [[$host_cert_mtime], [gt, $ovsdb_sb_start_time]] - - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_sb_start_time]] + northd_not_restarted_after_cert_update: | + systemd('ovn-northd') AND systemd('ovn-northd', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-northd', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-northd', 'start_time_secs') + # - systemd: ovn-northd + # - varops: [[$northd_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $northd_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $northd_start_time]] + nbdb_not_restarted_after_cert_update: | + systemd('ovn-ovsdb-server-nb') AND systemd('ovn-ovsdb-server-nb', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-ovsdb-server-nb', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-ovsdb-server-nb', 'start_time_secs') + # - systemd: ovn-ovsdb-server-nb + # - varops: [[$ovsdb_nb_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $ovsdb_nb_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_nb_start_time]] + sbdb_not_restarted_after_cert_update: | + systemd('ovn-ovsdb-server-sb') AND systemd('ovn-ovsdb-server-sb', 'start_time_secs') > 0 AND + file('etc/ovn/cert_host', 'mtime') > systemd('ovn-ovsdb-server-sb', 'start_time_secs') AND + file('etc/ovn/ovn-central.crt', 'mtime') > systemd('ovn-ovsdb-server-sb', 'start_time_secs') + # - systemd: ovn-ovsdb-server-sb + # - varops: [[$ovsdb_sb_start_time], [gt, 0]] + # - varops: [[$host_cert_mtime], [gt, $ovsdb_sb_start_time]] + # - varops: [[$ovn_central_cert_mtime], [gt, $ovsdb_sb_start_time]] northd_certs_invalid_logs: input: var/log/ovn/ovn-northd.log expr: $cert_invalid_expr diff --git a/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml b/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml index 9c830acfc..3240f33e1 100644 --- a/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml +++ b/hotsos/defs/scenarios/openvswitch/ovn/ovn_certs_valid.yaml @@ -1,30 +1,20 @@ -vars: - ml2_mechanism_driver: '@hotsos.core.plugins.openstack.neutron.Config.mechanism_drivers:plugins/ml2/ml2_conf.ini' - data_root_is_sosreport: '@hotsos.core.plugins.sosreport.SOSReportChecks.data_root_is_sosreport' - ovn_cert_host_exists: '@hotsos.core.host_helpers.filestat.FileFactory.exists:etc/ovn/cert_host' - ovn_cert_host_days: '@hotsos.core.host_helpers.ssl.SSLCertificatesFactory.days_to_expire:etc/ovn/cert_host' - neutron_ml2_cert_host_exists: - '@hotsos.core.host_helpers.filestat.FileFactory.exists:etc/neutron/plugins/ml2/cert_host' - neutron_ml2_cert_host_days: - '@hotsos.core.host_helpers.ssl.SSLCertificatesFactory.days_to_expire:etc/neutron/plugins/ml2/cert_host' checks: - is_not_sosreport_data_root: - varops: [[$data_root_is_sosreport], [ne, true]] + is_not_sosreport_data_root: | + @hotsos.core.plugins.sosreport.SOSReportChecks.data_root_is_sosreport != True is_ovn_central: apt: ovn-central - neutron_ml2_ovn_enabled: - - varops: [[$ml2_mechanism_driver], [ne, null]] - - varops: [[$ml2_mechanism_driver], [contains, 'ovn']] - central_cert_exists: - varops: [[$ovn_cert_host_exists], [truth]] - central_cert_expiring: - varops: [[$ovn_cert_host_days], [lt, 30]] + neutron_ml2_ovn_enabled: | + 'ovn' IN read_ini('etc/neutron/plugins/ml2/ml2_conf.ini', 'mechanism_drivers') + central_cert_exists: | + file('etc/ovn/cert_host', 'exists') + central_cert_expiring: | + read_cert('etc/ovn/cert_host', 'days_to_expire') < 30 is_neutron_server: apt: neutron-server - neutron_cert_exists: - varops: [[$neutron_ml2_cert_host_exists], [truth]] - neutron_cert_expiring: - varops: [[$neutron_ml2_cert_host_days], [lt, 30]] + neutron_cert_exists: | + file('etc/neutron/plugins/ml2/cert_host', 'exists') + neutron_cert_expiring: | + read_cert('etc/neutron/plugins/ml2/cert_host', 'days_to_expire') < 30 conclusions: central_missing_cert: decision: diff --git a/requirements.txt b/requirements.txt index aa597ba41..37d659746 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ simplejson sphinx>=4.3.2 python-dateutil pytz +pyparsing diff --git a/tests/unit/test_yexpr_parser.py b/tests/unit/test_yexpr_parser.py new file mode 100644 index 000000000..7df3f30e5 --- /dev/null +++ b/tests/unit/test_yexpr_parser.py @@ -0,0 +1,431 @@ +from unittest import mock + +from pyparsing import pyparsing_test as ppt +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + init_parser, + YExprInvalidArgumentException, + YExprNotFound +) + +from . import utils + + +class PropertyMock(mock.Mock): + """Property mock for testing.""" + + def __get__(self, obj, obj_type=None): + return self(obj, obj_type) + +# ___________________________________________________________________________# + + +# pylint: disable-next=too-many-public-methods +class TestYExprParser(ppt.TestParseResultsAsserts, utils.BaseTestCase): + """Unit tests for init_parser function.""" + + parser = init_parser() + + def parse_eval(self, expr_str): + return self.parser.parse_string(expr_str)[0].eval() + + def expect_true(self, expr_str): + self.assertTrue(self.parse_eval(expr_str)) + + def expect_false(self, expr_str): + self.assertFalse(self.parse_eval(expr_str)) + + def expect_isinstance(self, expr_str, type_x): + + self.assertTrue(isinstance(self.parse_eval(expr_str), type_x)) + + def test_expr_garbage(self): + input_v = ["\ttest", "g@rb@g€", "$!?/"] + for v in input_v: + with self.assertRaises(YExprInvalidArgumentException): + self.parser.parse_string(v) + + def test_and_expr_true(self): + self.expect_true("TrUe and True and True") + + def test_and_expr_false(self): + self.expect_false("TrUe and False and True") + + def test_and_expr_arg_is_expr(self): + self.expect_true("len('aaaa') == 4 and not False") + + def test_and_expr_not_a_bool(self): + with self.assertRaises(YExprInvalidArgumentException): + self.expect_false("Frue and Talse") + + def test_or_expr_true(self): + self.expect_true("False or True") + + def test_or_expr_false(self): + self.expect_false("False or False") + + def test_or_expr_chain(self): + self.expect_true("False or False or True or False") + + def test_and_or_chain_2(self): + self.expect_false("False or True and True and False") + + def test_not(self): + self.expect_true("not false") + self.expect_false("not true") + self.expect_false("not not not true") + self.expect_true("not None") + + def test_not_expr(self): + self.expect_false("not len('aaa') == 3") + + def test_not_and_or_precedence(self): + self.expect_true("not False and True") + self.expect_true("not False or False") + + def test_fn_len_string_literal(self): + self.assertEqual(self.parse_eval("len('aaaa')"), 4) + + def test_fn_len_none(self): + self.assertEqual(self.parse_eval("len(None)"), 0) + + def test_fn_len_int(self): + with self.assertRaises(TypeError): + self.parse_eval("len(1)") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=["this", "is", "a", "list"]) + def test_fn_len_property(self, _): + self.assertEqual(self.parse_eval("len(@dummy_prop)"), 4) + + def test_fn_not(self): + self.expect_true("False or not(True and False)") + + def test_fn_not_none(self): + self.expect_true("not(None)") + + def test_fn_not_2(self): + self.expect_false("False or not((True and False) or (False or True))") + + def test_fn_not_3(self): + self.expect_true("not(3 == 2)") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_fn_file_prop_exists(self, _, __): + self.expect_true("file('/etc/dummy', 'mtime') == 42") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_fn_file_prop_does_not_exist(self, _): + # Should raise exception + with self.assertRaises(Exception): + self.parse_eval("file('/etc/dummy', 'mtime') == 42") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_exists(self, _): + self.expect_true("systemd('dummy-service')") + + def test_fn_systemd_service_does_not_exist(self): + self.expect_isinstance("systemd('dummy-service')", YExprNotFound) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_and_property_exists(self, mock_systemd_helper): + # pylint: disable=duplicate-code + class DummyMock(mock.MagicMock): + """For testing purposes.""" + + # Create a mock for the services attribute + mock_services = DummyMock() + # Set the return value of get to another mock + mock_service = DummyMock() + mock_services.get.return_value = mock_service + # Set test_prop to a PropertyMock that returns 42 + mock_test_prop = PropertyMock(return_value="this is the value") + type(mock_service).test_prop = mock_test_prop + mock_systemd_helper.return_value.services = mock_services + self.assertEqual(self.parse_eval("systemd('dummy', 'test_prop')"), + "this is the value") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_fn_systemd_service_exists_no_prop(self, mock_systemd_helper): + # Create a mock for the services attribute + mock_services = mock.MagicMock() + # Set the return value of get to another mock + mock_service = object() + mock_services.get.return_value = mock_service + mock_systemd_helper.return_value.services = mock_services + with self.assertRaises(Exception): + self.parse_eval("systemd('dummy', 'test_prop')") + + def test_fn_read_ini_does_not_exist(self): + self.expect_isinstance("read_ini('/etc/no-such.ini', 'field')", + YExprNotFound) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=42) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field')"), + 42) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=56) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key_section(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field', 'sec')"), + 56) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + @mock.patch("builtins.open") + def test_fn_read_ini_read_by_key_default(self, _, __, ___): + self.assertEqual( + self.parse_eval("read_ini('/etc/no-such.ini', 'field', None, 55)"), + 55) + + def test_fn_read_cert_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + self.expect_true("read_cert('/etc/dummy-cert')") + + def test_fn_read_cert_does_not_exist(self): + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.expect_isinstance("read_cert('/etc/dummy-cert')", + YExprNotFound) + self.assertIn("Unable to read SSL certificate file" + " /etc/dummy-cert: [Errno 2] No such " + "file or directory: '/etc/dummy-cert'", + log.output[0]) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SSLCertificate.expiry_date', + new_callable=PropertyMock, + return_value=123456) + def test_fn_read_cert_prop(self, _): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + self.assertEqual( + self.parse_eval("read_cert('/etc/dummy-cert','expiry_date')"), + 123456) + + def test_fn_read_cert_prop_does_not_exist(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as _: + with self.assertRaises(Exception): + self.parse_eval("read_cert('/etc/dummy-cert','expiry_date')") + + def test_none(self): + self.assertEqual(self.parse_eval("None"), None) + + def test_string_literal(self): + self.assertEqual(self.parse_eval("'string literal'"), 'string literal') + + def test_integer_positive(self): + self.assertEqual(self.parse_eval("1234"), 1234) + + def test_integer_negative(self): + self.assertEqual(self.parse_eval("-1234"), -1234) + + def test_float_positive(self): + self.assertEqual(self.parse_eval("1234.56"), 1234.56) + + def test_float_negative(self): + self.assertEqual(self.parse_eval("-1234.56"), -1234.56) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=42) + def test_runtime_variable_exists(self, _): + self.expect_true("@prop.stuff == 42") + + def test_runtime_variable_does_not_exist(self): + with self.assertRaises(Exception): + with self.assertLogs(logger='hotsos', level='ERROR') as log: + self.assertIn("No module named 'prop'", log.output[0]) + self.expect_true("@prop.stuff == 42") + self.parse_eval("@prop.stuff.mod.x == 42") + + def test_sign_op_neg(self): + self.assertEqual(self.parse_eval("-(+1)"), -1) + + def test_sign_op_pos(self): + self.assertEqual(self.parse_eval("+(-1)"), -1) + + def test_power_op(self): + self.assertEqual(self.parse_eval("3**3**2"), 19683) + + def test_mul_op(self): + self.assertEqual(self.parse_eval("5*3"), 15) + + def test_mul_op_2(self): + self.assertEqual(self.parse_eval("-5*3"), -15) + + def test_div_op(self): + self.assertEqual(self.parse_eval("10/2"), 5) + + def test_div_op_2(self): + self.assertEqual(self.parse_eval("-10/2"), -5) + + def test_add_op(self): + self.assertEqual(self.parse_eval("5+3"), 8) + + def test_add_op_string(self): + self.assertEqual(self.parse_eval("'aa'+'bb'"), 'aabb') + + def test_add_op_2(self): + self.assertEqual(self.parse_eval("-5+3"), -2) + + def test_sub_op(self): + self.assertEqual(self.parse_eval("10-2"), 8) + + def test_sub_op_2(self): + self.assertEqual(self.parse_eval("1-2"), -1) + + def test_comparison_lt_true(self): + self.expect_true("-3 < -2") + + def test_comparison_lt_false(self): + self.expect_false("-3 < -4") + + def test_comparison_lte_true(self): + self.expect_true("-3 <= -3") + + def test_comparison_lte_false(self): + self.expect_false("-3 <= -4") + + def test_comparison_gt_true(self): + self.expect_true("-1 > -2") + + def test_comparison_gt_false(self): + self.expect_false("-5 > -4") + + def test_comparison_gte_true(self): + self.expect_true("-1 >= -1") + + def test_comparison_gte_false(self): + self.expect_false("-5 >= -4") + + def test_comparison_eq_true(self): + self.expect_true("-1 == (-2+1)") + + def test_comparison_eq_false(self): + self.expect_false("-5 == -4") + + def test_comparison_ne_true(self): + self.expect_true("(4+5) != (-2+1)") + + def test_comparison_ne_false(self): + self.expect_false("(-3-2) != (5+-10)") + + def test_comparison_in_string_true(self): + self.expect_true("'abcd' in '01234abcdefg'") + + def test_comparison_in_string_false(self): + self.expect_false("'abd' IN '01234abcdefg'") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=[42, 24]) + def test_comparison_in_prop_true(self, _): + self.expect_true("24 in @test.prop") + + def test_expr_complex_0(self): + self.expect_true( + """not((not True or True and False or True) + AND NOT(1 > 0) OR len('abcd') > 3 AND + 'ab' in 'bcabdef' and -5 == -4) AND TRUE + """ + ) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_1(self, _): + expr = """len('abc') > 2 and not(@hotsos.module.class.property_1 < 5.5) + or 3 * (4 + 4) / 2 == 12""" + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + def test_complex_expression_2(self, _, __, ___): + expr = """file('path/to/file', 'mtime') == 42 + and read_ini('config.ini', 'key') + or systemd('service')""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + def test_complex_expression_3(self): + self.expect_false( + """(len('abc') + 5 * 2 ** 3) / (3 - 1) > 10 + and (not(True) or False)""") + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_4(self, _): + expr = """@hotsos.module.class.property_1 + 123 == 456 + or read_cert('cert.pem') and True""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=4) + def test_complex_expression_5(self, _): + expr = """not not(len('test') > 3 or @module.class.property != None) + or file('/path/to/file')""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value="aaaaaa") + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value='value') + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_complex_expression_6(self, mock_systemd_helper, _, __): + mock_systemd_helper.services = mock.MagicMock() + mock_systemd_helper.services.get.return_value = None + expr = """# this is a python style comment + len(@hotsos.module.class.property_1) * 3 >= 10 and + /* and this is a c-style comment*/ + systemd('aaa') or + # another python comment + read_ini('config.ini', 'key') == 'value'""" + with mock.patch("builtins.open", + mock.mock_open(read_data="data")): + self.expect_true(expr) diff --git a/tests/unit/test_yexpr_token.py b/tests/unit/test_yexpr_token.py new file mode 100644 index 000000000..89e375842 --- /dev/null +++ b/tests/unit/test_yexpr_token.py @@ -0,0 +1,902 @@ +from unittest import mock + +from hotsos.core.ycheck.engine.properties.requires.types.expr import ( + YExprToken, + YExprNotFound, + YExprLogicalOpBase, + YExprLogicalAnd, + YExprLogicalOr, + YExprLogicalNot, + YExprFnLen, + YExprFnNot, + YExprFnFile, + YExprFnSystemd, + YExprFnReadIni, + YExprFnReadCert, + YExprArgBoolean, + YExprArgNone, + YExprArgStringLiteral, + YExprArgInteger, + YExprArgFloat, + YExprArgRuntimeVariable, + YExprSignOp, + YExprPowerOp, + YExprMulDivOp, + YExprAddSubOp, + YExprComparisonOp, +) +from hotsos.core.exceptions import ( + NotEnoughParametersError, + TooManyParametersError +) + +from . import utils + +# ___________________________________________________________________________ # + + +class MockToken(YExprToken): + """Mock token for testing.""" + def __init__(self, v): + super().__init__(None) + self.v = v + + def eval(self): + return self.v + + +class PropertyMock(mock.Mock): + """Property mock for testing.""" + + def __get__(self, obj, obj_type=None): + return self(obj, obj_type) + +# ___________________________________________________________________________ # + + +class TestYExprToken(utils.BaseTestCase): + """Unit tests for YExprToken class.""" + + def test_operator_operands(self): + input_v = ("a", "b", "c", "d", "e", "f") + expected = [("a", "b"), ("c", "d"), ("e", "f")] + for i, operands in enumerate(YExprToken.operator_operands(input_v)): + self.assertEqual(operands, expected[i]) + + def test_operator_operands_not_even(self): + input_v = ("a", "b", "c", "d", "e") + expected = [("a", "b"), ("c", "d")] + for i, operands in enumerate(YExprToken.operator_operands(input_v)): + self.assertEqual(operands, expected[i]) + + +# ___________________________________________________________________________ # + + +class TestYExprNotFound(utils.BaseTestCase): + """Unit tests for YExprNotFound class.""" + + def test_not_found(self): + obj = YExprNotFound("dummy") + self.assertEqual(str(obj), "<`dummy` not found>") + + +# ___________________________________________________________________________ # + + +class TestYExprLogicalAnd(utils.BaseTestCase): + """Unit tests for YExprLogicalAnd class.""" + + def test_base_default_fn(self): + self.assertFalse(YExprLogicalOpBase.logical_fn([True])) + + def test_true(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_true_multiple(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), None, + MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_false_multiple(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_repr(self): + uut = YExprLogicalAnd( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + + self.assertEqual(repr(uut), "True and True and False") + + +# ___________________________________________________________________________ # + + +class TestYExprLogicalOr(utils.BaseTestCase): + """Unit tests for YExprLogicalOr class.""" + + def test_true(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_true_multiple(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False), None, + MockToken(True)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_false_multiple(self): + uut = YExprLogicalOr( + [[MockToken(False), None, MockToken(False), + None, MockToken(False)]] + ) + self.assertFalse(uut.eval()) + + def test_repr(self): + uut = YExprLogicalOr( + [[MockToken(True), None, MockToken(True), + None, MockToken(False)]] + ) + + self.assertEqual(repr(uut), "True or True or False") + + +# ___________________________________________________________________________ # + +class TestYExprLogicalNot(utils.BaseTestCase): + """Unit tests for YExprLogicalNot class.""" + + def test_true(self): + uut = YExprLogicalNot( + [[None, MockToken(False)]] + ) + + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprLogicalNot( + [[None, MockToken(True)]] + ) + + self.assertFalse(uut.eval()) + +# ___________________________________________________________________________ # + + +class TestYExprFnLen(utils.BaseTestCase): + """Unit tests for YExprFnLen class.""" + + def test_len_str(self): + input_v = "This is a string." + uut = YExprFnLen( + [None, [MockToken(input_v)]] + ) + self.assertEqual(uut.eval(), len(input_v)) + + def test_len_list(self): + input_v = ['this', 'is', 'a', 'string.'] + uut = YExprFnLen( + [None, [MockToken(input_v)]] + ) + self.assertEqual(uut.eval(), len(input_v)) + +# ___________________________________________________________________________ # + + +class TestYExprFnNot(utils.BaseTestCase): + """Unit tests for YExprFnNot class.""" + + def test_true(self): + uut = YExprFnNot( + [None, [MockToken(False)]] + ) + self.assertTrue(uut.eval()) + + def test_false(self): + uut = YExprFnNot( + [None, [MockToken(True)]] + ) + self.assertFalse(uut.eval()) + +# ___________________________________________________________________________ # + + +class TestYExprFnFile(utils.BaseTestCase): + """Unit tests for YExprFnFile class.""" + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.mtime', new_callable=PropertyMock, + return_value=42) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_prop_exists(self, _, mock_property): + uut = YExprFnFile( + [None, [MockToken('does_not_matter'), MockToken('mtime')]] + ) + self.assertEqual(uut.eval(), 42) + mock_property.assert_called() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_exists_noprop(self, _): + uut = YExprFnFile( + [None, [MockToken('does_not_matter')]] + ) + self.assertEqual(uut.eval(), True) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.FileObj.exists', new_callable=PropertyMock, + return_value=True) + def test_prop_does_not_exists(self, _): + uut = YExprFnFile( + [None, [MockToken('does_not_matter'), MockToken('not_exists')]] + ) + + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________ # + + +class TestYExprFnSystemd(utils.BaseTestCase): + """Unit tests for YExprFnSystemd class.""" + + def test_no_arg(self): + uut = YExprFnSystemd( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnSystemd( + [None, [1, 2, 3]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_exists(self, mock_systemd_helper): + mock_systemd_helper.services = mock.MagicMock() + mock_systemd_helper.services.get.return_value = object() + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter')]] + ) + self.assertTrue(uut.eval()) + +# ___________________________________________________________________________# + + def test_service_does_not_exists(self): + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter')]] + ) + self.assertEqual(str(uut.eval()), + str(YExprNotFound('does_not_matter'))) + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_prop_exists(self, mock_systemd_helper): + # pylint: disable=duplicate-code + class DummyMock(mock.MagicMock): + """ For testing. """ + + # Create a mock for the services attribute + mock_services = DummyMock() + # Set the return value of get to another mock + mock_service = DummyMock() + mock_services.get.return_value = mock_service + # Set test_prop to a PropertyMock that returns 42 + mock_test_prop = PropertyMock(return_value="this is the value") + type(mock_service).test_prop = mock_test_prop + mock_systemd_helper.return_value.services = mock_services + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter'), MockToken('test_prop')]] + ) + self.assertEqual(uut.eval(), "this is the value") + +# ___________________________________________________________________________# + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SystemdHelper') + def test_service_prop_does_not_exists(self, mock_systemd_helper): + # Create a mock for the services attribute + mock_services = mock.MagicMock() + # Set the return value of get to another mock + mock_service = object() + mock_services.get.return_value = mock_service + mock_systemd_helper.return_value.services = mock_services + uut = YExprFnSystemd( + [None, [MockToken('does_not_matter'), MockToken('test_prop')]] + ) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprFnReadIni(utils.BaseTestCase): + """Unit tests for YExprFnReadIni class.""" + + def test_no_arg(self): + uut = YExprFnReadIni( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnReadIni( + [None, [1, 2, 3, 4, 5]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=42) + @mock.patch('builtins.open') + def test_ini_exists(self, _, __, ___): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter")]] + ) + + self.assertEqual(uut.eval(), 42) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=False) + def test_ini_does_not_exists(self, _): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter")]] + ) + self.assertTrue(isinstance(uut.eval(), YExprNotFound)) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + def test_ini_does_not_exists_w_section(self, mock_get, __): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter"), + MockToken("the_section")]] + ) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertEqual(uut.eval(), None) + self.assertIn("cannot parse config file", log.output[0]) + mock_get.assert_called_once_with("does_not_matter", + "the_section", + expand_to_list=False) + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.exists', new_callable=PropertyMock, + return_value=True) + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.IniConfigBase.get', + return_value=None) + def test_ini_does_not_exists_default(self, _, __): + uut = YExprFnReadIni( + [None, [MockToken("does_not_matter"), + MockToken("does_not_matter"), + MockToken(None), + MockToken("default")]] + ) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertEqual(uut.eval(), "default") + self.assertIn("cannot parse config file", log.output[0]) + + +# ___________________________________________________________________________# + + +class TestYExprFnReadCert(utils.BaseTestCase): + """Unit tests for YExprFnReadCert class.""" + + def test_no_arg(self): + uut = YExprFnReadCert( + [None, []] + ) + with self.assertRaises(NotEnoughParametersError): + uut.eval() + +# ___________________________________________________________________________# + + def test_too_many_arg(self): + uut = YExprFnReadCert( + [None, [1, 2, 3]] + ) + with self.assertRaises(TooManyParametersError): + uut.eval() + + def test_cert_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter")]]) + self.assertTrue(uut.eval()) + mock_file.assert_called_once() + + def test_cert_does_not_exists(self): + with mock.patch("builtins.open", mock.mock_open()) as mock_file: + mock_file.side_effect = OSError() + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter")]]) + with self.assertLogs(logger='hotsos', level='WARNING') as log: + self.assertTrue(isinstance(uut.eval(), YExprNotFound)) + self.assertIn("Unable to read SSL certificate", log.output[0]) + mock_file.assert_called_once() + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.SSLCertificate.expiry_date', + new_callable=PropertyMock, + return_value=123456) + def test_cert_read_prop_exists(self, _): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter"), + MockToken("expiry_date")]]) + self.assertEqual(uut.eval(), 123456) + mock_file.assert_called_once() + + def test_cert_read_prop_does_not_exists(self): + with mock.patch("builtins.open", + mock.mock_open(read_data="data")) as mock_file: + uut = YExprFnReadCert( + [None, [MockToken("does_not_matter"), + MockToken("not_a_prop")]]) + with self.assertRaises(Exception): + self.assertEqual(uut.eval(), 123456) + mock_file.assert_called_once() + + +# ___________________________________________________________________________# + + +class TestYExprArgBoolean(utils.BaseTestCase): + """Unit tests for tYExprArgBoolean class.""" + + def test_yexpr_boolean_true(self): + uut = YExprArgBoolean(["TrUe"]) + self.assertTrue(uut.eval()) + + def test_yexpr_boolean_false(self): + uut = YExprArgBoolean(["fAlSe"]) + self.assertFalse(uut.eval()) + + def test_yexpr_boolean_garbage(self): + uut = YExprArgBoolean(["not-a-boolean"]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprArgNone(utils.BaseTestCase): + """Unit tests for YExprArgNone class.""" + + def test_yexpr_none(self): + uut = YExprArgNone(["does-not-matter"]) + self.assertEqual(uut.eval(), None) + +# ___________________________________________________________________________# + + +class TestYExprStringLiteral(utils.BaseTestCase): + """Unit tests for YExprArgStringLiteral class.""" + + def test_yexpr_string_literal(self): + uut = YExprArgStringLiteral(["does-not-matter"]) + self.assertEqual(uut.eval(), "does-not-matter") + +# ___________________________________________________________________________# + + +class TestYExprArgInteger(utils.BaseTestCase): + """Unit tests for YExprArgInteger class.""" + + def test_yexpr_integer_from_int_str(self): + uut = YExprArgInteger(["1"]) + self.assertEqual(uut.eval(), 1) + + def test_yexpr_integer_not_an_int(self): + uut = YExprArgInteger(["garbage"]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_integer_not_an_int_float(self): + uut = YExprArgInteger(["1.1"]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprArgFloat(utils.BaseTestCase): + """Unit tests for YExprArgFloat class.""" + + def test_yexpr_float_from_int_str(self): + uut = YExprArgFloat(["1.1"]) + self.assertEqual(uut.eval(), 1.1) + + def test_yexpr_float_not_an_int(self): + uut = YExprArgFloat(["garbage"]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_float_not_an_int_float(self): + uut = YExprArgFloat(["1"]) + self.assertEqual(uut.eval(), 1.0) + + +# ___________________________________________________________________________# + + +class TestYExprArgRuntimeVariable(utils.BaseTestCase): + """Unit tests for YExprArgRuntimeVariable class.""" + + @mock.patch('hotsos.core.ycheck.engine.properties.requires' + '.types.expr.YExprArgRuntimeVariable.get_property', + return_value=1.1) + def test_yexpr_runtime_variable_exists(self, _): + # (mgilor): 'thisx' was 'this' at first, but it triggers + # a Python easter egg which prints out a poem-like text to the console. + uut = YExprArgRuntimeVariable(["@thisx.is.a.test"], context=None) + self.assertEqual(uut.eval(), float(1.1)) + + def test_yexpr_runtime_variable_does_not_exist(self): + uut = YExprArgRuntimeVariable(["@thisx.is.a.test"], context=None) + with self.assertRaises(Exception): + with self.assertLogs(logger='hotsos', level='ERROR') as log: + uut.eval() + self.assertIn("failed to import class a from thisx.is", + log.output[0]) + +# ___________________________________________________________________________# + + +class TestYExprSignOp(utils.BaseTestCase): + """Unit tests for YExprSignOp class.""" + + def test_yexpr_signop_plus_negative(self): + uut = YExprSignOp([["+", MockToken(-1)]]) + self.assertEqual(uut.eval(), -1) + + def test_yexpr_signop_plus_positive(self): + uut = YExprSignOp([["+", MockToken(1)]]) + self.assertEqual(uut.eval(), 1) + + def test_yexpr_signop_minus_positive(self): + uut = YExprSignOp([["-", MockToken(1)]]) + self.assertEqual(uut.eval(), -1) + + def test_yexpr_signop_minus_negative(self): + uut = YExprSignOp([["-", MockToken(-1)]]) + self.assertEqual(uut.eval(), 1) + +# ___________________________________________________________________________# + + +class TestYExprPowerOp(utils.BaseTestCase): + """Unit tests for YExprPowerOp class.""" + + def test_yexpr_power_op_non_even_args(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(2), "test"]]) + with self.assertRaises(TooManyParametersError): + uut.eval() + + def test_yexpr_power_op_positive_positive(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), 9) + + def test_yexpr_power_op_positive_negative(self): + uut = YExprPowerOp([[MockToken(3), "**", MockToken(-2)]]) + self.assertEqual(round(uut.eval(), 5), 0.11111) + + def test_yexpr_power_op_negative_positive(self): + uut = YExprPowerOp([[MockToken(-3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), 9) + + def test_yexpr_power_op_negative_negative(self): + uut = YExprPowerOp([[MockToken(-3), "**", MockToken(-2)]]) + self.assertEqual(round(uut.eval(), 5), 0.11111) + + def test_yexpr_power_op_chain_multiple(self): + uut = YExprPowerOp([[MockToken(-3), "**", + MockToken(-3), "**", MockToken(2)]]) + self.assertEqual(uut.eval(), -19683) + + def test_yexpr_power_op_not_enough_tokens(self): + uut = YExprPowerOp([[MockToken(-3), "**"]]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprMulDivOp(utils.BaseTestCase): + """Unit tests for YExprMulDivOp class.""" + + def test_yexpr_mult_op_positive_positive(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_mult_op_positive_negative(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(-2)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_mult_op_negative_positive(self): + uut = YExprMulDivOp([[MockToken(-3), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_mult_op_negative_negative(self): + uut = YExprMulDivOp([[MockToken(-3), "*", MockToken(-2)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_mult_op_chain_multiple(self): + uut = YExprMulDivOp([[MockToken(3), "*", + MockToken(2), "*", MockToken(2)]]) + self.assertEqual(uut.eval(), 12) + + def test_yexpr_mult_op_int_float(self): + uut = YExprMulDivOp([[MockToken(3.6), "*", + MockToken(2)]]) + self.assertEqual(uut.eval(), 7.2) + + def test_yexpr_mult_not_enough_args(self): + uut = YExprMulDivOp([[MockToken(3), "*"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_mult_not_odd_num_of_args(self): + uut = YExprMulDivOp([[MockToken(3), "*", MockToken(3), "*"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_div_op_positive_positive(self): + uut = YExprMulDivOp([[MockToken(6), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), 3) + + def test_yexpr_div_op_positive_negative(self): + uut = YExprMulDivOp([[MockToken(6), "/", MockToken(-2)]]) + self.assertEqual(uut.eval(), -3) + + def test_yexpr_div_op_negative_positive(self): + uut = YExprMulDivOp([[MockToken(-6), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), -3) + + def test_yexpr_div_op_negative_negative(self): + uut = YExprMulDivOp([[MockToken(-6), "/", MockToken(-2)]]) + self.assertEqual(uut.eval(), 3) + + def test_yexpr_div_op_chain_multiple(self): + uut = YExprMulDivOp([[MockToken(-12), "/", + MockToken(-3), "/", MockToken(2)]]) + self.assertEqual(uut.eval(), 2) + + def test_yexpr_div_not_enough_args(self): + uut = YExprMulDivOp([[MockToken(3), "/"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_div_not_odd_num_of_args(self): + uut = YExprMulDivOp([[MockToken(3), "/", MockToken(3), "/"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_muldiv_not_a_valid_op_token(self): + uut = YExprMulDivOp([[MockToken(3), "f", MockToken(3)]]) + with self.assertRaises(Exception): + uut.eval() + +# ___________________________________________________________________________# + + +class TestYExprAddSubOp(utils.BaseTestCase): + """Unit tests for YExprAddSubOp class.""" + + def test_yexpr_unrecognized_op(self): + uut = YExprAddSubOp([[MockToken(3), "?", MockToken(3)]]) + with self.assertRaisesRegex(NameError, r"Unrecognized operation \?"): + uut.eval() + + def test_yexpr_add_positive_positive(self): + uut = YExprAddSubOp([[MockToken(3), "+", MockToken(3)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_add_positive_negative(self): + uut = YExprAddSubOp([[MockToken(3), "+", MockToken(-3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_negative_positive(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_negative_negative(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(-3)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_add_chain_multiple(self): + uut = YExprAddSubOp([[MockToken(-3), "+", + MockToken(-3), "+", MockToken(6)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_add_not_enough_args(self): + uut = YExprAddSubOp([[MockToken(-3), "+"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_add_not_odd_number_of_args(self): + uut = YExprAddSubOp([[MockToken(-3), "+", MockToken(-3), "+"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_sub_positive_positive(self): + uut = YExprAddSubOp([[MockToken(3), "-", MockToken(3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_sub_positive_negative(self): + uut = YExprAddSubOp([[MockToken(3), "-", MockToken(-3)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_sub_negative_positive(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(3)]]) + self.assertEqual(uut.eval(), -6) + + def test_yexpr_sub_negative_negative(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(-3)]]) + self.assertEqual(uut.eval(), 0) + + def test_yexpr_sub_chain_multiple(self): + uut = YExprAddSubOp([[MockToken(-3), "-", + MockToken(-3), "+", MockToken(6)]]) + self.assertEqual(uut.eval(), 6) + + def test_yexpr_sub_not_enough_args(self): + uut = YExprAddSubOp([[MockToken(-3), "-"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_sub_not_odd_number_of_args(self): + uut = YExprAddSubOp([[MockToken(-3), "-", MockToken(-3), "-"]]) + with self.assertRaises(Exception): + uut.eval() + + def test_yexpr_addsub_not_a_valid_op_token(self): + uut = YExprMulDivOp([[MockToken(3), "f", MockToken(3)]]) + with self.assertRaises(Exception): + uut.eval() + + +# ___________________________________________________________________________# + + +class TestYExprComparisonOp(utils.BaseTestCase): + """Unit tests for YExprComparisonOp class.""" + + def test_yexpr_comp_op_lt_true(self): + for alt in ["<", "LT"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_lt_false(self): + for alt in ["<", "LT"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_lte_true(self): + for alt in ["<=", "LE"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_lte_false(self): + for alt in ["<=", "LE"]: + uut = YExprComparisonOp([[MockToken(5), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_gt_true(self): + for alt in [">", "GT"]: + uut = YExprComparisonOp([[MockToken(5), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_gt_false(self): + for alt in [">", "GT"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_gte_true(self): + for alt in [">=", "GE"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_gte_false(self): + for alt in [">=", "GE"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_ne_true(self): + for alt in ["!=", "NE", "<>"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(4)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_ne_false(self): + for alt in ["!=", "NE", "<>"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(3)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_eq_true(self): + for alt in ["==", "EQ"]: + uut = YExprComparisonOp([[MockToken(3), alt, MockToken(3)]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_eq_false(self): + for alt in ["==", "EQ"]: + uut = YExprComparisonOp([[MockToken(4), alt, MockToken(3)]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_in_list_true(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken(3), alt, + MockToken([1, 2, 3])]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_in_list_false(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken(3), alt, + MockToken([1, 2, 4])]]) + self.assertFalse(uut.eval()) + + def test_yexpr_comp_op_in_string_true(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken("a"), alt, + MockToken("this is a test")]]) + self.assertTrue(uut.eval()) + + def test_yexpr_comp_op_in_string_false(self): + for alt in ["IN"]: + uut = YExprComparisonOp([[MockToken("a"), alt, + MockToken("this is b test")]]) + self.assertFalse(uut.eval())