diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a0a394215..d994b279e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,15 +18,27 @@ jobs: - name: Run unit tests with python3.12 on el9 python: python3.12 container: ubi10 + - name: Run python linters with python3.12 on el9 + python: python3.12 + container: ubi10-lint - name: Run unit tests with python3.9 on el9 python: python3.9 container: ubi9 + - name: Run python linters with python3.9 on el9 + python: python3.9 + container: ubi9-lint - name: Run unit tests with python 3.6 on el8 python: python3.6 container: ubi8 + - name: Run python linters with python 3.6 on el8 + python: python3.6 + container: ubi8-lint - name: Run unit tests with python2.7 on el7 python: python2.7 container: ubi7 + - name: Run python linters with python2.7 on el7 + python: python2.7 + container: ubi7-lint steps: - name: Checkout code diff --git a/etc/leapp/leapp.conf b/etc/leapp/leapp.conf index d641b3000..a96dcce6a 100644 --- a/etc/leapp/leapp.conf +++ b/etc/leapp/leapp.conf @@ -4,3 +4,5 @@ repo_path=/etc/leapp/repos.d/ [database] path=/var/lib/leapp/leapp.db +[actor_config] +path=/etc/leapp/actor_conf.d/ diff --git a/leapp/actors/__init__.py b/leapp/actors/__init__.py index 9d83bf105..6af036bb5 100644 --- a/leapp/actors/__init__.py +++ b/leapp/actors/__init__.py @@ -1,7 +1,16 @@ +import functools import logging import os import sys +try: + # Python 3.3+ + from collections.abc import Sequence +except ImportError: + # Python 2.6 through 3.2 + from collections import Sequence + +from leapp.actors.config import Config, retrieve_config from leapp.compat import string_types from leapp.dialogs import Dialog from leapp.exceptions import (MissingActorAttributeError, RequestStopAfterPhase, StopActorExecution, @@ -41,6 +50,11 @@ class Actor(object): Write the actor's description as a docstring. """ + config_schemas = () + """ + Defines the structure of the configuration that the actor uses. + """ + consumes = () """ Tuple of :py:class:`leapp.models.Model` derived classes defined in the :ref:`repositories ` @@ -86,6 +100,7 @@ def serialize(self): 'path': os.path.dirname(sys.modules[type(self).__module__].__file__), 'class_name': type(self).__name__, 'description': self.description or type(self).__doc__, + 'config_schemas': [c.__name__ for c in self.config_schemas], 'consumes': [c.__name__ for c in self.consumes], 'produces': [p.__name__ for p in self.produces], 'tags': [t.__name__ for t in self.tags], @@ -100,6 +115,7 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs= This depends on the definition of such a configuration model being defined by the workflow and an actor that provides such a message. """ + Actor.current_instance = self install_translation_for_actor(type(self)) self._messaging = messaging @@ -107,8 +123,12 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs= self.skip_dialogs = skip_dialogs """ A configured logger instance for the current actor. """ + # self._configuration is the workflow configuration. + # self.config_schemas is the actor defined configuration. + # self.config is the actual actor configuration if config_model: self._configuration = next(self.consume(config_model), None) + self.config = retrieve_config(self.config_schemas) self._path = path @@ -359,6 +379,17 @@ def report_error(self, message, severity=ErrorSeverity.ERROR, details=None): actor=self, details=details) + def retrieve_config(self): + """ + Retrieve the configuration specific to the specified schema. + + :param schema: Configuration schemas + :type schema: :py:class:`leapp.models.Model` + :return: Dictionary containing requested configuration. + :rtype: dict + """ + return retrieve_config(self.config_schema) + def _is_type(value_type): def validate(actor, name, value): @@ -390,17 +421,23 @@ def _lint_warn(actor, name, type_name): logging.getLogger("leapp.linter").warning("Actor %s field %s should be a tuple of %s", actor, name, type_name) -def _is_model_tuple(actor, name, value): - if isinstance(value, type) and issubclass(value, Model): - _lint_warn(actor, name, "Models") +def _is_foo_sequence(cls, cls_name, actor, name, value): + if isinstance(value, type) and issubclass(value, cls): + _lint_warn(actor, name, cls_name) value = (value,) - _is_type(tuple)(actor, name, value) - if not all([True] + [isinstance(item, type) and issubclass(item, Model) for item in value]): + _is_type(Sequence)(actor, name, value) + if not all([True] + [isinstance(item, type) and issubclass(item, cls) for item in value]): raise WrongAttributeTypeError( - 'Actor {} attribute {} should contain only Models'.format(actor, name)) + 'Actor {} attribute {} should contain only {}'.format(actor, name, cls_name)) return value +_is_config_sequence = functools.partial(_is_foo_sequence, Config, "Configs") +_is_model_sequence = functools.partial(_is_foo_sequence, Model, "Models") +_is_tag_sequence = functools.partial(_is_foo_sequence, Tag, "Tags") +_is_api_sequence = functools.partial(_is_foo_sequence, WorkflowAPI, "WorkflowAPIs") + + def _is_dialog_tuple(actor, name, value): if isinstance(value, Dialog): _lint_warn(actor, name, "Dialogs") @@ -412,28 +449,6 @@ def _is_dialog_tuple(actor, name, value): return value -def _is_tag_tuple(actor, name, value): - if isinstance(value, type) and issubclass(value, Tag): - _lint_warn(actor, name, "Tags") - value = (value,) - _is_type(tuple)(actor, name, value) - if not all([True] + [isinstance(item, type) and issubclass(item, Tag) for item in value]): - raise WrongAttributeTypeError( - 'Actor {} attribute {} should contain only Tags'.format(actor, name)) - return value - - -def _is_api_tuple(actor, name, value): - if isinstance(value, type) and issubclass(value, WorkflowAPI): - _lint_warn(actor, name, "Apis") - value = (value,) - _is_type(tuple)(actor, name, value) - if not all([True] + [isinstance(item, type) and issubclass(item, WorkflowAPI) for item in value]): - raise WrongAttributeTypeError( - 'Actor {} attribute {} should contain only WorkflowAPIs'.format(actor, name)) - return value - - def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info='', resolve=None): if resolve: value = resolve(actor, name) @@ -464,13 +479,14 @@ def get_actor_metadata(actor): # # if path is not transformed into the realpath. ('path', os.path.dirname(os.path.realpath(sys.modules[actor.__module__].__file__))), _get_attribute(actor, 'name', _is_type(string_types), required=True), - _get_attribute(actor, 'tags', _is_tag_tuple, required=True, additional_info=additional_tag_info), - _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=(), resolve=get_api_models), - _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=(), resolve=get_api_models), + _get_attribute(actor, 'tags', _is_tag_sequence, required=True, additional_info=additional_tag_info), + _get_attribute(actor, 'consumes', _is_model_sequence, required=False, default_value=(), resolve=get_api_models), + _get_attribute(actor, 'produces', _is_model_sequence, required=False, default_value=(), resolve=get_api_models), _get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()), _get_attribute(actor, 'description', _is_type(string_types), required=False, default_value=actor.__doc__ or 'There has been no description provided for this actor.'), - _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=()) + _get_attribute(actor, 'config_schemas', _is_config_sequence, required=False, default_value=()), + _get_attribute(actor, 'apis', _is_api_sequence, required=False, default_value=()) ]) diff --git a/leapp/actors/config.py b/leapp/actors/config.py new file mode 100644 index 000000000..7bfade152 --- /dev/null +++ b/leapp/actors/config.py @@ -0,0 +1,349 @@ +""" +Config file format: + yaml file like this: + +--- +# Note: have to add a fields.Map type before we can use yaml mappings. +section_name: + field1_name: value + field2_name: + - listitem1 + - listitem2 +section2_name: + field3_name: value + +Config files are any yaml files in /etc/leapp/actor_config.d/ +(This is settable in /etc/leapp/leapp.conf) + +""" +__metaclass__ = type + +import abc +import glob +import logging +import os.path +from collections import defaultdict + +import six +import yaml + +from leapp.models.fields import ModelViolationError + + +try: + # Compiled versions if available, for speed + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +_ACTOR_CONFIG = None +_ACTOR_CONFIG_VALIDATED = False + +log = logging.getLogger('leapp.actors.config') + + +class SchemaError(Exception): + """Raised when a schema fails validation.""" + + +class ValidationError(Exception): + """ + Raised when a config file fails to validate against any of the available schemas. + """ + + +# pylint: disable=deprecated-decorator +# @abc.abstractproperty is deprecated in newer Python3 versions but it's +# necessary for Python <= 3.3 (including 2.7) +@six.add_metaclass(abc.ABCMeta) +class Config: + """ + An Actor config schema looks like this. + + :: + class RHUIConfig(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = 'Description here' + default = {"repo": "url"} + """ + @abc.abstractproperty + def section(self): + pass + + @abc.abstractproperty + def name(self): + pass + + @abc.abstractproperty + def type_(self): + pass + + @abc.abstractproperty + def description(self): + pass + + @abc.abstractproperty + def default(self): + pass + + @classmethod + def to_dict(cls): + """ + Return a dictionary representation of the config item that would be suitable for putting + into a config file. + """ + representation = { + cls.section: { + '{0}_description__'.format(cls.name): cls.description + } + } + # TODO: Retrieve the default values from the type field. + # Something like this maybe: + # representation[cls.section][cls.name] = cls.type_.get_default() + + return representation + + @classmethod + def serialize(cls): + """ + :return: Serialized information for the config + """ + return { + 'class_name': cls.__name__, + 'section': cls.section, + 'name': cls.name, + 'type': cls.type_.serialize(), + 'description': cls.description, + 'default': cls.default, + } +# pylint: enable=deprecated-decorator + + +def _merge_config(configuration, new_config): + """ + Merge two dictionaries representing configuration. fields in new_config overwrite + any existing fields of the same name in the same section in configuration. + """ + for section_name, section in new_config.items(): + if section_name not in configuration: + configuration[section_name] = section + else: + for field_name, field in section: + configuration[section_name][field_name] = field + + +def _get_config(config_dir='/etc/leapp/actor_conf.d'): + """ + Read all configuration files from the config_dir and return a dict with their values. + """ + config_files = glob.glob(os.path.join(config_dir, '*'), recursive=True) + config_files = [f for f in config_files if f.endswith('.yml') or f.endswith('.yaml')] + config_files.sort() + + configuration = {} + for config_file in config_files: + with open(config_file) as f: + raw_cfg = f.read() + + try: + parsed_config = yaml.load(raw_cfg, SafeLoader) + except Exception as e: + log.warning("Warning: unparsable yaml file %s in the config directory." + " Error: %s", config_file, str(e)) + raise + + _merge_config(configuration, parsed_config) + + return configuration + + +def normalize_schemas(schemas): + """ + Merge all schemas into a single dictionary and validate them for errors we can detect. + """ + added_fields = set() + normalized_schema = defaultdict(dict) + for schema in schemas: + for field in schema: + unique_name = (field.section, field.name) + + # Error if the field has been added by another schema + if unique_name in added_fields and added_fields[unique_name] != field: + # TODO: Also include information on what Actor contains the + # conflicting fields but that information isn't passed into + # this function right now. + message = ('Two actors added incompatible configuration items' + ' with the same name for Section: {section},' + ' Field: {field}'.format(section=field.section, + field=field.name)) + log.error(message) + raise SchemaError(message) + + # TODO: More validation here. + + # Store the fields from the schema in a way that we can easily look + # up while validating + added_fields.add(unique_name) + normalized_schema[field.section][field.name] = field + + return normalized_schema + + +def _validate_field_type(field_type, field_value, field_path): + """ + Return False if the field is not of the proper type. + + :param str field_path: Path in the config where the field is placed. + Example: A field 'target_clients' in a section 'rhui' would have a path 'rhui.target_clients' + """ + # TODO: I took a quick look at the Model code and this is what I came up + # with. This might not work right or there might be a much better way. + try: + # the name= parameter is displayed in error messages to let the user know what precisely is wrong + field_type._validate_model_value(field_value, name=field_path) + except ModelViolationError as e: # pylint: disable=broad-exception-caught,broad-except + # Any problems mean that the field did not validate. + log.info("Configuration value failed to validate with: %s", e) + return False + return True + + +def _normalize_config(actor_config, schema): + # Validate that the config values read from the config files obey the known + # structure. + for section_name, section in actor_config.items(): + if section_name not in schema: + # TODO: Also have information about which config file contains the unknown field. + message = "A config file contained an unknown section: {section}".format(section=section_name) + log.warning(message) + continue + + for field_name in actor_config: + # Any field names which end in "__" are reserved for LEAPP to use + # for its purposes. In particular, it places documentation of + # a field's value into these reserved field names. + if field_name.endswith("__"): + continue + + if field_name not in schema[section_name]: + # TODO: Also have information about which config file contains the unknown field. + message = ("A config file contained an unknown field: (Section:" + " {section}, Field: {field})".format( + section=section_name, field=field_name) + ) + log.warning(message) + + # Do several things: + # * Validate that the config values are of the proper types. + # * Add default values where no config value was provided. + normalized_actor_config = {} + for section_name, section in schema.items(): + for field_name, field in section.items(): + # TODO: We might be able to do this using the default piece of + # model.fields.Field(). Something using + # schema[section_name, field_name].type_ with the value from + # actor_config[section_name][field_name]. But looking at the Model + # code, I wasn't quite sure how this should be done so I think this + # will work for now. + + # For every item in the schema, either retrieve the value from the + # config files or set it to the default. + try: + value = actor_config[section_name][field_name] + except KeyError: + # Either section_name or field_name doesn't exist + section = actor_config[section_name] = actor_config.get(section_name, {}) + # May need to deepcopy default if these values are modified. + # However, it's probably an error if they are modified and we + # should possibly look into disallowing that. + value = field.default + section[field_name] = value + + field_path = '{0}.{1}'.format(section_name, field_name) + if not _validate_field_type(field.type_, value, field_path): + raise ValidationError("Config value for (Section: {section}," + " Field: {field}) is not of the correct" + " type".format(section=section_name, + field=field_name) + ) + + normalized_section = normalized_actor_config.get(section_name, {}) + normalized_section[field_name] = value + # If the section already exists, this is a no-op. Otherwise, it + # sets it to the newly created dict. + normalized_actor_config[section_name] = normalized_section + + return normalized_actor_config + + +def load(config_dir, schemas): + """ + Return Actor Configuration. + + :returns: a dict representing the configuration. + :raises ValueError: if the actor configuration does not match the schema. + + This function reads the config, validates it, and adds any default values. + """ + global _ACTOR_CONFIG + if _ACTOR_CONFIG: + return _ACTOR_CONFIG + + config = _get_config(config_dir) + config = _normalize_config(config, schemas) + + _ACTOR_CONFIG = config + return _ACTOR_CONFIG + + +def retrieve_config(schema): + """Called by the actor to retrieve the actor configuration specific to this actor.""" + # TODO: The use of _ACTOR_CONFIG isn't good API. Since this function is + # called by the Actors, we *know* that this is okay to do (as the + # configuration will have already been loaded.) However, there's nothing in + # the API that ensures that this is the case. Need to redesign this. + # Can't think of how it should look right now because loading requires + # information that the Actor doesn't know. + + configuration = defaultdict(dict) + for field in schema: + configuration[field.section][field.name] = _ACTOR_CONFIG[field.section][field.name] + + return dict(configuration) + +# +# The function after this needs some work to be ready. It isn't part of the +# upgrade or preupgrade workflows so we don't have to get it finished yet. +# + + +def format_config(): + """ + Read the configuration definitions from all of the known repositories and return a string that + can be used as an example config file. + + Example config file: + transaction: + to_install_description__: | + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + to_remove_description__: | + List of packages to be removed from the upgrade transaction + initial-setup should be removed to avoid it asking for EULA acceptance during upgrade + to_remove: + - initial-setup + to_keep_description__: | + List of packages to be kept in the upgrade transaction + to_keep: + - leapp + - python2-leapp + - python3-leapp + - leapp-repository + - snactor + """ + # TODO: This is just a placeholder. We need to do some additional + # formatting that includes the documentation, not just return it as is. + return yaml.dump(_ACTOR_CONFIG, SafeDumper) diff --git a/leapp/configs/__init__.py b/leapp/configs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/leapp/configs/actor/__init__.py b/leapp/configs/actor/__init__.py new file mode 100644 index 000000000..c48e433dc --- /dev/null +++ b/leapp/configs/actor/__init__.py @@ -0,0 +1,13 @@ +""" +:py:mod:`leapp.configs.actor` represents the import location for private actor config schema that +are placed in the actor's configs folder. + + +Example: + If your actor has a configs folder with a schemas.py python module, import it + from the actor like this:: + + from leapp.libraries.actor import schemas + +This directory is intended for the definitions of actor configuration fields. +""" diff --git a/leapp/configs/common/__init__.py b/leapp/configs/common/__init__.py new file mode 100644 index 000000000..30c0f0c9d --- /dev/null +++ b/leapp/configs/common/__init__.py @@ -0,0 +1,7 @@ +""" +:py:mod:`leapp.configs.common` represents an import location for shared libraries that +are placed in the repository's configs folder. +""" + +LEAPP_BUILTIN_COMMON_CONFIGS_INITIALIZED = False +""" Framework internal - were common configs initialized? """ diff --git a/leapp/libraries/stdlib/api.py b/leapp/libraries/stdlib/api.py index c49a03d52..267468b66 100644 --- a/leapp/libraries/stdlib/api.py +++ b/leapp/libraries/stdlib/api.py @@ -228,3 +228,10 @@ def get_actor_tool_path(name): :rtype: str or None """ return current_actor().get_actor_tool_path(name) + + +def retrieve_config(): + """ + Retrieve the configuration specific to the specified schema. + """ + return current_actor().retrieve_config() diff --git a/leapp/models/fields/__init__.py b/leapp/models/fields/__init__.py index 5e24be44d..8e05ef423 100644 --- a/leapp/models/fields/__init__.py +++ b/leapp/models/fields/__init__.py @@ -2,6 +2,12 @@ import copy import datetime import json +try: + # Python 3 + from collections.abc import Sequence +except ImportError: + # Python 2.7 + from collections import Sequence import six @@ -147,7 +153,7 @@ def to_builtin(self, source, name, target): def serialize(self): """ - :return: Serialized form of the workflow + :return: Serialized form of the field """ return { 'nullable': self._nullable, @@ -185,11 +191,13 @@ def _validate_builtin_value(self, value, name): self._validate(value=value, name=name, expected_type=self._builtin_type) def _validate(self, value, name, expected_type): - if not isinstance(expected_type, tuple): + if not isinstance(expected_type, Sequence): expected_type = (expected_type,) + if value is None and self._nullable: return - if not any(isinstance(value, t) for t in expected_type): + + if not isinstance(value, expected_type): names = ', '.join(['{}'.format(t.__name__) for t in expected_type]) raise ModelViolationError("Fields {} is of type: {} expected: {}".format(name, type(value).__name__, names)) @@ -457,6 +465,70 @@ def serialize(self): return result +class StringMap(Field): + """ + Map from strings to instances of a given value type. + """ + def __init__(self, value_type, **kwargs): + super(StringMap, self).__init__(**kwargs) + + if self._default is not None: + self._default = copy.copy(self._default) + + if not isinstance(value_type, Field): + raise ModelMisuseError("elem_field must be an instance of a type derived from Field") + + self._value_type = value_type + + def _validate_model_value_using_validator(self, new_map, name, validation_method): + list_validator_fn = getattr(super(StringMap, self), validation_method) + list_validator_fn(new_map, name) + + if isinstance(new_map, dict): + for key in new_map: + # Check that the key is trully a string + if not isinstance(key, str): + err = 'Expected a key of type `str`, but got a key `{}` of type `{}`' + raise ModelViolationError(err.format(key, type(key).__name__)) + + value = new_map[key] # avoid using .items(), as it creates a list of all items (slow) in py2 + + # _value_type's validation will check whether the value has a correct type + value_validator_fn = getattr(self._value_type, validation_method) + value_validator_fn(value, name='{}[{}]'.format(name, key)) + elif value is not None: + raise ModelViolationError('Expected a dict but got {} for the {} field'.format(type(value).__name__, name)) + + def _validate_model_value(self, value, name): + self._validate_model_value_using_validator(value, name, '_validate_model_value') + + def _validate_builtin_value(self, value, name): + self._validate_model_value_using_validator(value, name, '_validate_builtin_value') + + def _convert_to_model(self, value, name): + self._validate_builtin_value(value=value, name=name) + + if value is None: + return value + + converter = self._value_type._convert_to_model + return {key: converter(value[key], name='{0}["{1}"]'.format(name, key)) for key in value} + + def _convert_from_model(self, value, name): + self._validate_model_value(value=value, name=name) + + if value is None: + return value + + converter = self._value_type._convert_from_model + return {key: converter(value[key], name='{0}["{1}"]'.format(name, key)) for key in value} + + def serialize(self): + result = super(StringMap, self).serialize() + result['value_type'] = self._value_type.serialize() + return result + + class Model(Field): """ Model is used to use other Models as fields diff --git a/leapp/repository/__init__.py b/leapp/repository/__init__.py index 58b6d1648..8b7aac480 100644 --- a/leapp/repository/__init__.py +++ b/leapp/repository/__init__.py @@ -7,6 +7,7 @@ from leapp.exceptions import RepoItemPathDoesNotExistError, UnsupportedDefinitionKindError from leapp.models import get_models, resolve_model_references import leapp.libraries.common # noqa # pylint: disable=unused-import +import leapp.configs.common # noqa # pylint: disable=unused-import from leapp.repository.actor_definition import ActorDefinition from leapp.repository.definition import DefinitionKind from leapp.tags import get_tags @@ -144,8 +145,10 @@ def load(self, resolve=True, stage=None, skip_actors_discovery=False): self.log.debug("Extending LEAPP_COMMON_FILES for common file paths") self._extend_environ_paths('LEAPP_COMMON_FILES', self.files) self.log.debug("Installing repository provided common libraries loader hook") + sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.libraries.common', paths=self.libraries)) sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.workflows.api', paths=self.apis)) + sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.configs.common', paths=self.configs)) if not skip_actors_discovery: if not stage or stage is _LoadStage.ACTORS: @@ -189,6 +192,15 @@ def mapped_actor_data(data): }) return data + # Note: `configs` are not present here because we are not yet making + # them globally accessible. This is to force people to copy the config + # schema to their Actors instead of importing them from other Actors. + # That copy, in turn, is a good idea so the framework can return an + # error if two Actors share the same config but they have different + # schemafor it (for instance, if Actor Foo and Bar were sharing the + # same config entry but Actor Foo updated the entry in a later version. + # We need to error so Actor Bar can either be ported to the new + # definition or use a different config entry for it's needs.) return { 'repo_dir': self._repo_dir, 'actors': [mapped_actor_data(a.serialize()) for a in self.actors], @@ -199,7 +211,8 @@ def mapped_actor_data(data): 'workflows': filtered_serialization(get_workflows, self.workflows), 'tools': [dict([('path', path)]) for path in self.relative_paths(self.tools)], 'files': [dict([('path', path)]) for path in self.relative_paths(self.files)], - 'libraries': [dict([('path', path)]) for path in self.relative_paths(self.libraries)] + 'libraries': [dict([('path', path)]) for path in self.relative_paths(self.libraries)], + 'configs': [dict([('path', path)]) for path in self.relative_paths(self.configs)], } def _extend_environ_paths(self, name, paths): @@ -268,6 +281,13 @@ def libraries(self): """ return tuple(self._definitions.get(DefinitionKind.LIBRARIES, ())) + @property + def configs(self): + """ + :return: Tuple of configs in the repository + """ + return tuple(self._definitions.get(DefinitionKind.CONFIGS, ())) + @property def files(self): """ diff --git a/leapp/repository/actor_definition.py b/leapp/repository/actor_definition.py index 0e9cc0a3f..d43342104 100644 --- a/leapp/repository/actor_definition.py +++ b/leapp/repository/actor_definition.py @@ -175,6 +175,7 @@ def serialize(self): 'class_name': self.class_name, 'description': self.description, 'tags': self.tags, + 'config_schemas': self.config_schemas, 'consumes': self.consumes, 'produces': self.produces, 'apis': self.apis, @@ -182,6 +183,7 @@ def serialize(self): 'tools': self.tools, 'files': self.files, 'libraries': self.libraries, + 'configs': self.configs, 'tests': self.tests } @@ -212,7 +214,10 @@ def discover(self): if p.exitcode != 0: self.log.error("Process inspecting actor in %s failed with %d", self.directory, p.exitcode) raise ActorInspectionFailedError('Inspection of actor in {path} failed'.format(path=self.directory)) - result = q.get() + + with self.injected_context(): + result = q.get() + if not result: self.log.error("Process inspecting actor in %s returned no result", self.directory) raise ActorInspectionFailedError( @@ -279,6 +284,13 @@ def description(self): """ return self.discover()['description'] + @property + def config_schemas(self): + """ + :return: Actor config_schemas + """ + return self.discover()['config_schemas'] + @contextlib.contextmanager def injected_context(self): """ @@ -301,11 +313,18 @@ def injected_context(self): if self.tools: os.environ['LEAPP_TOOLS'] = os.path.join(self._repo_dir, self._directory, self.tools[0]) + meta_path_backup = sys.meta_path[:] + sys.meta_path.append( LeappLibrariesFinder( module_prefix='leapp.libraries.actor', paths=[os.path.join(self._repo_dir, self.directory, x) for x in self.libraries])) + sys.meta_path.append( + LeappLibrariesFinder( + module_prefix='leapp.configs.actor', + paths=[os.path.join(self._repo_dir, self.directory, x) for x in self.configs])) + previous_path = os.getcwd() os.chdir(os.path.join(self._repo_dir, self._directory)) try: @@ -326,6 +345,8 @@ def injected_context(self): else: os.environ.pop('LEAPP_TOOLS', None) + sys.meta_path = meta_path_backup + @property def apis(self): """ @@ -361,6 +382,13 @@ def files(self): """ return tuple(self._definitions.get(DefinitionKind.FILES, ())) + @property + def configs(self): + """ + :return: Tuple with path to the configs folder of the actor, empty tuple if none + """ + return tuple(self._definitions.get(DefinitionKind.CONFIGS, ())) + @property def tests(self): """ diff --git a/leapp/repository/definition.py b/leapp/repository/definition.py index e853a6a5f..344ec77ec 100644 --- a/leapp/repository/definition.py +++ b/leapp/repository/definition.py @@ -14,8 +14,9 @@ def __init__(self, kind): LIBRARIES = _Kind('libraries') TOOLS = _Kind('tools') FILES = _Kind('files') + CONFIGS = _Kind('configs') TESTS = _Kind('tests') API = _Kind('api') - REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES) - ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, TESTS) + REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES, CONFIGS) + ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, CONFIGS, TESTS) diff --git a/leapp/repository/scan.py b/leapp/repository/scan.py index 378940a4b..94e0512f9 100644 --- a/leapp/repository/scan.py +++ b/leapp/repository/scan.py @@ -89,6 +89,7 @@ def scan(repository, path): ('workflows', scan_workflows), ('files', scan_files), ('libraries', scan_libraries), + ('configs', scan_configs), ('tests', scan_tests), ('tools', scan_tools), ('apis', scan_apis)) @@ -224,6 +225,21 @@ def scan_libraries(repo, path, repo_path): repo.add(DefinitionKind.LIBRARIES, os.path.relpath(path, repo_path)) +def scan_configs(repo, path, repo_path): + """ + Scans configs and adds them to the repository. + + :param repo: Instance of the repository + :type repo: :py:class:`leapp.repository.Repository` + :param path: path to the configs + :type path: str + :param repo_path: path to the repository + :type repo_path: str + """ + if os.listdir(path): + repo.add(DefinitionKind.CONFIGS, os.path.relpath(path, repo_path)) + + def scan_tools(repo, path, repo_path): """ Scans tools and adds them to the repository. diff --git a/leapp/utils/audit/__init__.py b/leapp/utils/audit/__init__.py index 16db10796..2af4f1b38 100644 --- a/leapp/utils/audit/__init__.py +++ b/leapp/utils/audit/__init__.py @@ -637,7 +637,7 @@ def store_workflow_metadata(workflow): ent.store() -def store_actor_metadata(actor_definition, phase): +def store_actor_metadata(actor_definition, phase, config): """ Store the metadata of the given actor given as an ``actor_definition`` object into the database. @@ -651,10 +651,15 @@ def store_actor_metadata(actor_definition, phase): 'consumes': sorted(model.__name__ for model in _metadata.get('consumes', ())), 'produces': sorted(model.__name__ for model in _metadata.get('produces', ())), 'tags': sorted(tag.__name__ for tag in _metadata.get('tags', ())), + 'config_schemas': [field.serialize() for field in _metadata.get('config_schemas', ())], }) _metadata['phase'] = phase - actor_metadata_fields = ('class_name', 'name', 'description', 'phase', 'tags', 'consumes', 'produces', 'path') + # TODO(dkubek): Is configuration metadata? Does it make sense to include it here? + _metadata['config'] = config + + actor_metadata_fields = ('class_name', 'name', 'description', 'phase', + 'tags', 'consumes', 'produces', 'config_schemas', 'config', 'path') metadata = json.dumps({field: _metadata[field] for field in actor_metadata_fields}, sort_keys=True) metadata_hash_id = hashlib.sha256(metadata.encode('utf-8')).hexdigest() diff --git a/leapp/workflows/__init__.py b/leapp/workflows/__init__.py index 1b6fc9804..27fac7855 100644 --- a/leapp/workflows/__init__.py +++ b/leapp/workflows/__init__.py @@ -4,6 +4,7 @@ import sys import uuid +from leapp.actors.config import retrieve_config from leapp.dialogs import RawMessageDialog from leapp.exceptions import CommandError, MultipleConfigActorsError, WorkflowConfigNotAvailable from leapp.messaging.answerstore import AnswerStore @@ -301,7 +302,8 @@ def run(self, context=None, until_phase=None, until_actor=None, skip_phases_unti for phase in self._phase_actors: for stage in phase[1:]: for actor in stage.actors: - store_actor_metadata(actor, phase[0].name) + actor_config = retrieve_config(actor.config_schemas) + store_actor_metadata(actor, phase[0].name, actor_config) self._stop_after_phase_requested = False for phase in self._phase_actors: diff --git a/packaging/leapp.spec b/packaging/leapp.spec index ab38d207d..29ece5341 100644 --- a/packaging/leapp.spec +++ b/packaging/leapp.spec @@ -134,6 +134,7 @@ Provides: leapp-framework-dependencies = %{framework_dependencies} Requires: python-six Requires: python-setuptools Requires: python-requests +Requires: PyYAML %else # <> rhel 7 # for Fedora & RHEL 8+ deliver just python3 stuff # NOTE: requirement on python3 refers to the general version of Python @@ -144,6 +145,7 @@ Requires: python3 Requires: python3-six Requires: python3-setuptools Requires: python3-requests +Requires: python3-PyYAML %endif Requires: findutils ################################################## diff --git a/requirements.txt b/requirements.txt index ec831ad60..2951f23de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ six==1.12 requests +PyYAML diff --git a/res/container-tests/Containerfile.ubi10 b/res/container-tests/Containerfile.ubi10 index 6d6507cef..1d15d2995 100644 --- a/res/container-tests/Containerfile.ubi10 +++ b/res/container-tests/Containerfile.ubi10 @@ -20,19 +20,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ pip install -U pytest && \ pip install -U six && \ pip install -U . && \ - export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running pylint ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running flake8 ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - flake8 $LINTABLES && echo '===> flake8 PASSED' && \ echo '==================================================' && \ echo '==================================================' && \ echo '=============== Running tests ===============' && \ diff --git a/res/container-tests/Containerfile.ubi10-lint b/res/container-tests/Containerfile.ubi10-lint new file mode 100644 index 000000000..a32e44028 --- /dev/null +++ b/res/container-tests/Containerfile.ubi10-lint @@ -0,0 +1,35 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +VOLUME /payload + +RUN dnf update -y && \ + dnf install python3 python312 python3-pip make -y && \ + python3 -m pip install --upgrade pip==20.3.4 + +RUN pip install virtualenv + +WORKDIR /payload +ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ + source testenv/bin/activate && \ + pip install -U setuptools && \ + pip install -U funcsigs && \ + pip install -U -r requirements-tests.txt && \ + # NOTE(mmatuska): The pytest ver defined in requirements-tests is too old \ + # for Python 3.12 (missing imp module), there do an update here until we \ + # bump that. Similarly for six. \ + pip install -U pytest && \ + pip install -U six && \ + pip install -U . && \ + export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running pylint ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running flake8 ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + flake8 $LINTABLES && echo '===> flake8 PASSED' diff --git a/res/container-tests/Containerfile.ubi7 b/res/container-tests/Containerfile.ubi7 index 9e362801b..fd5b965c5 100644 --- a/res/container-tests/Containerfile.ubi7 +++ b/res/container-tests/Containerfile.ubi7 @@ -13,19 +13,6 @@ ENTRYPOINT virtualenv testenv && \ pip install -U funcsigs && \ pip install -U -r requirements-tests.txt && \ pip install -U . && \ - export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running pylint ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo $LINTABLES | xargs pylint --py3k && echo '===> pylint PASSED' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running flake8 ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - flake8 $LINTABLES && echo '===> flake8 PASSED' && \ echo '==================================================' && \ echo '==================================================' && \ echo '=============== Running tests ===============' && \ diff --git a/res/container-tests/Containerfile.ubi7-lint b/res/container-tests/Containerfile.ubi7-lint new file mode 100644 index 000000000..2e1de4a63 --- /dev/null +++ b/res/container-tests/Containerfile.ubi7-lint @@ -0,0 +1,28 @@ +FROM registry.access.redhat.com/ubi7/ubi:7.9 + +VOLUME /payload + +RUN yum -y install python27-python-pip && \ + scl enable python27 -- pip install -U --target /usr/lib/python2.7/site-packages/ pip==20.3.0 && \ + python -m pip install --ignore-installed pip==20.3.4 virtualenv + +WORKDIR /payload +ENTRYPOINT virtualenv testenv && \ + source testenv/bin/activate && \ + pip install -U setuptools && \ + pip install -U funcsigs && \ + pip install -U -r requirements-tests.txt && \ + pip install -U . && \ + export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running pylint ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo $LINTABLES | xargs pylint --py3k && echo '===> pylint PASSED' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running flake8 ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + flake8 $LINTABLES && echo '===> flake8 PASSED' diff --git a/res/container-tests/Containerfile.ubi8 b/res/container-tests/Containerfile.ubi8 index 03d499dac..960f4b09b 100644 --- a/res/container-tests/Containerfile.ubi8 +++ b/res/container-tests/Containerfile.ubi8 @@ -16,19 +16,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ pip install -U funcsigs && \ pip install -U -r requirements-tests.txt && \ pip install -U . && \ - export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running pylint ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running flake8 ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - flake8 $LINTABLES && echo '===> flake8 PASSED' && \ echo '==================================================' && \ echo '==================================================' && \ echo '=============== Running tests ===============' && \ diff --git a/res/container-tests/Containerfile.ubi8-lint b/res/container-tests/Containerfile.ubi8-lint new file mode 100644 index 000000000..75143d91a --- /dev/null +++ b/res/container-tests/Containerfile.ubi8-lint @@ -0,0 +1,31 @@ +FROM registry.access.redhat.com/ubi8/ubi:latest + +VOLUME /payload + +ENV PYTHON_VENV "python3.6" + +RUN yum update -y && \ + yum install python3 python39 python3-virtualenv make -y && \ + yum -y install python3-pip && \ + python3 -m pip install --upgrade pip==20.3.4 + +WORKDIR /payload +ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ + source testenv/bin/activate && \ + pip install -U setuptools && \ + pip install -U funcsigs && \ + pip install -U -r requirements-tests.txt && \ + pip install -U . && \ + export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running pylint ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running flake8 ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + flake8 $LINTABLES && echo '===> flake8 PASSED' diff --git a/res/container-tests/Containerfile.ubi9 b/res/container-tests/Containerfile.ubi9 index b1b5b71bd..755e874e8 100644 --- a/res/container-tests/Containerfile.ubi9 +++ b/res/container-tests/Containerfile.ubi9 @@ -15,19 +15,6 @@ ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ pip install -U funcsigs && \ pip install -U -r requirements-tests.txt && \ pip install -U . && \ - export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running pylint ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ - echo '==================================================' && \ - echo '==================================================' && \ - echo '=============== Running flake8 ===============' && \ - echo '==================================================' && \ - echo '==================================================' && \ - flake8 $LINTABLES && echo '===> flake8 PASSED' && \ echo '==================================================' && \ echo '==================================================' && \ echo '=============== Running tests ===============' && \ diff --git a/res/container-tests/Containerfile.ubi9-lint b/res/container-tests/Containerfile.ubi9-lint new file mode 100644 index 000000000..9a1d18621 --- /dev/null +++ b/res/container-tests/Containerfile.ubi9-lint @@ -0,0 +1,30 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +VOLUME /payload + +RUN dnf update -y && \ + dnf install python3 python39 make python3-pip -y && \ + python3 -m pip install --upgrade pip==20.3.4 + +RUN pip install virtualenv + +WORKDIR /payload +ENTRYPOINT virtualenv testenv -p "/usr/bin/$PYTHON_VENV" && \ + source testenv/bin/activate && \ + pip install -U setuptools && \ + pip install -U funcsigs && \ + pip install -U -r requirements-tests.txt && \ + pip install -U . && \ + export LINTABLES=$(find . -name '*.py' | grep -E -e '^\./leapp\/' -e '^\./tests/scripts/' | sort -u ) && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running pylint ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo $LINTABLES | xargs pylint && echo '===> pylint PASSED' && \ + echo '==================================================' && \ + echo '==================================================' && \ + echo '=============== Running flake8 ===============' && \ + echo '==================================================' && \ + echo '==================================================' && \ + flake8 $LINTABLES && echo '===> flake8 PASSED' diff --git a/setup.py b/setup.py index 40e76569b..3ebe8b24a 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ name='leapp', version=main_ns['VERSION'], packages=find_packages(exclude=EXCLUSION), - install_requires=['six', 'requests'], + install_requires=['six', 'requests', 'PyYAML'], entry_points=''' [console_scripts] snactor=leapp.snactor:main diff --git a/tests/scripts/test_repository_actor_definition.py b/tests/scripts/test_repository_actor_definition.py index 3fcf75750..8ae698148 100644 --- a/tests/scripts/test_repository_actor_definition.py +++ b/tests/scripts/test_repository_actor_definition.py @@ -13,6 +13,7 @@ 'name': 'fake-actor', 'path': 'actors/test', 'tags': (), + 'config_schemas': (), 'consumes': (), 'produces': (), 'dialogs': (), @@ -40,6 +41,7 @@ def test_actor_definition(repository_dir): assert definition.consumes == _FAKE_META_DATA['consumes'] assert definition.produces == _FAKE_META_DATA['produces'] assert definition.tags == _FAKE_META_DATA['tags'] + assert definition.config_schemas == _FAKE_META_DATA['config_schemas'] assert definition.class_name == _FAKE_META_DATA['class_name'] assert definition.dialogs == _FAKE_META_DATA['dialogs'] assert definition.name == _FAKE_META_DATA['name'] @@ -52,11 +54,13 @@ def test_actor_definition(repository_dir): assert dumped.pop('produces') == definition.produces assert dumped.pop('apis') == definition.apis assert dumped.pop('tags') == definition.tags + assert dumped.pop('config_schemas') == definition.config_schemas assert dumped.pop('dialogs') == [dialog.serialize() for dialog in definition.dialogs] assert dumped.pop('path') == _FAKE_META_DATA['path'] assert dumped.pop('name') == definition.name assert dumped.pop('files') == ('.',) assert dumped.pop('libraries') == ('.',) + assert dumped.pop('configs') == ('.',) assert dumped.pop('tests') == ('.',) assert dumped.pop('tools') == ('.',) # Assert to ensure we covered all keys