From 133e73c29289206e3ec66d71387ca6ffc0d6a796 Mon Sep 17 00:00:00 2001 From: JulienBortolussiAda Date: Tue, 19 Sep 2023 15:00:02 +0200 Subject: [PATCH] Add support for key_set qualifier key_set qualifiers expect semi-colon separated list as value. --- src/e3/anod/qualifiers_manager.py | 246 +++++++++++++++--- src/e3/anod/spec.py | 4 +- tests/tests_e3/anod/spec_test.py | 20 ++ tests/tests_e3/anod/test_qualifier_manager.py | 246 +++++++++++++++++- 4 files changed, 484 insertions(+), 32 deletions(-) diff --git a/src/e3/anod/qualifiers_manager.py b/src/e3/anod/qualifiers_manager.py index 80ab3685..ef0e6d6c 100644 --- a/src/e3/anod/qualifiers_manager.py +++ b/src/e3/anod/qualifiers_manager.py @@ -68,7 +68,7 @@ def __init__( ) @property - def default(self) -> str | bool | None: + def default(self) -> str | bool | frozenset[str] | None: """Return default value for qualifier. :return: if None is returned it means the qualifier has not @@ -77,11 +77,13 @@ def default(self) -> str | bool | None: return None # all: no cover @abc.abstractmethod - def value(self, value: str | bool) -> str | bool: - """Validate qualifier value and return the effective one.""" + def value(self, value: str) -> str | bool | frozenset[str]: + """Compute the value of qualifier given the use input.""" @abc.abstractmethod - def repr(self, value: str | bool, hash_pool: list[str] | None) -> str: + def repr( + self, value: str | bool | frozenset[str], hash_pool: list[str] | None + ) -> str: """Compute a string representation of a qualifier. :param value: the effective value associated with the qualifier @@ -140,12 +142,14 @@ def __init__( self._default = default @property - def default(self) -> str | bool | None: + def default(self) -> str | bool | frozenset[str] | None: """See QualifierDeclaration.default.""" return self._default - def value(self, value: str | bool) -> str | bool: + def value(self, value: str) -> str | bool | frozenset[str]: """See QualifierDeclaration.value.""" + if not isinstance(value, str): + raise AnodError("Key value qualifiers can only parse a string value.") if self.choices is not None and value not in self.choices: choices_str = ", ".join((f"'{choice}'" for choice in self.choices)) raise AnodError( @@ -154,10 +158,13 @@ def value(self, value: str | bool) -> str | bool: ) return value - def repr(self, value: str | bool, hash_pool: list[str] | None) -> str: + def repr( + self, value: str | bool | frozenset[str], hash_pool: list[str] | None + ) -> str: """See QualifierDeclaration.repr.""" if not value: - # An empty value for tag value should lead to an empty representation + # An empty value for a key_value qualifier should lead to an empty + # representation str_repr = "" elif ( value == self.default @@ -173,8 +180,9 @@ def repr(self, value: str | bool, hash_pool: list[str] | None) -> str: list_repr = [] if not self.repr_omit_key: list_repr.append(self.repr_name) - if value: - list_repr.append(str(value)) + + # value as been checked early on + list_repr.append(str(value)) # And join them with a dash. str_repr = "-".join(list_repr) @@ -191,23 +199,18 @@ class TagDeclaration(QualifierDeclaration): """Tag qualifier declaration.""" @property - def default(self) -> bool: + def default(self) -> str | bool | frozenset[str] | None: """See QualifierDeclaration.value.""" return False - def value(self, value: str | bool) -> bool: + def value(self, value: str) -> str | bool | frozenset[str]: """See QualifierDeclaration.value.""" - # The presence of a tag in input is defined by the existence - # of the key (the value has no meaning). The value set by the - # user might be None or "". The False value can only come from - # the defaults initialization. Thus do not change following - # condition into if not value. - if value is False: - return False - else: - return True + # As soon as a tag qualifier is passed, its value is True + return True - def repr(self, value: str | bool, hash_pool: list[str] | None) -> str: + def repr( + self, value: str | bool | frozenset[str], hash_pool: list[str] | None + ) -> str: """See QualifierDeclaration.repr.""" if hash_pool is not None and self.repr_in_hash: if value: @@ -219,6 +222,118 @@ def repr(self, value: str | bool, hash_pool: list[str] | None) -> str: return "" +class KeySetDeclaration(QualifierDeclaration): + # The separator use to distinguish all the element of the list + LIST_SEPARATOR = ";" + + def __init__( + self, + origin: str, + name: str, + description: str, + repr_in_hash: bool = False, + repr_name: str | None = None, + repr_omit_key: bool = False, + default: set[str] | None = None, + choices: list[str] | None = None, + ) -> None: + """Initialize a key-set qualifier declaration. + + :param origin: a string giving the origin of the declaration + (used in error messages) + :param name: see QualifierDeclaration.__init__ + :param description: see QualifierDeclaration.__init__ + :param repr_in_hash: see QualifierDeclaration.__init__ + :param repr_name: see QualifierDeclaration.__init__ + :param repr_omit_key: if True discard qualifier name in string representation + :param default: default value for the qualifier + :param choices: list of valid value for the qualifier + """ + super().__init__( + origin=origin, + name=name, + description=description, + repr_in_hash=repr_in_hash, + repr_name=repr_name, + ) + self.repr_omit_key = repr_omit_key + + # Check if the default is valid + if default is not None and choices is not None: + wrong_values = default - set(choices) + if wrong_values: + choices_str = ", ".join((f"'{choice}'" for choice in choices)) + wrong_values_str = ", ".join( + (f"'{value}'" for value in sorted(wrong_values)) + ) + raise AnodError( + f"{self.origin}: In '{self.name}', default value(s) " + f"({wrong_values_str}) should be in ({choices_str})" + ) + + self.choices = choices + self._default: frozenset[str] | None = ( + frozenset(default) if default is not None else None + ) + + @property + def default(self) -> str | bool | frozenset[str] | None: + """See QualifierDeclaration.default.""" + return self._default + + def value(self, value: str) -> str | bool | frozenset[str]: + """See QualifierDeclaration.value.""" + if not isinstance(value, str): + raise AnodError("Key set qualifiers can only parse a string value.") + + # Make sure '' value is the empty set + value_set = ( + frozenset(value.split(self.LIST_SEPARATOR)) if value else frozenset({}) + ) + + # Check if the values are in choices + if self.choices: + wrong_values = value_set - set(self.choices) + + if wrong_values: + choices_str = ", ".join((f"'{choice}'" for choice in self.choices)) + wrong_values_str = ", ".join( + (f"'{value}'" for value in sorted(wrong_values)) + ) + raise AnodError( + f"{self.origin}: Invalid value(s) for qualifier {self.name}: " + f"({wrong_values_str}) not in ({choices_str})" + ) + + return value_set + + def repr( + self, value: str | bool | frozenset[str], hash_pool: list[str] | None + ) -> str: + """See QualifierDeclaration.repr.""" + assert isinstance(value, frozenset) + if not value: + # An empty value for key_set qualifier should lead to an empty + # representation + str_repr = "" + else: + # Otherwise compute components of the representation. + list_repr = [] + if not self.repr_omit_key: + list_repr.append(self.repr_name) + list_repr.extend((str(v) for v in sorted(value))) + + # And join them with a dash. + str_repr = "-".join(list_repr) + + if hash_pool is not None and self.repr_in_hash: + if str_repr: + hash_pool.append(str_repr) + return "" + else: + return str_repr + + class QualifiersManager: """Parse the qualifiers and build an unique name. @@ -275,11 +390,15 @@ def __init__( # Hold the declared components. The keys are the qualifier configurations # (tuples) and the value are the component names. # It is construct by end_declaration_phase using raw_component_decls. - self.component_names: dict[tuple[tuple[str, str | bool], ...], str] = {} - self.build_space_names: dict[tuple[tuple[str, str | bool], ...], str] = {} + self.component_names: dict[ + tuple[tuple[str, str | bool | frozenset[str]], ...], str + ] = {} + self.build_space_names: dict[ + tuple[tuple[str, str | bool | frozenset[str]], ...], str + ] = {} # Hold the final qualifier values for anod_instance. - self.qualifier_values: dict[str, str | bool] = {} + self.qualifier_values: dict[str, str | bool | frozenset[str]] = {} # When the first name has been generated it is no longer possible to add # neither new qualifiers nor new components. @@ -436,6 +555,75 @@ def declare_key_value_qualifier( repr_omit_key=repr_omit_key, ) + def declare_key_set_qualifier( + self, + name: str, + description: str, + test_only: bool = False, + default: set[str] | None = None, + choices: list[str] | None = None, + repr_alias: str | None = None, + repr_in_hash: bool = False, + repr_omit_key: bool = False, + ) -> None: + """Declare a new key set qualifier. + + Declare a key set qualifier to allow it use in the spec. It will have an + impact on the build_space and component names. + + A key set qualifier is a 'list' qualifier. They require the user to + provide their values as a semi-colon separated list. + + This method cannot be called after the end of the declaration phase. + + :param name: The name of the qualifier. It used to identify it and pass it to + the spec. + :param description: A description of the qualifier purposes. It is used to + make the help/error clearer. + :param test_only: By default the qualifier are used by all anod actions + (install, build, test...). If test_only is True, then this qualifier is + only available for test. + :param default: The default value given to the qualifier if no value was + provided by the user. If no default value is set, then the user must + provide a qualifier value at runtime. + :param choices: The list of all authorized values for the qualifier. + :param repr_alias: An alias for the qualifier name used by the name generation. + By default, the repr_alias is the qualifier name itself. + :param repr_in_hash: False by default. If True, the qualifier is included in + the hash at the end of the generated name. The result is less readable but + shorter. + :param repr_omit_key: If True, then the name generation don't display the + qualifier name/alias. It only use its value. + """ + if self.is_declaration_phase_finished: + raise AnodError( + f"{self.origin}: qualifier can only be declared in " + " declare_qualifiers_and_components" + ) + + # Make sure {} is read as the empty set + if default == {}: + default = set({}) + + # Make sure the default is None or a set as key_set qualifier are not used the + # same way as the more standard key_value qualifier + if default is not None and not isinstance(default, set): + raise AnodError( + "The default of key_set qualifier must be either None or a set" + ) + + if not test_only or self.anod_instance.kind == "test": + self.qualifier_decls[name] = KeySetDeclaration( + origin=self.origin, + name=name, + description=description, + repr_name=repr_alias, + repr_in_hash=repr_in_hash, + default=default, + choices=choices, + repr_omit_key=repr_omit_key, + ) + def declare_component( self, name: str, @@ -499,7 +687,7 @@ def declare_build_space_name( def compute_qualifier_values( self, qualifier_dict: dict[str, str], - ) -> dict[str, str | bool]: + ) -> dict[str, str | bool | frozenset[str]]: """Given a user qualifier dict compute and validate final values. :param qualifier_dict: User qualifiers @@ -540,8 +728,8 @@ def compute_qualifier_values( return result def serialize_qualifier_values( - self, qualifier_values: dict[str, str | bool] - ) -> tuple[tuple[str, str | bool], ...]: + self, qualifier_values: dict[str, str | bool | frozenset[str]] + ) -> tuple[tuple[str, str | bool | frozenset[str]], ...]: """Return a hashable and deterministic representation of qualifier values. :param qualifier_values: qualifier values as returned by @@ -664,7 +852,7 @@ def compute_build_space_name(self) -> None: self.build_space_name = "_".join([el for el in bs if el]) - def __getitem__(self, key: str) -> str | bool: + def __getitem__(self, key: str) -> str | bool | frozenset[str]: """Return the parsed value of the requested qualifier. :return: The qualifier value after the parsing. diff --git a/src/e3/anod/spec.py b/src/e3/anod/spec.py index 54241d21..c8d8a2cd 100644 --- a/src/e3/anod/spec.py +++ b/src/e3/anod/spec.py @@ -293,7 +293,7 @@ def declare_qualifiers_and_components( pass @property - def args(self) -> dict[str, str | bool]: + def args(self) -> dict[str, str | bool | frozenset[str]]: """Access to final qualifier values (with defaults set).""" if self.enable_name_generator: return self.qualifiers_manager.qualifier_values @@ -412,7 +412,7 @@ def __getitem__(self, key: str) -> Any: elif key.isupper(): return getattr(self.build_space, key.lower(), None) - def get_qualifier(self, qualifier_name: str) -> str | bool | None: + def get_qualifier(self, qualifier_name: str) -> str | bool | frozenset[str] | None: """Return a qualifier value. Requires that qualifiers_manager attribute has been initialized and its parse diff --git a/tests/tests_e3/anod/spec_test.py b/tests/tests_e3/anod/spec_test.py index 67ce6f21..b4772b0b 100644 --- a/tests/tests_e3/anod/spec_test.py +++ b/tests/tests_e3/anod/spec_test.py @@ -127,3 +127,23 @@ def test_api_version(): with pytest.raises(AnodError): check_api_version("0.0") + + +def test_spec_qualifier(): + class GeneratorEnabled(Anod): + enable_name_generator = True + + def declare_qualifiers_and_components(self, qm): + qm.declare_tag_qualifier( + "q1", + description="???", + ) + + class GeneratorDisabled(Anod): + enable_name_generator = False + + spec_enable = GeneratorEnabled(qualifier="q1", kind="build") + spec_disable = GeneratorDisabled(qualifier="q1", kind="build") + + assert spec_enable.args == {"q1": True} + assert spec_disable.args == {"q1": ""} diff --git a/tests/tests_e3/anod/test_qualifier_manager.py b/tests/tests_e3/anod/test_qualifier_manager.py index 26a09643..d56bb82c 100644 --- a/tests/tests_e3/anod/test_qualifier_manager.py +++ b/tests/tests_e3/anod/test_qualifier_manager.py @@ -1,5 +1,9 @@ from e3.anod.error import AnodError -from e3.anod.qualifiers_manager import QualifiersManager +from e3.anod.qualifiers_manager import ( + QualifiersManager, + KeyValueDeclaration, + KeySetDeclaration, +) from e3.anod.spec import Anod from e3.env import BaseEnv @@ -256,6 +260,31 @@ def declare_qualifiers_and_components(self, qm): "{'test_qual1': ''} for build space/component comp1" ) + # Call value with something else that a str + qualifiers_manager = QualifiersManager(Anod("", kind="build")) + with pytest.raises(AnodError) as err: + qualifiers_manager.declare_key_set_qualifier( + name="q1", + description="???", + default="v1", + ) + assert ( + str(err.value) + == "The default of key_set qualifier must be either None or a set" + ) + + # Call value with something else that a str + key_set = KeySetDeclaration(origin="origin", name="q1", description="???") + key_value = KeyValueDeclaration(origin="origin", name="q1", description="???") + + with pytest.raises(AnodError) as err: + key_set.value(1) + assert str(err.value) == "Key set qualifiers can only parse a string value." + + with pytest.raises(AnodError) as err: + key_value.value(1) + assert str(err.value) == "Key value qualifiers can only parse a string value." + def test_qualifiers_manager(): class Simple(Anod): @@ -467,3 +496,218 @@ def declare_qualifiers_and_components(self, qualifiers_manager): anod = GrandChildAnod(kind="build", qualifier="", env=env) assert anod.build_space_name == "grandchild" + + # Change add_target_info after parse + with pytest.raises(AnodError) as err: + anod.qualifiers_manager.add_target_info() + assert str(err.value) == ( + "build(name=grandchild, qual={}): build space name computation settings can " + "only be changed in 'declare_qualifiers_and_components'" + ) + + with pytest.raises(AnodError) as err: + anod.qualifiers_manager.remove_target_info() + assert str(err.value) == ( + "build(name=grandchild, qual={}): build space name computation settings can " + "only be changed in 'declare_qualifiers_and_components'" + ) + + +def test_key_value_qualifier(): + class Spec(Anod): + name = "spec" + enable_name_generator = True + + def declare_qualifiers_and_components(self, qualifiers_manager): + super().declare_qualifiers_and_components(qualifiers_manager) + + qualifiers_manager.declare_key_value_qualifier( + name="q1", + description="???", + default="default", + choices=["default", "val"], + ) + + spec1 = Spec(qualifier="q1=default", kind="build") + spec2 = Spec(qualifier="q1=val", kind="build") + + assert spec1.build_space_name == "spec" + assert spec2.build_space_name == "spec_q1-val" + + # Declare a key value after the end of the declaration phase + with pytest.raises(AnodError) as err: + spec1.qualifiers_manager.declare_key_value_qualifier( + name="name", + description="???", + ) + assert str(err.value) == ( + "build(name=spec, qual={'q1': 'default'}): qualifier can " + "only be declared in declare_qualifiers_and_components" + ) + + +def test_tag_qualifier(): + class Spec(Anod): + name = "spec" + enable_name_generator = True + + def declare_qualifiers_and_components(self, qualifiers_manager): + super().declare_qualifiers_and_components(qualifiers_manager) + + qualifiers_manager.declare_tag_qualifier( + name="q1", + description="???", + ) + qualifiers_manager.declare_tag_qualifier( + name="q2", + description="???", + repr_in_hash=True, + ) + + spec1 = Spec(qualifier="", kind="build") + spec2 = Spec(qualifier="q2", kind="build") + + assert spec1.build_space_name == "spec" + assert spec2.build_space_name == "spec_f237cb03" + + # Explicitly test value + from e3.anod.qualifiers_manager import TagDeclaration + + tag = TagDeclaration( + origin="origin", + name="tag", + description="???", + ) + + assert tag.value("") is True + assert tag.value(None) is True + + +def test_key_set_qualifier(): + class Spec(Anod): + name = "spec" + enable_name_generator = True + + def declare_qualifiers_and_components(self, qualifiers_manager): + super().declare_qualifiers_and_components(qualifiers_manager) + + qualifiers_manager.declare_key_set_qualifier( + name="q1", + description="???", + ) + + qualifiers_manager.declare_key_set_qualifier( + name="q2", + description="???", + default={"1", "2"}, + choices=["1", "2", "3"], + ) + + qualifiers_manager.declare_key_set_qualifier( + name="q3", + description="???", + repr_omit_key=True, + default={"1"}, + ) + + qualifiers_manager.declare_key_set_qualifier( + name="q4", + description="???", + default={"1"}, + repr_in_hash=True, + choices=["1", "2"], + ) + + qualifiers_manager.declare_key_set_qualifier( + name="q5", + description="???", + default={}, + repr_in_hash=True, + choices=["1", "2"], + ) + + spec1 = Spec(qualifier="q1=a;b,q2=3;2,q4=2", kind="build") + spec2 = Spec(qualifier="q1=a", kind="build") + spec3 = Spec(qualifier="q1=", kind="build") + + assert spec1.build_space_name == "spec_q1-a-b_q2-2-3_1_29c6909d" + assert spec2.build_space_name == "spec_q1-a_q2-1-2_1_733c1a4a" + assert spec3.build_space_name == "spec_q2-1-2_1_733c1a4a" + + # Use wrong value + with pytest.raises(AnodError) as err: + Spec(qualifier="q1=a,q2=1;2;5;3;4", kind="build") + assert str(err.value) == ( + "build(name=spec): Invalid value(s) for qualifier q2: " + "('4', '5') not in ('1', '2', '3')" + ) + + class BadSpec(Spec): + def declare_qualifiers_and_components(self, qualifiers_manager): + super().declare_qualifiers_and_components(qualifiers_manager) + + qualifiers_manager.declare_key_set_qualifier( + "q3", + description="???", + default={"5"}, + choices=["1", "2"], + ) + + with pytest.raises(AnodError) as err: + BadSpec(qualifier="q1=a", kind="build") + assert str(err.value) == ( + "build(name=spec): In 'q3', default value(s) ('5') should be in ('1', '2')" + ) + + # Declare a key value after the end of the declaration phase + with pytest.raises(AnodError) as err: + spec1.qualifiers_manager.declare_key_set_qualifier( + name="name", + description="???", + ) + assert str(err.value) == ( + "build(name=spec, qual={'q1': 'a;b', 'q2': '3;2', 'q4': '2'}): qualifier can " + "only be declared in declare_qualifiers_and_components" + ) + + class SpecTest(Anod): + name = "spec" + enable_name_generator = True + + def declare_qualifiers_and_components(self, qualifiers_manager): + super().declare_qualifiers_and_components(qualifiers_manager) + + qualifiers_manager.declare_key_set_qualifier( + name="q1", + description="???", + test_only=True, + ) + + spec1 = SpecTest(qualifier="", kind="build") + spec2 = SpecTest(qualifier="q1=1;2;3", kind="test") + + assert spec1.build_space_name == "spec" + assert spec2.build_space_name == "spec_q1-1-2-3_test" + + +def test_declare_build_space(): + class Spec(Anod): + name = "spec" + enable_name_generator = True + + def declare_qualifiers_and_components(self, qualifiers_manager): + qualifiers_manager.declare_tag_qualifier( + name="q1", + description="???", + ) + + qualifiers_manager.declare_build_space_name( + "my_spec", + {"q1": ""}, + ) + + spec1 = Spec(qualifier="", kind="build") + spec2 = Spec(qualifier="q1", kind="build") + + assert spec1.build_space_name == "spec" + assert spec2.build_space_name == "my_spec"