From 92b2ebd37807f1e46343e85cb17e6257aba19b61 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Fri, 1 Sep 2023 11:45:52 +0200 Subject: [PATCH 01/15] fix(model-elements)!: Fix duplication of dynamically typed work items There are serializers that change the Capella type (e.g. `SystemComponent` -> `SystemActor`) based on attributes (`is_actor`). If these types are not appearing in the config, then no work items of this type won't be fetched via the REST API and in return they are created as new work items. This commit filters work items with invalid/unconfigured types out and logs these types. --- capella2polarion/elements/element.py | 42 +++++++++++++++++++--------- tests/test_elements.py | 2 ++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 616065ed..9b95df21 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -11,7 +11,12 @@ import polarion_rest_api_client as polarion_api from capellambse.model import common -from capella2polarion.elements import POL2CAPELLA_TYPES, api_helper, serialize +from capella2polarion.elements import ( + POL2CAPELLA_TYPES, + api_helper, + helpers, + serialize, +) logger = logging.getLogger(__name__) @@ -23,22 +28,33 @@ def create_work_items(ctx: dict[str, t.Any]) -> None: """Create a set of work items in Polarion.""" - - def serialize_for_create( - obj: common.GenericElement, - ) -> serialize.CapellaWorkItem | None: - logger.debug( - "Create work item for model element %r...", obj._short_repr_() - ) - return serialize.element(obj, ctx, serialize.generic_work_item) - objects = chain.from_iterable(ctx["ELEMENTS"].values()) - work_items = [ - serialize_for_create(obj) + _work_items = [ + serialize.element(obj, ctx, serialize.generic_work_item) for obj in objects if obj.uuid not in ctx["POLARION_ID_MAP"] ] - work_items = list(filter(None.__ne__, work_items)) + _work_items = list(filter(None.__ne__, _work_items)) + valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) + work_items: list[polarion_api.CapellaWorkItem] = [] + missing_types: set[str] = set() + for work_item in _work_items: + assert work_item is not None + if work_item.type in valid_types: + work_items.append(work_item) + else: + missing_types.add(work_item.type) + logger.debug( + "%r are missing in the capella2polarion configuration", + ", ".join(missing_types), + ) + + for work_item in work_items: + assert work_item is not None + obj = ctx["MODEL"].by_uuid(work_item.uuid_capella) + logger.debug( + "Create work item for model element %r...", obj._short_repr_() + ) if work_items: try: ctx["API"].create_work_items(work_items) diff --git a/tests/test_elements.py b/tests/test_elements.py index cacf4fd7..b0907f7c 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -189,6 +189,8 @@ def context() -> dict[str, t.Any]: def test_create_work_items( monkeypatch: pytest.MonkeyPatch, context: dict[str, t.Any] ): + context["MODEL"] = model = mock.MagicMock() + model.by_uuid.side_effect = context["ELEMENTS"]["FakeModelObject"] monkeypatch.setattr( serialize, "generic_work_item", From b09ce59767ae2846212a8b747d708a52f86e4f92 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Fri, 1 Sep 2023 11:50:25 +0200 Subject: [PATCH 02/15] fix(serializer): Improve logging message for inline image errors --- capella2polarion/elements/serialize.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 34f6491b..b5f5131a 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -115,7 +115,7 @@ def _generic_work_item( ) -> CapellaWorkItem: xtype = ctx["POLARION_TYPE_MAP"].get(obj.uuid, type(obj).__name__) raw_description = getattr(obj, "description", markupsafe.Markup("")) - uuids, value = _sanitize_description(raw_description, ctx) + uuids, value = _sanitize_description(obj, raw_description, ctx) ctx.setdefault("DESCR_REFERENCES", {})[obj.uuid] = uuids return CapellaWorkItem( type=helpers.resolve_element_type(xtype), @@ -128,7 +128,7 @@ def _generic_work_item( def _sanitize_description( - descr: markupsafe.Markup, ctx: dict[str, t.Any] + obj: common.GenericElement, descr: markupsafe.Markup, ctx: dict[str, t.Any] ) -> tuple[list[str], markupsafe.Markup]: referenced_uuids: list[str] = [] replaced_markup = RE_DESCR_LINK_PATTERN.sub( @@ -151,7 +151,11 @@ def repair_images(node: etree._Element) -> None: b64_img = base64.b64encode(img.read()).decode("utf8") node.attrib["src"] = f"data:{mime_type};base64,{b64_img}" except FileNotFoundError: - logger.error("Inline image can't be found from %r", file_path) + logger.error( + "Inline image can't be found from %r for %r", + file_path, + obj._short_repr_(), + ) repaired_markup = chelpers.process_html_fragments( replaced_markup, repair_images From 291bbbf56494a35fbdbf07bc656fa7b3c06325b9 Mon Sep 17 00:00:00 2001 From: Martin Lehmann Date: Mon, 11 Sep 2023 11:09:44 +0200 Subject: [PATCH 03/15] chore: Update pre-commit hooks Also switch to the pre-commit black mirror, which runs about 2x faster. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c8d02ae..06b612d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,8 +25,8 @@ repos: - id: end-of-file-fixer - id: fix-byte-order-marker - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 23.3.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.9.1 hooks: - id: black - repo: https://github.com/PyCQA/isort @@ -34,7 +34,7 @@ repos: hooks: - id: isort - repo: https://github.com/PyCQA/docformatter - rev: v1.7.2 + rev: v1.7.5 hooks: - id: docformatter additional_dependencies: @@ -47,7 +47,7 @@ repos: additional_dependencies: - pydocstyle[toml] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.5.1 hooks: - id: mypy additional_dependencies: @@ -97,10 +97,10 @@ repos: - --comment-style - "..| |" - repo: https://github.com/fsfe/reuse-tool - rev: v1.1.2 + rev: v2.1.0 hooks: - id: reuse - repo: https://github.com/qoomon/git-conventional-commits - rev: v2.6.4 + rev: v2.6.5 hooks: - id: conventional-commits From f80bf347e2d2f9c0342f1ae30f8048fe7a350654 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Sat, 2 Sep 2023 10:04:07 +0200 Subject: [PATCH 04/15] ci(ci-templates)!: Install packages on special tags for experiments This is just for the staging pipeline. --- .gitlab-ci.yml | 1 + ci-templates/gitlab/synchronise_elements.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c217c946..f2dca510 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: .patch-pyproject-toml: &patch-pyproject-toml - sed -i -e 's/\(^ "polarion-rest-api-client\).*",/\1",/' pyproject.toml + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl wheel: stage: build diff --git a/ci-templates/gitlab/synchronise_elements.yml b/ci-templates/gitlab/synchronise_elements.yml index a089ada8..bffb77dd 100644 --- a/ci-templates/gitlab/synchronise_elements.yml +++ b/ci-templates/gitlab/synchronise_elements.yml @@ -10,7 +10,8 @@ capella2polarion_synchronise_elements: artifacts: true script: - - pip install capella2polarion --pre + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/capella2polarion/less-requests/capella2polarion-less_requests-py3-none-any.whl - > python \ -m capella2polarion \ From 63426efb38f39d399144fb806455cb64b1910fe9 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 12 Sep 2023 16:49:48 +0200 Subject: [PATCH 05/15] fix: don't use get_linked_text for conditions as it does not work properly for deleted references --- capella2polarion/elements/serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 13b73a96..3a50d308 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -186,7 +186,7 @@ def include_pre_and_post_condition( def get_condition(cap: PrePostConditionElement, name: str) -> str: if not (condition := getattr(cap, name)): return "" - return get_linked_text(condition, ctx) + return condition.specification["capella:linkedText"].striptags() def strike_through(string: str) -> str: if match := RE_DESCR_DELETED_PATTERN.match(string): From 30469109362fe2145dac787669f9ce94e73425da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20W=C3=BCrger?= Date: Wed, 13 Sep 2023 13:21:07 +0200 Subject: [PATCH 06/15] fix: Prevent infinite SystemActor creation Picked changes from b09ce597. --- capella2polarion/elements/element.py | 20 +++++++++++++++++--- tests/test_elements.py | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 516409c7..fe39b2ee 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -13,7 +13,7 @@ from capellambse.model import diagram as diag from capella2polarion import elements -from capella2polarion.elements import serialize +from capella2polarion.elements import helpers, serialize logger = logging.getLogger(__name__) @@ -28,11 +28,25 @@ def create_work_items( ) -> list[serialize.CapellaWorkItem]: """Create a set of work items in Polarion.""" objects = chain.from_iterable(ctx["ELEMENTS"].values()) - work_items = [ + _work_items = [ serialize.element(obj, ctx, serialize.generic_work_item) for obj in objects ] - return list(filter(None.__ne__, work_items)) # type: ignore[arg-type] + _work_items = list(filter(None.__ne__, _work_items)) + valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) + work_items: list[polarion_api.CapellaWorkItem] = [] + missing_types: set[str] = set() + for work_item in _work_items: + assert work_item is not None + if work_item.type in valid_types: + work_items.append(work_item) + else: + missing_types.add(work_item.type) + logger.debug( + "%r are missing in the capella2polarion configuration", + ", ".join(missing_types), + ) + return work_items def create_links( diff --git a/tests/test_elements.py b/tests/test_elements.py index c0502c22..d83b2954 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -192,6 +192,8 @@ def test_create_work_items( monkeypatch: pytest.MonkeyPatch, context: dict[str, t.Any] ): del context["ELEMENTS"]["UnsupportedFakeModelObject"] + context["MODEL"] = model = mock.MagicMock() + model.by_uuid.side_effect = context["ELEMENTS"]["FakeModelObject"] monkeypatch.setattr( serialize, "generic_work_item", From 373c00b03c3298a82f807938bc0ca429a71dcb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20W=C3=BCrger?= <50786483+ewuerger@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:59:57 +0200 Subject: [PATCH 07/15] fix: Apply suggestions from code review Co-authored-by: micha91 --- capella2polarion/elements/__init__.py | 12 ++++++------ capella2polarion/elements/serialize.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index c08f97ff..c8b161f5 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -103,13 +103,13 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: ctx The context for the workitem operation to be processed. """ - work_items = [ - wi - for wi in ctx["WORK_ITEMS"].values() - if wi.uuid_capella not in ctx["POLARION_ID_MAP"] - ] - for work_item in work_items: + work_items: list[serialize.CapellaWorkItem] = [] + for work_item in ctx["WORK_ITEMS"].values(): + if work_item.uuid_capella not in ctx["POLARION_ID_MAP"]: + continue + assert work_item is not None + work_items.append(work_item) logger.info("Create work item for %r...", work_item.title) if work_items: try: diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 3a50d308..674c3670 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -48,7 +48,6 @@ class Condition(t.TypedDict): uuid_capella: str | None preCondition: Condition | None postCondition: Condition | None - checksum: str | None def element( From 94cf5a32094ec6511bf19157e9e6f87f18497d3f Mon Sep 17 00:00:00 2001 From: ewuerger Date: Wed, 11 Oct 2023 16:33:45 +0200 Subject: [PATCH 08/15] feat(model-elements): Add `input|output_exchanges` link handler --- capella2polarion/elements/element.py | 41 ++++++++++++++++++++++---- capella2polarion/elements/serialize.py | 8 +++-- tests/test_elements.py | 24 +++++++++++++-- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index fe39b2ee..46b831e6 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -4,6 +4,7 @@ from __future__ import annotations import collections.abc as cabc +import functools import logging import typing as t from itertools import chain @@ -11,6 +12,7 @@ import polarion_rest_api_client as polarion_api from capellambse.model import common from capellambse.model import diagram as diag +from capellambse.model.crosslayer import fa from capella2polarion import elements from capella2polarion.elements import helpers, serialize @@ -80,7 +82,7 @@ def create_links( continue if isinstance(refs, common.ElementList): - new = refs.by_uuid + new: cabc.Iterable[str] = refs.by_uuid # type: ignore[assignment] else: assert hasattr(refs, "uuid") new = [refs.uuid] @@ -141,7 +143,9 @@ def _handle_diagram_reference_links( return ref_links -def _collect_uuids(nodes: list[common.GenericElement]) -> cabc.Iterator[str]: +def _collect_uuids( + nodes: cabc.Iterable[common.GenericElement], +) -> cabc.Iterator[str]: type_resolvers = TYPE_RESOLVERS for node in nodes: uuid = node.uuid @@ -171,7 +175,34 @@ def _create( return list(filter(None.__ne__, _new_links)) -CUSTOM_LINKS = { - "description_reference": _handle_description_reference_links, - "diagram_elements": _handle_diagram_reference_links, +def _handle_exchanges( + context: dict[str, t.Any], + obj: fa.Function, + role_id: str, + links: dict[str, polarion_api.WorkItemLink], + attr: str = "inputs", +) -> list[polarion_api.WorkItemLink]: + wid = context["POLARION_ID_MAP"][obj.uuid] + exchanges: list[str] = [] + for element in getattr(obj, attr): + uuids = element.exchanges.by_uuid + exs = _get_work_item_ids(context, wid, uuids, role_id) + exchanges.extend(set(exs)) + return _create(context, wid, role_id, exchanges, links) + + +CustomLinkMaker = cabc.Callable[ + [ + dict[str, t.Any], + diag.Diagram | common.GenericElement, + str, + dict[str, t.Any], + ], + list[polarion_api.WorkItemLink], +] +CUSTOM_LINKS: dict[str, CustomLinkMaker] = { + "description_reference": _handle_description_reference_links, # type: ignore[dict-item] + "diagram_elements": _handle_diagram_reference_links, # type: ignore[dict-item] + "input_exchanges": functools.partial(_handle_exchanges, attr="inputs"), + "output_exchanges": functools.partial(_handle_exchanges, attr="outputs"), } diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 674c3670..5c01e6cd 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -173,8 +173,7 @@ def replace_markup( f'id="fake" data-item-id="{pid}" data-option-id="long">' "" ) - else: - return non_matcher(match.group(0)) + return non_matcher(match.group(0)) def include_pre_and_post_condition( @@ -262,7 +261,10 @@ def physical_component( return work_item -SERIALIZERS = { +Serializer = cabc.Callable[ + [common.GenericElement, dict[str, t.Any]], CapellaWorkItem +] +SERIALIZERS: dict[str, Serializer] = { "CapabilityRealization": include_pre_and_post_condition, "LogicalComponent": component_or_actor, "OperationalCapability": include_pre_and_post_condition, diff --git a/tests/test_elements.py b/tests/test_elements.py index d83b2954..4a070740 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -137,7 +137,7 @@ def __init__( self, uuid: str, name: str = "", - attribute: FakeModelObject | None = None, + attribute: t.Any | None = None, ): self.uuid = uuid self.name = name @@ -237,6 +237,26 @@ def test_create_links_custom_resolver(context: dict[str, t.Any]): assert links == [expected] + @staticmethod + def test_create_links_custom_exchanges_resolver(context: dict[str, t.Any]): + function_uuid = "ceffa011-7b66-4b3c-9885-8e075e312ffa" + obj = context["MODEL"].by_uuid(function_uuid) + context["POLARION_ID_MAP"][function_uuid] = "Obj-1" + context["POLARION_ID_MAP"][ + "1a414995-f4cd-488c-8152-486e459fb9de" + ] = "Obj-2" + context["ROLES"] = {"SystemFunction": ["input_exchanges"]} + expected = polarion_api.WorkItemLink( + "Obj-1", + "Obj-2", + "input_exchanges", + secondary_work_item_project="project_id", + ) + + links = element.create_links(obj, context) + + assert links == [expected] + @staticmethod def test_create_links_missing_attribute( context: dict[str, t.Any], caplog: pytest.LogCaptureFixture @@ -282,7 +302,7 @@ def test_create_links_from_ElementList(context: dict[str, t.Any]): secondary_work_item_project="project_id", ) - links = element.create_links(obj, context) + links = element.create_links(obj, context) # type: ignore[arg-type] assert expected_link in links assert expected_link1 in links From 24968560f5a7897d4e6c55111d927c1f830925a5 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Wed, 11 Oct 2023 17:04:35 +0200 Subject: [PATCH 09/15] fix!: Fix `post_work_items` --- capella2polarion/elements/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index c8b161f5..a2899692 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -105,7 +105,7 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: """ work_items: list[serialize.CapellaWorkItem] = [] for work_item in ctx["WORK_ITEMS"].values(): - if work_item.uuid_capella not in ctx["POLARION_ID_MAP"]: + if work_item.uuid_capella in ctx["POLARION_ID_MAP"]: continue assert work_item is not None From 85f28c1543945684903d59b48838a364ffe9cf13 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:38:30 +0200 Subject: [PATCH 10/15] fix: Fix various bugs - Fix debug log message for empty missing types - Merge diagram work item creator into generic work item creator - Handle all roles from the wildcard entry in the config. Also include `Diagram`. --- capella2polarion/__main__.py | 63 ++++++++++++++++++++++----- capella2polarion/elements/__init__.py | 18 ++++---- capella2polarion/elements/diagram.py | 24 ---------- capella2polarion/elements/element.py | 29 ++++++++---- tests/data/model_elements/config.yaml | 15 ++++--- tests/test_cli.py | 5 --- tests/test_elements.py | 8 ++-- 7 files changed, 94 insertions(+), 68 deletions(-) delete mode 100644 capella2polarion/elements/diagram.py diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 46bd34a0..097ade32 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -79,13 +79,30 @@ def _get_roles_from_config(ctx: dict[str, t.Any]) -> dict[str, list[str]]: roles[key] = list(role_ids) else: roles[typ] = [] - roles["Diagram"] = ["diagram_elements"] return roles def _sanitize_config( - config: dict[str, list[str | dict[str, t.Any]]], special: dict[str, t.Any] + config: dict[str, list[str | dict[str, t.Any]]], + special: list[str | dict[str, t.Any]], ) -> dict[str, t.Any]: + special_config: dict[str, t.Any] = {} + for typ in special: + if isinstance(typ, str): + special_config[typ] = None + else: + special_config.update(typ) + + lookup: dict[str, dict[str, list[str]]] = {} + for layer, xtypes in config.items(): + for xt in xtypes: + if isinstance(xt, str): + item: dict[str, list[str]] = {xt: []} + else: + item = xt + + lookup.setdefault(layer, {}).update(item) + new_config: dict[str, t.Any] = {} for layer, xtypes in config.items(): new_entries: list[str | dict[str, t.Any]] = [] @@ -93,16 +110,36 @@ def _sanitize_config( if isinstance(xtype, dict): for sub_key, sub_value in xtype.items(): new_value = ( - special.get("*", []) - + special.get(sub_key, []) + special_config.get("*", []) + + special_config.get(sub_key, []) + sub_value ) new_entries.append({sub_key: new_value}) else: - if new_value := special.get("*", []) + special.get(xtype, []): + star = special_config.get("*", []) + special_xtype = special_config.get(xtype, []) + if new_value := star + special_xtype: new_entries.append({xtype: new_value}) else: new_entries.append(xtype) + + wildcard_values = special_config.get("*", []) + for key, value in special_config.items(): + if key == "*": + continue + + if isinstance(value, list): + new_value = ( + lookup.get(layer, {}).get(key, []) + + wildcard_values + + value + ) + new_entries.append({key: new_value}) + elif value is None and key not in [ + entry if isinstance(entry, str) else list(entry.keys())[0] + for entry in new_entries + ]: + new_entries.append({key: wildcard_values}) new_config[layer] = new_entries return new_config @@ -189,16 +226,20 @@ def model_elements( ctx.obj["POLARION_ID_MAP"] = { uuid: wi.id for uuid, wi in ctx.obj["POLARION_WI_MAP"].items() } - diagrams = ctx.obj["ELEMENTS"].pop("Diagram", []) - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["ELEMENTS"]["Diagram"] = diagrams - pdiagrams = elements.diagram.create_diagrams(ctx.obj) - ctx.obj["WORK_ITEMS"] = { - wi.uuid_capella: wi for wi in work_items + pdiagrams + duuids = { + diag["uuid"] for diag in ctx.obj["DIAGRAM_IDX"] if diag["success"] } + ctx.obj["ELEMENTS"]["Diagram"] = [ + diag for diag in ctx.obj["ELEMENTS"]["Diagram"] if diag.uuid in duuids + ] + work_items = elements.element.create_work_items(ctx.obj) + ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} elements.delete_work_items(ctx.obj) elements.post_work_items(ctx.obj) + + work_items = elements.element.create_work_items(ctx.obj) + ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} elements.patch_work_items(ctx.obj) elements.make_model_elements_index(ctx.obj) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index a2899692..ab9c208a 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -114,6 +114,11 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: if work_items: try: ctx["API"].create_work_items(work_items) + workitems = {wi.uuid_capella: wi for wi in work_items if wi.id} + ctx["POLARION_WI_MAP"].update(workitems) + ctx["POLARION_ID_MAP"] = { + uuid: wi.id for uuid, wi in ctx["POLARION_WI_MAP"].items() + } except polarion_api.PolarionApiException as error: logger.error("Creating work items failed. %s", error.args[0]) @@ -140,21 +145,16 @@ def add_content( setattr(work_item, key, value) return work_item - ctx["POLARION_WI_MAP"] = get_polarion_wi_map(ctx) ctx["POLARION_ID_MAP"] = uuids = { uuid: wi.id for uuid, wi in ctx["POLARION_WI_MAP"].items() - if wi.status == "open" and uuid in ctx["WORK_ITEMS"] + if wi.status == "open" and wi.uuid_capella and wi.id } for uuid in uuids: elements = ctx["MODEL"] if uuid.startswith("_"): elements = ctx["MODEL"].diagrams - try: - obj = elements.by_uuid(uuid) - except KeyError: - logger.error("Weird %r", uuid) - continue + obj = elements.by_uuid(uuid) links = element.create_links(obj, ctx) @@ -189,6 +189,9 @@ def get_elements_and_type_map( if isinstance(typ, dict): typ = list(typ.keys())[0] + if typ == "Diagram": + continue + xtype = convert_type.get(typ, typ) objects = ctx["MODEL"].search(xtype, below=below) elements.setdefault(typ, []).extend(objects) @@ -277,7 +280,6 @@ def make_model_elements_index(ctx: dict[str, t.Any]) -> None: from . import ( # pylint: disable=cyclic-import api_helper, - diagram, element, helpers, serialize, diff --git a/capella2polarion/elements/diagram.py b/capella2polarion/elements/diagram.py deleted file mode 100644 index 4c82436e..00000000 --- a/capella2polarion/elements/diagram.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright DB Netz AG and contributors -# SPDX-License-Identifier: Apache-2.0 -"""Objects for synchronization of Capella diagrams to polarion.""" -from __future__ import annotations - -import logging -import typing as t - -from capella2polarion.elements import serialize - -logger = logging.getLogger(__name__) - - -def create_diagrams(ctx: dict[str, t.Any]) -> list[serialize.CapellaWorkItem]: - """Return a set of new work items of type ``diagram``.""" - uuids = {diag["uuid"] for diag in ctx["DIAGRAM_IDX"] if diag["success"]} - diagrams = [ - diag for diag in ctx["ELEMENTS"]["Diagram"] if diag.uuid in uuids - ] - work_items = [ - serialize.element(diagram, ctx, serialize.diagram) - for diagram in diagrams - ] - return list(filter(None.__ne__, work_items)) # type:ignore[arg-type] diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 46b831e6..84ae08d5 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -28,12 +28,21 @@ def create_work_items( ctx: dict[str, t.Any] ) -> list[serialize.CapellaWorkItem]: - """Create a set of work items in Polarion.""" + """Create a list of work items for Polarion.""" objects = chain.from_iterable(ctx["ELEMENTS"].values()) - _work_items = [ - serialize.element(obj, ctx, serialize.generic_work_item) - for obj in objects + _work_items = [] + serializer: cabc.Callable[ + [diag.Diagram | common.GenericElement, dict[str, t.Any]], + serialize.CapellaWorkItem, ] + for obj in objects: + if isinstance(obj, diag.Diagram): + serializer = serialize.diagram + else: + serializer = serialize.generic_work_item + + _work_items.append(serialize.element(obj, ctx, serializer)) + _work_items = list(filter(None.__ne__, _work_items)) valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) work_items: list[polarion_api.CapellaWorkItem] = [] @@ -44,10 +53,12 @@ def create_work_items( work_items.append(work_item) else: missing_types.add(work_item.type) - logger.debug( - "%r are missing in the capella2polarion configuration", - ", ".join(missing_types), - ) + + if missing_types: + logger.debug( + "%r are missing in the capella2polarion configuration", + ", ".join(missing_types), + ) return work_items @@ -118,7 +129,7 @@ def _handle_description_reference_links( role_id: str, links: dict[str, polarion_api.WorkItemLink], ) -> list[polarion_api.WorkItemLink]: - refs = context["DESCR_REFERENCES"].get(obj.uuid) + refs = context["DESCR_REFERENCES"].get(obj.uuid, []) wid = context["POLARION_ID_MAP"][obj.uuid] refs = set(_get_work_item_ids(context, wid, refs, role_id)) return _create(context, wid, role_id, refs, links) diff --git a/tests/data/model_elements/config.yaml b/tests/data/model_elements/config.yaml index e8647854..e032dcbb 100644 --- a/tests/data/model_elements/config.yaml +++ b/tests/data/model_elements/config.yaml @@ -2,11 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 "*": # All layers - "*": # All class types - - parent # Specify workitem links - - description_reference # Custom attribute - Class: - - state_machines + - "*": # All class types + - parent # Specify workitem links + - description_reference # Custom attribute + - Class: + - state_machines + - Diagram: + - diagram_elements + - Constraint oa: # Specify below - OperationalCapability: # Capella Type with references @@ -19,7 +22,6 @@ oa: # Specify below - CommunicationMean - Class - StateMachine - - Constraint sa: - SystemComponent: @@ -34,7 +36,6 @@ sa: - exchanged_items - ExchangeItem - Class - - Constraint pa: - PhysicalComponent: diff --git a/tests/test_cli.py b/tests/test_cli.py index a766ff82..0b752e17 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,10 +67,6 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): {}, ), ) - mock_create_diagrams = mock.MagicMock() - monkeypatch.setattr( - elements.diagram, "create_diagrams", mock_create_diagrams - ) mock_delete_work_items = mock.MagicMock() monkeypatch.setattr(elements, "delete_work_items", mock_delete_work_items) mock_post_work_items = mock.MagicMock() @@ -93,5 +89,4 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): assert mock_delete_work_items.call_count == 1 assert mock_patch_work_items.call_count == 1 assert mock_post_work_items.call_count == 1 - assert mock_create_diagrams.call_count == 1 assert ELEMENTS_IDX_PATH.exists() diff --git a/tests/test_elements.py b/tests/test_elements.py index 4a070740..de07e915 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -14,7 +14,7 @@ from capellambse.model import common from capella2polarion import elements -from capella2polarion.elements import diagram, element, helpers, serialize +from capella2polarion.elements import element, helpers, serialize # pylint: disable-next=relative-beyond-top-level, useless-suppression from .conftest import TEST_DIAGRAM_CACHE, TEST_HOST # type: ignore[import] @@ -90,7 +90,7 @@ def context( def test_create_diagrams(context: dict[str, t.Any]): context["ELEMENTS"] = {"Diagram": context["MODEL"].diagrams} - diagrams = diagram.create_diagrams(context) + diagrams = element.create_work_items(context) assert len(diagrams) == 1 work_item = diagrams[0] @@ -116,7 +116,7 @@ def test_create_diagrams_filters_non_diagram_elements( attributes.return_value = None monkeypatch.setattr(serialize, "element", attributes) - diagram.create_diagrams(context) + element.create_work_items(context) assert context["API"].create_work_items.call_count == 0 @@ -385,7 +385,7 @@ def test_update_work_items_filters_work_items_with_same_checksum( @staticmethod def test_update_links_with_no_elements(context: dict[str, t.Any]): - context["POLARION_ID_MAP"] = {} + context["POLARION_WI_MAP"] = {} elements.patch_work_items(context) From 5088d8470e0bd5c46faad0cd4b90f098a31a78a2 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:47:08 +0200 Subject: [PATCH 11/15] refactor: Simplify high level calls --- capella2polarion/__main__.py | 7 +++---- capella2polarion/elements/element.py | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 097ade32..23e0f2e8 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -232,14 +232,13 @@ def model_elements( ctx.obj["ELEMENTS"]["Diagram"] = [ diag for diag in ctx.obj["ELEMENTS"]["Diagram"] if diag.uuid in duuids ] - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} + elements.element.create_work_items(ctx.obj) elements.delete_work_items(ctx.obj) elements.post_work_items(ctx.obj) - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} + # Create missing links b/c of unresolved references + elements.element.create_work_items(ctx.obj) elements.patch_work_items(ctx.obj) elements.make_model_elements_index(ctx.obj) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 84ae08d5..c64ef780 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -59,6 +59,7 @@ def create_work_items( "%r are missing in the capella2polarion configuration", ", ".join(missing_types), ) + ctx["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} return work_items @@ -212,8 +213,8 @@ def _handle_exchanges( list[polarion_api.WorkItemLink], ] CUSTOM_LINKS: dict[str, CustomLinkMaker] = { - "description_reference": _handle_description_reference_links, # type: ignore[dict-item] - "diagram_elements": _handle_diagram_reference_links, # type: ignore[dict-item] + "description_reference": _handle_description_reference_links, + "diagram_elements": _handle_diagram_reference_links, "input_exchanges": functools.partial(_handle_exchanges, attr="inputs"), "output_exchanges": functools.partial(_handle_exchanges, attr="outputs"), } From 678ba2f74fdb79e56ac3a01a418f9bffda959f01 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:56:05 +0200 Subject: [PATCH 12/15] fix: Prepare for production pipelines --- .gitlab-ci.yml | 1 - ci-templates/gitlab/synchronise_elements.yml | 3 +-- pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2dca510..c217c946 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,6 @@ stages: .patch-pyproject-toml: &patch-pyproject-toml - sed -i -e 's/\(^ "polarion-rest-api-client\).*",/\1",/' pyproject.toml - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl wheel: stage: build diff --git a/ci-templates/gitlab/synchronise_elements.yml b/ci-templates/gitlab/synchronise_elements.yml index bffb77dd..a089ada8 100644 --- a/ci-templates/gitlab/synchronise_elements.yml +++ b/ci-templates/gitlab/synchronise_elements.yml @@ -10,8 +10,7 @@ capella2polarion_synchronise_elements: artifacts: true script: - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/capella2polarion/less-requests/capella2polarion-less_requests-py3-none-any.whl + - pip install capella2polarion --pre - > python \ -m capella2polarion \ diff --git a/pyproject.toml b/pyproject.toml index b9a600b9..0bb414cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "capellambse", "click", "PyYAML", - "polarion-rest-api-client @ git+https://github.com/DSD-DBS/polarion-rest-api-client.git@feat-add-attachments-workitem-relations", + "polarion-rest-api-client @ git+https://github.com/DSD-DBS/polarion-rest-api-client.git@v0.2.1", "requests", ] From a013f43e12c08f1219730ec5bfb54309b5dc7b9d Mon Sep 17 00:00:00 2001 From: ewuerger Date: Mon, 23 Oct 2023 16:16:33 +0200 Subject: [PATCH 13/15] refactor: Apply changes from code review --- capella2polarion/elements/api_helper.py | 12 ++++++++---- capella2polarion/elements/element.py | 4 ++-- capella2polarion/elements/serialize.py | 8 ++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/capella2polarion/elements/api_helper.py b/capella2polarion/elements/api_helper.py index d72403f7..966c2fc9 100644 --- a/capella2polarion/elements/api_helper.py +++ b/capella2polarion/elements/api_helper.py @@ -29,12 +29,16 @@ def patch_work_item( ctx The context to execute the patch for. obj - The Capella object to update the WorkItem from + The Capella object to update the WorkItem from. receiver A function that receives the WorkItem from the created - instances. + instances. This function alters the WorkItem instances by adding + attributes, e.g.: `linked_work_items`. It can be useful to add + attributes which can only be computed after the work item and + its default attributes were instantiated. name - The name of the object, which should be displayed in log messages. + The name of the object, which should be displayed in log + messages. _type The type of element, which should be shown in log messages. """ @@ -46,7 +50,7 @@ def patch_work_item( log_args = (wid, _type, name) logger.info("Update work item %r for model %s %r...", *log_args) - if new.uuid_capella: + if "uuid_capella" in new.additional_attributes: del new.additional_attributes["uuid_capella"] old.linked_work_items = ctx["API"].get_all_work_item_links(old.id) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index c64ef780..a3db417d 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -43,7 +43,7 @@ def create_work_items( _work_items.append(serialize.element(obj, ctx, serializer)) - _work_items = list(filter(None.__ne__, _work_items)) + _work_items = list(filter(None, _work_items)) valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) work_items: list[polarion_api.CapellaWorkItem] = [] missing_types: set[str] = set() @@ -184,7 +184,7 @@ def _create( ) for id in new ] - return list(filter(None.__ne__, _new_links)) + return list(filter(None, _new_links)) def _handle_exchanges( diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 5c01e6cd..d6c5f5b8 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -224,9 +224,7 @@ def constraint( ) -> CapellaWorkItem: """Return attributes for a ``Constraint``.""" work_item = _generic_work_item(obj, ctx) - work_item.description = ( # pylint: disable=attribute-defined-outside-init - get_linked_text(obj, ctx) - ) + work_item.description = get_linked_text(obj, ctx) return work_item @@ -244,9 +242,7 @@ def component_or_actor( xtype = RE_CAMEL_CASE_2ND_WORD_PATTERN.sub( r"\1Actor", type(obj).__name__ ) - work_item.type = helpers.resolve_element_type( # pylint: disable=attribute-defined-outside-init - xtype - ) + work_item.type = helpers.resolve_element_type(xtype) return work_item From 8a44a03d7ca30f572acf430b46408722a89e41fd Mon Sep 17 00:00:00 2001 From: ewuerger Date: Mon, 23 Oct 2023 16:23:34 +0200 Subject: [PATCH 14/15] ci: Please `pylint` --- capella2polarion/elements/serialize.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index d6c5f5b8..64d94bc6 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -224,6 +224,7 @@ def constraint( ) -> CapellaWorkItem: """Return attributes for a ``Constraint``.""" work_item = _generic_work_item(obj, ctx) + # pylint: disable-next=attribute-defined-outside-init work_item.description = get_linked_text(obj, ctx) return work_item @@ -242,6 +243,7 @@ def component_or_actor( xtype = RE_CAMEL_CASE_2ND_WORD_PATTERN.sub( r"\1Actor", type(obj).__name__ ) + # pylint: disable-next=attribute-defined-outside-init work_item.type = helpers.resolve_element_type(xtype) return work_item @@ -253,7 +255,8 @@ def physical_component( work_item = component_or_actor(obj, ctx) xtype = work_item.type if obj.nature is not None: - work_item.type = f"{xtype}{obj.nature.name.capitalize()}" # pylint: disable=attribute-defined-outside-init + # pylint: disable-next=attribute-defined-outside-init + work_item.type = f"{xtype}{obj.nature.name.capitalize()}" return work_item From 35a93117954436cd8ec5f56c3c09824ada277fc2 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Wed, 25 Oct 2023 10:19:08 +0200 Subject: [PATCH 15/15] fix: Add missing parameter to `get_linked_text` --- capella2polarion/elements/serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 7138f69b..6a0bfcf8 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -217,7 +217,7 @@ def get_linked_text( ) -> markupsafe.Markup: """Return sanitized markup of the given ``obj`` linked text.""" description = obj.specification["capella:linkedText"].striptags() - uuids, value = _sanitize_description(description, ctx) + uuids, value = _sanitize_description(obj, description, ctx) if uuids: ctx.setdefault("DESCR_REFERENCES", {})[obj.uuid] = uuids return value