From 9547978d4967ea06feb04dde50ab4ee752c701f6 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 18 Jan 2024 12:58:23 +0100 Subject: [PATCH 01/31] feat: use capellambse to get diagram images --- capella2polarion/__main__.py | 34 +++++++++---------- capella2polarion/cli.py | 31 ----------------- .../converters/element_converter.py | 22 +----------- capella2polarion/converters/link_converter.py | 6 ++-- .../converters/model_converter.py | 21 +++--------- tests/test_elements.py | 24 ++----------- 6 files changed, 27 insertions(+), 111 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 8c653c5..723880d 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -3,13 +3,13 @@ """Main entry point into capella2polarion.""" from __future__ import annotations +import json import logging import pathlib import typing import capellambse import click -from capellambse import cli_helpers from capella2polarion.cli import Capella2PolarionCli from capella2polarion.connectors import polarion_worker as pw @@ -47,7 +47,7 @@ ), default=None, ) -@click.option("--capella-model", type=cli_helpers.ModelCLI(), default=None) +@click.option("--capella-model", type=str, default=None) @click.option( "--synchronize-config", type=click.File(mode="r", encoding="utf8"), @@ -63,18 +63,27 @@ def cli( polarion_pat: str, polarion_delete_work_items: bool, capella_diagram_cache_folder_path: pathlib.Path, - capella_model: capellambse.MelodyModel, + capella_model: str, synchronize_config: typing.TextIO, ) -> None: """Synchronise data from Capella to Polarion.""" + if capella_model.startswith("{"): + logger.warning( + "DEPRECATED: Providing the model as json will be removed in a future version." + ) + capella_model = json.loads(capella_model)["path"] + + model = capellambse.MelodyModel( + capella_model, diagram_cache=capella_diagram_cache_folder_path + ) + capella2polarion_cli = Capella2PolarionCli( debug, polarion_project_id, polarion_url, polarion_pat, polarion_delete_work_items, - capella_diagram_cache_folder_path, - capella_model, + model, synchronize_config, force_update, ) @@ -96,28 +105,17 @@ def synchronize(ctx: click.core.Context) -> None: """Synchronise model elements.""" capella_to_polarion_cli: Capella2PolarionCli = ctx.obj logger.info( - "Synchronising diagrams from diagram cache at " - "%s to Polarion project with id %s...", - str(capella_to_polarion_cli.capella_diagram_cache_folder_path), + "Synchronising model elements at to Polarion project with id %s...", capella_to_polarion_cli.polarion_params.project_id, ) capella_to_polarion_cli.load_synchronize_config() - capella_to_polarion_cli.load_capella_diagram_cache_index() - - assert ( - capella_to_polarion_cli.capella_diagram_cache_index_content is not None - ) converter = model_converter.ModelConverter( capella_to_polarion_cli.capella_model, - capella_to_polarion_cli.capella_diagram_cache_folder_path, capella_to_polarion_cli.polarion_params.project_id, ) - converter.read_model( - capella_to_polarion_cli.config, - capella_to_polarion_cli.capella_diagram_cache_index_content, - ) + converter.read_model(capella_to_polarion_cli.config) polarion_worker = pw.CapellaPolarionWorker( capella_to_polarion_cli.polarion_params, diff --git a/capella2polarion/cli.py b/capella2polarion/cli.py index c38c2bd..9de5163 100644 --- a/capella2polarion/cli.py +++ b/capella2polarion/cli.py @@ -3,7 +3,6 @@ """Tool for CLI work.""" from __future__ import annotations -import json import logging import pathlib import typing @@ -27,7 +26,6 @@ def __init__( polarion_url: str, polarion_pat: str, polarion_delete_work_items: bool, - capella_diagram_cache_folder_path: pathlib.Path | None, capella_model: capellambse.MelodyModel, synchronize_config_io: typing.TextIO, force_update: bool = False, @@ -39,22 +37,9 @@ def __init__( polarion_pat, polarion_delete_work_items, ) - if capella_diagram_cache_folder_path is None: - raise ValueError("CapellaDiagramCacheFolderPath not filled") - self.capella_diagram_cache_folder_path = ( - capella_diagram_cache_folder_path - ) - self.capella_diagram_cache_index_file_path = ( - self.capella_diagram_cache_folder_path / "index.json" - ) - self.capella_diagram_cache_index_content: list[ - dict[str, typing.Any] - ] = [] self.capella_model: capellambse.MelodyModel = capella_model self.synchronize_config_io: typing.TextIO = synchronize_config_io - self.synchronize_config_content: dict[str, typing.Any] = {} - self.synchronize_config_roles: dict[str, list[str]] | None = None self.config = converter_config.ConverterConfig() self.force_update = force_update @@ -99,10 +84,6 @@ def _value(value): string_value = self._none_save_value_string(string_value) click.echo(f"{lighted_member_var}: '{string_value}'") - echo = ("NO", "YES")[ - self.capella_diagram_cache_index_file_path.is_file() - ] - click.echo(f"""Capella Diagram Cache Index-File exists: {echo}""") echo = ("YES", "NO")[self.synchronize_config_io.closed] click.echo(f"""Synchronize Config-IO is open: {echo}""") @@ -133,15 +114,3 @@ def load_synchronize_config(self) -> None: if not self.synchronize_config_io.readable(): raise RuntimeError("synchronize config io stream is not readable") self.config.read_config_file(self.synchronize_config_io) - - def load_capella_diagram_cache_index(self) -> None: - """Load Capella Diagram Cache index file content.""" - if not self.capella_diagram_cache_index_file_path.is_file(): - raise ValueError( - "capella diagramm cache index.json file does not exist" - ) - - l_text_content = self.capella_diagram_cache_index_file_path.read_text( - encoding="utf8" - ) - self.capella_diagram_cache_index_content = json.loads(l_text_content) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index ec3fb9d..e8594b5 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -57,22 +57,6 @@ def strike_through(string: str) -> str: return f'{string}' -def _decode_diagram(diagram_path: pathlib.Path) -> str: - mime_type, _ = mimetypes.guess_type(diagram_path) - if mime_type is None: - logger.error( - "Do not understand the MIME subtype for the diagram '%s'!", - diagram_path, - ) - return "" - content = diagram_path.read_bytes() - content_encoded = base64.standard_b64encode(content) - assert mime_type is not None - image_data = b"data:" + mime_type.encode() + b";base64," + content_encoded - src = image_data.decode() - return src - - def _format_texts( type_texts: dict[str, list[str]] ) -> dict[str, dict[str, str]]: @@ -136,12 +120,10 @@ class CapellaWorkItemSerializer: def __init__( self, - diagram_cache_path: pathlib.Path, model: capellambse.MelodyModel, capella_polarion_mapping: polarion_repo.PolarionDataRepository, converter_session: data_session.ConverterSession, ): - self.diagram_cache_path = diagram_cache_path self.model = model self.capella_polarion_mapping = capella_polarion_mapping self.converter_session = converter_session @@ -179,9 +161,7 @@ def _diagram( ) -> data_models.CapellaWorkItem: """Serialize a diagram for Polarion.""" diag = converter_data.capella_element - diagram_path = self.diagram_cache_path / f"{diag.uuid}.svg" - src = _decode_diagram(diagram_path) - description = _generate_image_html(src) + description = _generate_image_html(diag.as_datauri_svg) converter_data.work_item = data_models.CapellaWorkItem( type=converter_data.type_config.p_type, title=diag.name, diff --git a/capella2polarion/converters/link_converter.py b/capella2polarion/converters/link_converter.py index 89e1b74..3a3b70f 100644 --- a/capella2polarion/converters/link_converter.py +++ b/capella2polarion/converters/link_converter.py @@ -136,9 +136,11 @@ def _handle_diagram_reference_links( refs = set(self._collect_uuids(obj.nodes)) refs = set(self._get_work_item_ids(work_item_id, refs, role_id)) ref_links = self._create(work_item_id, role_id, refs, links) - except StopIteration: + except Exception as err: logger.exception( - "Could not create links for diagram %r", obj._short_repr_() + "Could not create links for diagram %r, because an error occured %s", + obj._short_repr_(), + err, ) ref_links = [] return ref_links diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index 66bafd2..bc18027 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -5,7 +5,6 @@ from __future__ import annotations import logging -import pathlib import typing as t import capellambse @@ -29,18 +28,15 @@ class ModelConverter: def __init__( self, model: capellambse.MelodyModel, - diagram_cache_path: pathlib.Path, project_id: str, ): self.model = model - self.diagram_cache_path = diagram_cache_path self.project_id = project_id self.converter_session: data_session.ConverterSession = {} def read_model( self, config: converter_config.ConverterConfig, - diagram_idx: list[dict[str, t.Any]], ): """Read the model using a given config and diagram_idx.""" missing_types: set[tuple[str, str, dict[str, t.Any]]] = set() @@ -65,16 +61,10 @@ def read_model( missing_types.add((layer, c_type, attributes)) if config.diagram_config: - diagrams_from_cache = { - d["uuid"] for d in diagram_idx if d["success"] - } for d in self.model.diagrams: - if d.uuid in diagrams_from_cache: - self.converter_session[ - d.uuid - ] = data_session.ConverterData( - "", config.diagram_config, d - ) + self.converter_session[d.uuid] = data_session.ConverterData( + "", config.diagram_config, d + ) if missing_types: for missing_type in missing_types: @@ -99,10 +89,7 @@ def generate_work_items( Links are not created in this step by default. """ serializer = element_converter.CapellaWorkItemSerializer( - self.diagram_cache_path, - self.model, - polarion_data_repo, - self.converter_session, + self.model, polarion_data_repo, self.converter_session ) work_items = serializer.serialize_all() for work_item in work_items: diff --git a/tests/test_elements.py b/tests/test_elements.py index 6afb4f3..02105d3 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -165,7 +165,6 @@ def write(self, text: str): polarion_url=TEST_HOST, polarion_pat="PrivateAccessToken", polarion_delete_work_items=True, - capella_diagram_cache_folder_path=TEST_DIAGRAM_CACHE, capella_model=model, synchronize_config_io=MyIO(), ) @@ -179,9 +178,7 @@ def write(self, text: str): c2p_cli.config = mock.Mock(converter_config.ConverterConfig) mc = model_converter.ModelConverter( - model, - c2p_cli.capella_diagram_cache_folder_path, - c2p_cli.polarion_params.project_id, + model, c2p_cli.polarion_params.project_id ) mc.converter_session = { @@ -283,7 +280,6 @@ def write(self, text: str): polarion_url=TEST_HOST, polarion_pat="PrivateAccessToken", polarion_delete_work_items=True, - capella_diagram_cache_folder_path=pathlib.Path(""), capella_model=model, synchronize_config_io=MyIO(), ) @@ -303,9 +299,7 @@ def write(self, text: str): ) mc = model_converter.ModelConverter( - model, - c2p_cli.capella_diagram_cache_folder_path, - c2p_cli.polarion_params.project_id, + model, c2p_cli.polarion_params.project_id ) mc.converter_session = { @@ -495,9 +489,6 @@ def test_create_links_custom_exchanges_resolver( work_item_obj_2, ) - base_object.c2pcli.synchronize_config_roles = { - "SystemFunction": ["input_exchanges"] - } expected = polarion_api.WorkItemLink( "Obj-1", "Obj-2", @@ -1029,7 +1020,6 @@ def test_diagram(model: capellambse.MelodyModel): diag = model.diagrams.by_uuid(TEST_DIAG_UUID) serializer = element_converter.CapellaWorkItemSerializer( - TEST_DIAGRAM_CACHE, model, polarion_repo.PolarionDataRepository(), { @@ -1052,14 +1042,6 @@ def test_diagram(model: capellambse.MelodyModel): linked_work_items=[], ) - @staticmethod - def test__decode_diagram(): - diagram_path = TEST_DIAGRAM_CACHE / "_APMboAPhEeynfbzU12yy7w.svg" - - diagram = element_converter._decode_diagram(diagram_path) - - assert diagram.startswith("data:image/svg+xml;base64,") - @staticmethod @pytest.mark.parametrize( "layer,uuid,expected", @@ -1215,7 +1197,6 @@ def test_generic_work_item( assert type_config is not None serializer = element_converter.CapellaWorkItemSerializer( - pathlib.Path(""), model, polarion_repo.PolarionDataRepository( [ @@ -1247,7 +1228,6 @@ def test_add_context_diagram(self, model: capellambse.MelodyModel): "test", "add_context_diagram", [] ) serializer = element_converter.CapellaWorkItemSerializer( - pathlib.Path(""), model, polarion_repo.PolarionDataRepository(), { From 83e8f7a6c213e512e6b5e74e2863d295196328fa Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 18 Jan 2024 13:25:29 +0100 Subject: [PATCH 02/31] feat: Add logging, if a diagram image is not successfully taken from cache --- capella2polarion/converters/element_converter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index e8594b5..b3f1eee 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -161,7 +161,16 @@ def _diagram( ) -> data_models.CapellaWorkItem: """Serialize a diagram for Polarion.""" diag = converter_data.capella_element - description = _generate_image_html(diag.as_datauri_svg) + + try: + src = diag.render("datauri_svg") + except Exception as error: + logger.exception( + "Failed to get diagram from cache. Error: %s", error + ) + src = diag.as_datauri_svg + + description = _generate_image_html(src) converter_data.work_item = data_models.CapellaWorkItem( type=converter_data.type_config.p_type, title=diag.name, From 549410d3924f71cd6281c8a501f6f2fd2712353f Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 18 Jan 2024 14:02:10 +0100 Subject: [PATCH 03/31] feat: Move diagram cache to the capella-model cli parameter and warn the user, when he tries to execute this tool without specifying a diagram cache --- capella2polarion/__main__.py | 34 ++++---------------- ci-templates/gitlab/synchronise_elements.yml | 1 - tests/conftest.py | 7 ++-- tests/test_cli.py | 5 ++- 4 files changed, 14 insertions(+), 33 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 723880d..88161ee 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -3,13 +3,12 @@ """Main entry point into capella2polarion.""" from __future__ import annotations -import json import logging -import pathlib import typing import capellambse import click +from capellambse import cli_helpers from capella2polarion.cli import Capella2PolarionCli from capella2polarion.connectors import polarion_worker as pw @@ -35,19 +34,7 @@ ) @click.option("--polarion-pat", envvar="POLARION_PAT", type=str) @click.option("--polarion-delete-work-items", is_flag=True, default=False) -@click.option( - "--capella-diagram-cache-folder-path", - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - readable=True, - resolve_path=True, - path_type=pathlib.Path, - ), - default=None, -) -@click.option("--capella-model", type=str, default=None) +@click.option("--capella-model", type=cli_helpers.ModelCLI(), default=None) @click.option( "--synchronize-config", type=click.File(mode="r", encoding="utf8"), @@ -62,20 +49,13 @@ def cli( polarion_url: str, polarion_pat: str, polarion_delete_work_items: bool, - capella_diagram_cache_folder_path: pathlib.Path, - capella_model: str, + capella_model: capellambse.MelodyModel, synchronize_config: typing.TextIO, ) -> None: """Synchronise data from Capella to Polarion.""" - if capella_model.startswith("{"): - logger.warning( - "DEPRECATED: Providing the model as json will be removed in a future version." - ) - capella_model = json.loads(capella_model)["path"] - - model = capellambse.MelodyModel( - capella_model, diagram_cache=capella_diagram_cache_folder_path - ) + # if capella_model.diagram_cache is None: + if not hasattr(capella_model, "_diagram_cache"): + logger.warning("It's highly recommended to define a diagram cache!") capella2polarion_cli = Capella2PolarionCli( debug, @@ -83,7 +63,7 @@ def cli( polarion_url, polarion_pat, polarion_delete_work_items, - model, + capella_model, synchronize_config, force_update, ) diff --git a/ci-templates/gitlab/synchronise_elements.yml b/ci-templates/gitlab/synchronise_elements.yml index 94d4a0f..dfcae1e 100644 --- a/ci-templates/gitlab/synchronise_elements.yml +++ b/ci-templates/gitlab/synchronise_elements.yml @@ -18,6 +18,5 @@ capella2polarion_synchronise_elements: $([[ $CAPELLA2POLARION_FORCE_UPDATE -eq 1 ]] && echo '--force-update') \ --polarion-project-id=${CAPELLA2POLARION_PROJECT_ID:?} \ --capella-model="${CAPELLA2POLARION_MODEL_JSON:?}" \ - --capella-diagram-cache-folder-path=./diagram_cache \ --synchronize-config=${CAPELLA2POLARION_CONFIG:?} \ synchronize diff --git a/tests/conftest.py b/tests/conftest.py index cbf1ea8..76ce35d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,10 @@ TEST_DIAGRAM_CACHE = TEST_DATA_ROOT / "diagram_cache" TEST_MODEL_ELEMENTS = TEST_DATA_ROOT / "model_elements" TEST_MODEL_ELEMENTS_CONFIG = TEST_MODEL_ELEMENTS / "config.yaml" -TEST_MODEL = TEST_DATA_ROOT / "model" / "Melody Model Test.aird" +TEST_MODEL = { + "path": str(TEST_DATA_ROOT / "model" / "Melody Model Test.aird"), + "diagram_cache": str(TEST_DIAGRAM_CACHE), +} TEST_HOST = "https://api.example.com" @@ -32,7 +35,7 @@ def diagram_cache_index() -> list[dict[str, t.Any]]: @pytest.fixture def model() -> capellambse.MelodyModel: """Return the test model.""" - return capellambse.MelodyModel(path=TEST_MODEL) + return capellambse.MelodyModel(**TEST_MODEL) @pytest.fixture diff --git a/tests/test_cli.py b/tests/test_cli.py index 5fbadd9..77777f6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json from unittest import mock import polarion_rest_api_client as polarion_api @@ -50,10 +51,8 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): "--polarion-pat", "AlexandersPrivateAcessToken", "--polarion-delete-work-items", - "--capella-diagram-cache-folder-path", - str(TEST_DIAGRAM_CACHE), "--capella-model", - str(TEST_MODEL), + json.dumps(TEST_MODEL), "--synchronize-config", str(TEST_MODEL_ELEMENTS_CONFIG), "synchronize", From 59511ea5d1f34ace88d55e99a6b04be8df1fea59 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 18 Jan 2024 18:37:43 +0100 Subject: [PATCH 04/31] feat: modify diagram cache check for new capellambse release --- capella2polarion/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 88161ee..0ac7b8c 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -53,8 +53,7 @@ def cli( synchronize_config: typing.TextIO, ) -> None: """Synchronise data from Capella to Polarion.""" - # if capella_model.diagram_cache is None: - if not hasattr(capella_model, "_diagram_cache"): + if capella_model.diagram_cache is None: logger.warning("It's highly recommended to define a diagram cache!") capella2polarion_cli = Capella2PolarionCli( From aaa19a6ef072d4a5393d23733a55e96d1225ba2f Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 19 Jan 2024 07:49:58 +0100 Subject: [PATCH 05/31] docs: Adjust documentation for pipeline configuration --- docs/source/pipeline templates/gitlab.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/pipeline templates/gitlab.rst b/docs/source/pipeline templates/gitlab.rst index 060787d..fb9f6b2 100644 --- a/docs/source/pipeline templates/gitlab.rst +++ b/docs/source/pipeline templates/gitlab.rst @@ -12,7 +12,9 @@ following template can be used inside the `.gitlab-ci.yml` file: :language: yaml :lines: 4- -A `.gitlab-ci.yml` with a capella2polarion synchronization job could look like +We highly recommend using the diagram cache as a separate job and defined it as a dependency in our template for that +reason. The diagram cache artifacts have to be included in the capella2polarion job and its path must be defined in the +`CAPELLA2POLARION_MODEL_JSON` variable. A `.gitlab-ci.yml` with a capella2polarion synchronization job could look like this: .. code:: yaml @@ -33,6 +35,6 @@ this: CAPELLA_VERSION: 6.1.0 ENTRYPOINT: model.aird CAPELLA2POLARION_PROJECT_ID: syncproj - CAPELLA2POLARION_MODEL_JSON: '{"path": "PATH_TO_CAPELLA"}' + CAPELLA2POLARION_MODEL_JSON: '{"path": "PATH_TO_CAPELLA", "diagram_cache": "./diagram_cache"}' CAPELLA2POLARION_CONFIG: capella2polarion_config.yaml CAPELLA2POLARION_DEBUG: 1 From 376fda1610a5949fff5cddad7e03af203f005b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20W=C3=BCrger?= <50786483+ewuerger@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:31:37 +0100 Subject: [PATCH 06/31] docs: Fix typo --- capella2polarion/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 0ac7b8c..1228eed 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -84,7 +84,7 @@ def synchronize(ctx: click.core.Context) -> None: """Synchronise model elements.""" capella_to_polarion_cli: Capella2PolarionCli = ctx.obj logger.info( - "Synchronising model elements at to Polarion project with id %s...", + "Synchronising model elements to Polarion project with id %s...", capella_to_polarion_cli.polarion_params.project_id, ) capella_to_polarion_cli.load_synchronize_config() From c1b666bef26b0a045c5651f0d1023692f587ba5a Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 5 Feb 2024 17:13:14 +0100 Subject: [PATCH 07/31] fix: tests after rebase --- tests/test_elements.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_elements.py b/tests/test_elements.py index 02105d3..2d22cbc 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -1255,7 +1255,6 @@ def test_multiple_serializers(self, model: capellambse.MelodyModel): [], ) serializer = element_converter.CapellaWorkItemSerializer( - pathlib.Path(""), model, polarion_repo.PolarionDataRepository(), { From d32f072464bb1aa247e368b52d06d1cb5ac43502 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 1 Feb 2024 17:00:37 +0100 Subject: [PATCH 08/31] feat: Move images to attachments - WIP --- capella2polarion/__main__.py | 4 +- .../connectors/polarion_worker.py | 180 +++++++++++++++--- .../converters/element_converter.py | 100 +++++++--- .../converters/model_converter.py | 22 +++ capella2polarion/data_models.py | 39 ++++ pyproject.toml | 3 +- 6 files changed, 295 insertions(+), 53 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 1228eed..1d6550f 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -110,7 +110,9 @@ def synchronize(ctx: click.core.Context) -> None: polarion_worker.post_work_items(converter.converter_session) # Create missing links for new work items - converter.generate_work_items(polarion_worker.polarion_data_repo, True) + converter.generate_work_items( + polarion_worker.polarion_data_repo, True, True + ) polarion_worker.patch_work_items(converter.converter_session) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index c5c97de..4b7dc97 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -4,11 +4,14 @@ from __future__ import annotations import collections.abc as cabc +import json import logging import typing as t from urllib import parse import polarion_rest_api_client as polarion_api +from capellambse import helpers as chelpers +from lxml import etree from capella2polarion import data_models from capella2polarion.connectors import polarion_repo @@ -169,51 +172,90 @@ def patch_work_item( log_args = (old.id, new.type, new.title) logger.info("Update work item %r for model %s %r...", *log_args) - del new.additional_attributes["uuid_capella"] + if old.get_current_checksum()[0] != "{": + old_checksums = {"__C2P__WORK_ITEM": old.get_current_checksum()} + else: + old_checksums = json.loads(old.get_current_checksum()) - old = self.client.get_work_item(old.id) + new_checksums = json.loads(new.get_current_checksum()) - if old.linked_work_items_truncated: - old.linked_work_items = self.client.get_all_work_item_links(old.id) + new_work_item_check_sum = new_checksums.pop("__C2P__WORK_ITEM") + old_work_item_check_sum = old_checksums.pop("__C2P__WORK_ITEM") - del old.additional_attributes["uuid_capella"] + work_item_changed = new_work_item_check_sum != old_work_item_check_sum - # Type will only be updated, if it is set and should be used carefully - if new.type == old.type: - new.type = None - new.status = "open" + try: + old_attachments = self.client.get_all_work_item_attachments( + work_item_id=old.id + ) + self.update_attachments( + new, old_checksums, new_checksums, old_attachments + ) + except polarion_api.PolarionApiException as error: + logger.error( + "Getting Attachments for WorkItem %r (%s %s) failed. %s", + *log_args, + error.args[0], + ) + return + + self._refactor_attached_images(new) + assert new.id is not None + delete_links = None + create_links = None - # If additional fields were present in the past, but aren't anymore, - # we have to set them to an empty value manually - defaults = DEFAULT_ATTRIBUTE_VALUES - for attribute, value in old.additional_attributes.items(): - if attribute not in new.additional_attributes: - new.additional_attributes[attribute] = defaults.get( - type(value) + if work_item_changed or self.force_update: + old = self.client.get_work_item(old.id) + + del new.additional_attributes["uuid_capella"] + if old.linked_work_items_truncated: + old.linked_work_items = self.client.get_all_work_item_links( + old.id ) - elif new.additional_attributes[attribute] == value: - del new.additional_attributes[attribute] - assert new.id is not None + # Type will only be updated, if it is set and should be used carefully + if new.type == old.type: + new.type = None + new.status = "open" + + # If additional fields were present in the past, but aren't anymore, + # we have to set them to an empty value manually + defaults = DEFAULT_ATTRIBUTE_VALUES + for attribute, value in old.additional_attributes.items(): + if attribute not in new.additional_attributes: + new.additional_attributes[attribute] = defaults.get( + type(value) + ) + elif new.additional_attributes[attribute] == value: + del new.additional_attributes[attribute] + + delete_links = CapellaPolarionWorker.get_missing_link_ids( + old.linked_work_items, new.linked_work_items + ) + create_links = CapellaPolarionWorker.get_missing_link_ids( + new.linked_work_items, old.linked_work_items + ) + else: + new.additional_attributes = {} + del new.type + del new.status + new.description = None + new.description_type = None + new.title = None + try: self.client.update_work_item(new) - if delete_link_ids := CapellaPolarionWorker.get_missing_link_ids( - old.linked_work_items, new.linked_work_items - ): - id_list_str = ", ".join(delete_link_ids.keys()) + if delete_links: + id_list_str = ", ".join(delete_links.keys()) logger.info( "Delete work item links %r for model %s %r", id_list_str, new.type, new.title, ) - self.client.delete_work_item_links( - list(delete_link_ids.values()) - ) + self.client.delete_work_item_links(list(delete_links.values())) - if create_links := CapellaPolarionWorker.get_missing_link_ids( - new.linked_work_items, old.linked_work_items - ): + if create_links: id_list_str = ", ".join(create_links.keys()) logger.info( "Create work item links %r for model %s %r", @@ -230,6 +272,86 @@ def patch_work_item( error.args[0], ) + def _refactor_attached_images(self, new: data_models.CapellaWorkItem): + def set_attachment_id(node: etree._Element) -> None: + if node.tag != "img": + return + if img_src := node.attrib.get("src"): + if img_src.startswith("attachment:"): + file_name = img_src[11:] + for attachment in new.attachments: + if attachment.file_name == file_name: + node.attrib["src"] = f"attachment:{attachment.id}" + return + + new.description = chelpers.process_html_fragments( + new.description, set_attachment_id + ) + for _, value in new.additional_attributes.items(): + if isinstance(value, dict): + if value.get("type") == "text/html": + value["value"] = chelpers.process_html_fragments( + value["value"], set_attachment_id + ) + + def update_attachments( + self, + new: data_models.CapellaWorkItem, + old_checksums: dict[str, str], + new_checksums: dict[str, str], + old_attachments: list[polarion_api.WorkItemAttachment], + ): + """Delete, create and update attachments in one go. + + After execution all attachments of the new work item should have + IDs. + """ + new_attachment_dict = { + attachment.file_name: attachment for attachment in new.attachments + } + old_attachment_dict = { + attachment.file_name: attachment for attachment in old_attachments + } + + old_attachment_file_names = set(old_attachment_dict) + new_attachment_file_names = set(new_attachment_dict) + + for file_name in old_attachment_file_names - new_attachment_file_names: + self.client.delete_work_item_attachment( + old_attachment_dict[file_name] + ) + + if new_attachments := list( + map( + new_attachment_dict.get, + new_attachment_file_names - old_attachment_file_names, + ) + ): + self.client.create_work_item_attachments(new_attachments) + + attachments_for_update = {} + for common_attachment_file_name in ( + old_attachment_file_names & new_attachment_file_names + ): + attachment = new_attachment_dict[common_attachment_file_name] + attachment.id = old_attachment_dict[common_attachment_file_name].id + if ( + new_checksums[attachment.file_name] + != old_checksums.get(attachment.file_name) + or self.force_update + ): + attachments_for_update[attachment.file_name] = attachment + + for file_name, attachment in attachments_for_update: + # SVGs should only be updated if their PNG differs + if ( + attachment.mime_type == "image/svg+xml" + and file_name[:3] + "png" not in attachments_for_update + ): + continue + + self.client.update_work_item_attachment(attachment) + @staticmethod def get_missing_link_ids( left: cabc.Iterable[polarion_api.WorkItemLink], diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index b3f1eee..0c8428e 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -3,8 +3,8 @@ """Objects for serialization of capella objects to workitems.""" from __future__ import annotations -import base64 import collections +import hashlib import logging import mimetypes import pathlib @@ -14,6 +14,7 @@ import capellambse import markupsafe +import polarion_rest_api_client as polarion_api from capellambse import helpers as chelpers from capellambse.model import common from capellambse.model.crosslayer import interaction @@ -43,6 +44,7 @@ ] logger = logging.getLogger(__name__) +C2P_IMAGE_PREFIX = "__C2P__" def resolve_element_type(type_: str) -> str: @@ -103,12 +105,12 @@ def _condition( return {"type": _type, "value": value} -def _generate_image_html(src: str) -> str: +def _generate_image_html(attachment_id: str) -> str: """Generate an image as HTMl with the given source.""" style = "; ".join( (f"{key}: {value}" for key, value in DIAGRAM_STYLES.items()) ) - description = f'

' + description = f'

' return description @@ -160,24 +162,35 @@ def _diagram( self, converter_data: data_session.ConverterData ) -> data_models.CapellaWorkItem: """Serialize a diagram for Polarion.""" - diag = converter_data.capella_element + diagram = converter_data.capella_element try: - src = diag.render("datauri_svg") + diagram_svg = diagram.render("svg") except Exception as error: logger.exception( "Failed to get diagram from cache. Error: %s", error ) - src = diag.as_datauri_svg + diagram_svg = diagram.as_svg + + file_name = f"{C2P_IMAGE_PREFIX}_diagram.svg" - description = _generate_image_html(src) converter_data.work_item = data_models.CapellaWorkItem( type=converter_data.type_config.p_type, - title=diag.name, + title=diagram.name, description_type="text/html", - description=description, + description=_generate_image_html(file_name), status="open", - uuid_capella=diag.uuid, + uuid_capella=diagram.uuid, + attachments=[ + polarion_api.WorkItemAttachment( + "", + "", + "Context Diagram", + diagram_svg, + "image/svg+xml", + file_name, + ) + ], ) return converter_data.work_item @@ -186,8 +199,9 @@ def _generic_work_item( ) -> data_models.CapellaWorkItem: obj = converter_data.capella_element raw_description = getattr(obj, "description", None) - uuids, value = self._sanitize_description( - obj, raw_description or markupsafe.Markup("") + uuids, value, attachments = self._sanitize_description( + obj, raw_description + or markupsafe.Markup("") ) converter_data.description_references = uuids requirement_types = _get_requirement_types_text(obj) @@ -198,19 +212,24 @@ def _generic_work_item( description=value, status="open", uuid_capella=obj.uuid, + attachments=attachments, **requirement_types, ) return converter_data.work_item def _sanitize_description( self, obj: common.GenericElement, descr: markupsafe.Markup - ) -> tuple[list[str], markupsafe.Markup]: + ) -> tuple[ + list[str], markupsafe.Markup, list[polarion_api.WorkItemAttachment] + ]: referenced_uuids: list[str] = [] replaced_markup = RE_DESCR_LINK_PATTERN.sub( lambda match: self._replace_markup(match, referenced_uuids, 2), descr, ) + attachments: list[polarion_api.WorkItemAttachment] = [] + def repair_images(node: etree._Element) -> None: if node.tag != "img": return @@ -225,8 +244,20 @@ def repair_images(node: etree._Element) -> None: ] try: with filehandler.open(file_path, "r") as img: - b64_img = base64.b64encode(img.read()).decode("utf8") - node.attrib["src"] = f"data:{mime_type};base64,{b64_img}" + content = img.read() + file_name = f"{hashlib.md5(str(file_path).encode('utf8')).hexdigest()}.{file_path.suffix}" + attachments.append( + polarion_api.WorkItemAttachment( + "", + "", + file_path.name, + content, + mime_type, + file_name, + ) + ) + node.attrib["src"] = f"attachment:{file_name}" + except FileNotFoundError: logger.error( "Inline image can't be found from %r for %r", @@ -237,7 +268,7 @@ def repair_images(node: etree._Element) -> None: repaired_markup = chelpers.process_html_fragments( replaced_markup, repair_images ) - return referenced_uuids, repaired_markup + return referenced_uuids, repaired_markup, attachments def _replace_markup( self, @@ -294,17 +325,19 @@ def matcher(match: re.Match) -> str: def _get_linked_text( self, converter_data: data_session.ConverterData - ) -> markupsafe.Markup: + ) -> tuple[markupsafe.Markup, list[polarion_api.WorkItemAttachment]]: """Return sanitized markup of the given ``obj`` linked text.""" obj = converter_data.capella_element default = {"capella:linkedText": markupsafe.Markup("")} description = getattr(obj, "specification", default)[ "capella:linkedText" ].striptags() - uuids, value = self._sanitize_description(obj, description) + uuids, value, attachments = self._sanitize_description( + obj, description + ) if uuids: converter_data.description_references = uuids - return value + return value, attachments def _linked_text_as_description( self, converter_data: data_session.ConverterData @@ -312,9 +345,10 @@ def _linked_text_as_description( """Return attributes for a ``Constraint``.""" # pylint: disable-next=attribute-defined-outside-init assert converter_data.work_item, "No work item set yet" - converter_data.work_item.description = self._get_linked_text( + converter_data.work_item.description, attachments = self._get_linked_text( converter_data ) + converter_data.work_item.attachments += attachments return converter_data.work_item def _add_context_diagram( @@ -323,9 +357,20 @@ def _add_context_diagram( """Add a new custom field context diagram.""" assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.context_diagram + file_name = f"{C2P_IMAGE_PREFIX}_context_diagram.svg" + converter_data.work_item.attachments.append( + polarion_api.WorkItemAttachment( + "", + "", + "Context Diagram", + diagram.as_svg, + "image/svg+xml", + file_name, + ) + ) converter_data.work_item.additional_attributes["context_diagram"] = { "type": "text/html", - "value": _generate_image_html(diagram.as_datauri_svg), + "value": _generate_image_html(file_name), } return converter_data.work_item @@ -335,8 +380,19 @@ def _add_tree_diagram( """Add a new custom field tree diagram.""" assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.tree_view + file_name = f"{C2P_IMAGE_PREFIX}_tree_view.svg" + converter_data.work_item.attachments.append( + polarion_api.WorkItemAttachment( + "", + "", + "Tree View", + diagram.as_svg, + "image/svg+xml", + file_name, + ) + ) converter_data.work_item.additional_attributes["tree_view"] = { "type": "text/html", - "value": _generate_image_html(diagram.as_datauri_svg), + "value": _generate_image_html(file_name), } return converter_data.work_item diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index bc18027..0ce4299 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -7,6 +7,7 @@ import logging import typing as t +import cairosvg import capellambse import polarion_rest_api_client as polarion_api @@ -81,6 +82,7 @@ def generate_work_items( self, polarion_data_repo: polarion_repo.PolarionDataRepository, generate_links: bool = False, + pngs_for_svgs: bool = False, ) -> dict[str, data_models.CapellaWorkItem]: """Return a work items mapping from model elements for Polarion. @@ -99,8 +101,28 @@ def generate_work_items( if generate_links: self.generate_work_item_links(polarion_data_repo) + if pngs_for_svgs: + self.generate_pngs_for_svgs() + return {wi.uuid_capella: wi for wi in work_items} + def generate_pngs_for_svgs(self): + """Generate PNG files for all SVGs for all work items.""" + for converter_data in self.converter_session.values(): + if converter_data.work_item is not None: + converter_data.work_item.attachments += [ + polarion_api.WorkItemAttachment( + attachment.work_item_id, + "", + attachment.title, + cairosvg.svg2png(attachment.content_bytes), + "image/png", + attachment.file_name[:-3] + "png", + ) + for attachment in converter_data.work_item.attachments + if attachment.mime_type == "image/svg+xml" + ] + def generate_work_item_links( self, polarion_data_repo: polarion_repo.PolarionDataRepository ): diff --git a/capella2polarion/data_models.py b/capella2polarion/data_models.py index 2f750e7..ca9c0f0 100644 --- a/capella2polarion/data_models.py +++ b/capella2polarion/data_models.py @@ -3,6 +3,9 @@ """Module providing the CapellaWorkItem class.""" from __future__ import annotations +import base64 +import hashlib +import json import typing as t import polarion_rest_api_client as polarion_api @@ -20,3 +23,39 @@ class Condition(t.TypedDict): uuid_capella: str preCondition: Condition | None postCondition: Condition | None + + def calculate_checksum(self) -> str: + """Calculate and return a checksum for this WorkItem. + + In addition, the checksum will be written to self._checksum. + """ + data = self.to_dict() + del data["checksum"] + del data["id"] + + attachments = data.pop("attachments") + attachment_checksums = {} + for attachment in attachments: + # Don't store checksums for SVGs as we can check their PNGs instead + if attachment["mime_type"] == "image/svg+xml": + continue + try: + attachment["content_bytes"] = base64.b64encode( + attachment["content_bytes"] + ).decode("utf8") + except TypeError: + pass + + del attachment["id"] + attachment_checksums[attachment["file_name"]] = hashlib.sha256( + json.dumps(attachment).encode("utf8") + ).hexdigest() + + data = dict(sorted(data.items())) + + converted = json.dumps(data).encode("utf8") + self._checksum = json.dumps( + {"__C2P__WORK_ITEM": hashlib.sha256(converted).hexdigest()} + | dict(sorted(attachment_checksums.items())) + ) + return self._checksum diff --git a/pyproject.toml b/pyproject.toml index dbbcceb..ede3b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,8 @@ dependencies = [ "click", "PyYAML", "polarion-rest-api-client==0.3.0", - "bidict" + "bidict", + "cairosvg", ] [project.urls] From ba2b3a3c03aba859d48ebb79502463b9b97617fb Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 2 Feb 2024 08:24:19 +0100 Subject: [PATCH 09/31] fix: minor errors and tests --- .../connectors/polarion_worker.py | 21 ++++++++++++------- .../converters/element_converter.py | 11 +++++++--- capella2polarion/converters/link_converter.py | 3 ++- tests/test_elements.py | 7 +++---- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 4b7dc97..5afebb0 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -163,12 +163,14 @@ def patch_work_item( """Patch a given WorkItem.""" new = converter_session[uuid].work_item _, old = self.polarion_data_repo[uuid] - if not self.force_update and new == old: - return assert old is not None assert new is not None + new.calculate_checksum() + if not self.force_update and new == old: + return + log_args = (old.id, new.type, new.title) logger.info("Update work item %r for model %s %r...", *log_args) @@ -208,17 +210,19 @@ def patch_work_item( old = self.client.get_work_item(old.id) del new.additional_attributes["uuid_capella"] + del old.additional_attributes["uuid_capella"] + if old.linked_work_items_truncated: old.linked_work_items = self.client.get_all_work_item_links( old.id ) - # Type will only be updated, if it is set and should be used carefully + # Type will only be updated, if set and should be used carefully if new.type == old.type: new.type = None new.status = "open" - # If additional fields were present in the past, but aren't anymore, + # If additional fields were present, but aren't anymore, # we have to set them to an empty value manually defaults = DEFAULT_ATTRIBUTE_VALUES for attribute, value in old.additional_attributes.items(): @@ -284,12 +288,13 @@ def set_attachment_id(node: etree._Element) -> None: node.attrib["src"] = f"attachment:{attachment.id}" return - new.description = chelpers.process_html_fragments( - new.description, set_attachment_id - ) + if new.description: + new.description = chelpers.process_html_fragments( + new.description, set_attachment_id + ) for _, value in new.additional_attributes.items(): if isinstance(value, dict): - if value.get("type") == "text/html": + if value.get("type") == "text/html" and value.get("value"): value["value"] = chelpers.process_html_fragments( value["value"], set_attachment_id ) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 0c8428e..88a245d 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -110,7 +110,10 @@ def _generate_image_html(attachment_id: str) -> str: style = "; ".join( (f"{key}: {value}" for key, value in DIAGRAM_STYLES.items()) ) - description = f'

' + description = ( + f'

' + ) return description @@ -245,7 +248,9 @@ def repair_images(node: etree._Element) -> None: try: with filehandler.open(file_path, "r") as img: content = img.read() - file_name = f"{hashlib.md5(str(file_path).encode('utf8')).hexdigest()}.{file_path.suffix}" + file_name = hashlib.md5( + str(file_path).encode("utf8") + ).hexdigest() attachments.append( polarion_api.WorkItemAttachment( "", @@ -253,7 +258,7 @@ def repair_images(node: etree._Element) -> None: file_path.name, content, mime_type, - file_name, + f"{file_name}.{file_path.suffix}", ) ) node.attrib["src"] = f"attachment:{file_name}" diff --git a/capella2polarion/converters/link_converter.py b/capella2polarion/converters/link_converter.py index 3a3b70f..8ac5de5 100644 --- a/capella2polarion/converters/link_converter.py +++ b/capella2polarion/converters/link_converter.py @@ -138,7 +138,8 @@ def _handle_diagram_reference_links( ref_links = self._create(work_item_id, role_id, refs, links) except Exception as err: logger.exception( - "Could not create links for diagram %r, because an error occured %s", + "Could not create links for diagram %r, " + "because an error occured %s", obj._short_repr_(), err, ) diff --git a/tests/test_elements.py b/tests/test_elements.py index 2d22cbc..3bc8fbf 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -54,9 +54,7 @@ TEST_SCENARIO = "afdaa095-e2cd-4230-b5d3-6cb771a90f51" TEST_CAP_REAL = "b80b3141-a7fc-48c7-84b2-1467dcef5fce" TEST_CONSTRAINT = "95cbd4af-7224-43fe-98cb-f13dda540b8e" -TEST_DIAG_DESCR = ( - '

Test requirement 1 really l o n g text that is way too long to " From 57168fcdc7a61d8b3b85a9dc4cdd530cdae1e989 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 2 Feb 2024 09:29:38 +0100 Subject: [PATCH 10/31] test: refactor existing tests to check for attachments for diagrams and context_diagrams --- .../converters/element_converter.py | 23 +++++++++++++++---- tests/test_elements.py | 23 +++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 88a245d..7b6908c 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -175,7 +175,10 @@ def _diagram( ) diagram_svg = diagram.as_svg - file_name = f"{C2P_IMAGE_PREFIX}_diagram.svg" + if isinstance(diagram_svg, str): + diagram_svg = diagram_svg.encode("utf8") + + file_name = f"{C2P_IMAGE_PREFIX}diagram.svg" converter_data.work_item = data_models.CapellaWorkItem( type=converter_data.type_config.p_type, @@ -188,7 +191,7 @@ def _diagram( polarion_api.WorkItemAttachment( "", "", - "Context Diagram", + "Diagram", diagram_svg, "image/svg+xml", file_name, @@ -362,13 +365,18 @@ def _add_context_diagram( """Add a new custom field context diagram.""" assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.context_diagram - file_name = f"{C2P_IMAGE_PREFIX}_context_diagram.svg" + file_name = f"{C2P_IMAGE_PREFIX}context_diagram.svg" + + diagram_svg = diagram.as_svg + if isinstance(diagram_svg, str): + diagram_svg = diagram_svg.encode("utf8") + converter_data.work_item.attachments.append( polarion_api.WorkItemAttachment( "", "", "Context Diagram", - diagram.as_svg, + diagram_svg, "image/svg+xml", file_name, ) @@ -386,12 +394,17 @@ def _add_tree_diagram( assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.tree_view file_name = f"{C2P_IMAGE_PREFIX}_tree_view.svg" + + diagram_svg = diagram.as_svg + if isinstance(diagram_svg, str): + diagram_svg = diagram_svg.encode("utf8") + converter_data.work_item.attachments.append( polarion_api.WorkItemAttachment( "", "", "Tree View", - diagram.as_svg, + diagram_svg, "image/svg+xml", file_name, ) diff --git a/tests/test_elements.py b/tests/test_elements.py index 3bc8fbf..e7a17eb 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -1029,18 +1029,25 @@ def test_diagram(model: capellambse.MelodyModel): ) serialized_diagram = serializer.serialize(TEST_DIAG_UUID) - if serialized_diagram is not None: - serialized_diagram.description = None assert serialized_diagram == data_models.CapellaWorkItem( type="diagram", uuid_capella=TEST_DIAG_UUID, title="[CC] Capability", description_type="text/html", + description='

', status="open", linked_work_items=[], ) + attachment = serialized_diagram.attachments[0] + attachment.content_bytes = None + + assert attachment == polarion_api.WorkItemAttachment( + "", "", "Diagram", None, "image/svg+xml", "__C2P__diagram.svg" + ) + @staticmethod @pytest.mark.parametrize( "layer,uuid,expected", @@ -1246,6 +1253,18 @@ def test_add_context_diagram(self, model: capellambse.MelodyModel): work_item.additional_attributes["context_diagram"]["value"] ).startswith(TEST_DIAG_DESCR) + attachment = work_item.attachments[0] + attachment.content_bytes = None + + assert attachment == polarion_api.WorkItemAttachment( + "", + "", + "Context Diagram", + None, + "image/svg+xml", + "__C2P__context_diagram.svg", + ) + def test_multiple_serializers(self, model: capellambse.MelodyModel): cap = model.by_uuid(TEST_OCAP_UUID) type_config = converter_config.CapellaTypeConfig( From dd456d8e146ef8f7de55e785b9885cd9c5aec05e Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 2 Feb 2024 15:38:52 +0100 Subject: [PATCH 11/31] test: add tests for adding and updating attachments --- .../connectors/polarion_worker.py | 7 +- tests/test_elements.py | 1 - tests/test_workitem_attachments.py | 279 ++++++++++++++++++ 3 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 tests/test_workitem_attachments.py diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 5afebb0..a5d46fb 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -341,17 +341,18 @@ def update_attachments( attachment = new_attachment_dict[common_attachment_file_name] attachment.id = old_attachment_dict[common_attachment_file_name].id if ( - new_checksums[attachment.file_name] + new_checksums.get(attachment.file_name) != old_checksums.get(attachment.file_name) or self.force_update + or attachment.mime_type == "image/svg+xml" ): attachments_for_update[attachment.file_name] = attachment - for file_name, attachment in attachments_for_update: + for file_name, attachment in attachments_for_update.items(): # SVGs should only be updated if their PNG differs if ( attachment.mime_type == "image/svg+xml" - and file_name[:3] + "png" not in attachments_for_update + and file_name[:-3] + "png" not in attachments_for_update ): continue diff --git a/tests/test_elements.py b/tests/test_elements.py index e7a17eb..45acd17 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -4,7 +4,6 @@ from __future__ import annotations import logging -import pathlib import typing as t from unittest import mock diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py new file mode 100644 index 0000000..9a4f8c2 --- /dev/null +++ b/tests/test_workitem_attachments.py @@ -0,0 +1,279 @@ +# Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest import mock + +import capellambse +import polarion_rest_api_client as polarion_api +import pytest + +from capella2polarion import data_models +from capella2polarion.connectors import polarion_repo, polarion_worker +from capella2polarion.converters import ( + converter_config, + data_session, + model_converter, +) + +from .test_elements import TEST_DIAG_UUID + +DIAGRAM_WI_CHECKSUM = ( + "e03b0295aacbc8211eb6c9801afefa036f080f56a2785331732ad5e23fd4cf4f" +) +DIAGRAM_PNG_CHECKSUM = ( + "c6d7880529ae26da4f1643740e235a44e71099dfd5e646849026445d9fb5024b" +) +DIAGRAM_CHECKSUM = json.dumps( + { + "__C2P__WORK_ITEM": DIAGRAM_WI_CHECKSUM, + "__C2P__diagram.png": DIAGRAM_PNG_CHECKSUM, + } +) + + +@pytest.fixture +def worker(monkeypatch: pytest.MonkeyPatch): + mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) + monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + config = mock.Mock(converter_config.ConverterConfig) + worker = polarion_worker.CapellaPolarionWorker( + polarion_worker.PolarionWorkerParams( + "TEST", + "http://localhost", + "TESTPAT", + False, + ), + config, + ) + + return worker + + +def set_attachment_ids(attachments: list[polarion_api.WorkItemAttachment]): + counter = 0 + attachments = sorted(attachments, key=lambda a: a.file_name) + for attachment in attachments: + attachment.id = f"{counter}-{attachment.file_name}" + counter += 1 + + +def test_diagram_attachments_new( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [data_models.CapellaWorkItem("TEST-ID", uuid_capella=TEST_DIAG_UUID)] + ) + worker.client.create_work_item_attachments = mock.MagicMock() + worker.client.create_work_item_attachments.side_effect = set_attachment_ids + + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(TEST_DIAG_UUID, converter.converter_session) + + assert worker.client.update_work_item.call_count == 1 + assert worker.client.create_work_item_attachments.call_count == 1 + + created_attachments: list[ + polarion_api.WorkItemAttachment + ] = worker.client.create_work_item_attachments.call_args.args[0] + work_item: data_models.CapellaWorkItem = ( + worker.client.update_work_item.call_args.args[0] + ) + + assert len(created_attachments) == 2 + assert created_attachments[0].title == created_attachments[1].title + assert ( + created_attachments[0].file_name[:3] + == created_attachments[0].file_name[:3] + ) + + assert ( + work_item.description == '

' + "

" + ) + assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM + + +def test_diagram_attachments_updated( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [data_models.CapellaWorkItem("TEST-ID", uuid_capella=TEST_DIAG_UUID)] + ) + worker.client.get_all_work_item_attachments = mock.MagicMock() + worker.client.get_all_work_item_attachments.return_value = [ + polarion_api.WorkItemAttachment( + "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + ), + polarion_api.WorkItemAttachment( + "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + ), + ] + + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(TEST_DIAG_UUID, converter.converter_session) + + assert worker.client.update_work_item.call_count == 1 + assert worker.client.create_work_item_attachments.call_count == 0 + assert worker.client.update_work_item_attachment.call_count == 2 + + work_item: data_models.CapellaWorkItem = ( + worker.client.update_work_item.call_args.args[0] + ) + + assert ( + work_item.description == '

' + "

" + ) + + +def test_diagram_attachments_unchanged_work_item_changed( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [ + data_models.CapellaWorkItem( + "TEST-ID", + uuid_capella=TEST_DIAG_UUID, + checksum=json.dumps( + { + "__C2P__WORK_ITEM": "123", + "__C2P__diagram.png": DIAGRAM_PNG_CHECKSUM, + } + ), + ) + ] + ) + worker.client.get_all_work_item_attachments = mock.MagicMock() + worker.client.get_all_work_item_attachments.return_value = [ + polarion_api.WorkItemAttachment( + "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + ), + polarion_api.WorkItemAttachment( + "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + ), + ] + + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(TEST_DIAG_UUID, converter.converter_session) + + assert worker.client.update_work_item.call_count == 1 + assert worker.client.create_work_item_attachments.call_count == 0 + assert worker.client.update_work_item_attachment.call_count == 0 + + work_item: data_models.CapellaWorkItem = ( + worker.client.update_work_item.call_args.args[0] + ) + + assert ( + work_item.description == '

' + "

" + ) + + +def test_diagram_attachments_fully_unchanged( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [ + data_models.CapellaWorkItem( + "TEST-ID", + uuid_capella=TEST_DIAG_UUID, + checksum=DIAGRAM_CHECKSUM, + ) + ] + ) + + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(TEST_DIAG_UUID, converter.converter_session) + + assert worker.client.update_work_item.call_count == 0 + assert worker.client.create_work_item_attachments.call_count == 0 + assert worker.client.update_work_item_attachment.call_count == 0 + assert worker.client.get_all_work_item_attachments.call_count == 0 + + +def test_add_context_diagram( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + uuid = "11906f7b-3ae9-4343-b998-95b170be2e2b" + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [data_models.CapellaWorkItem("TEST-ID", uuid_capella=uuid)] + ) + + converter.converter_session[uuid] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("test", "add_context_diagram", []), + model.by_uuid(uuid), + ) + + worker.client.create_work_item_attachments = mock.MagicMock() + worker.client.create_work_item_attachments.side_effect = set_attachment_ids + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(uuid, converter.converter_session) + + assert worker.client.update_work_item.call_count == 1 + assert worker.client.create_work_item_attachments.call_count == 1 + + created_attachments: list[ + polarion_api.WorkItemAttachment + ] = worker.client.create_work_item_attachments.call_args.args[0] + work_item: data_models.CapellaWorkItem = ( + worker.client.update_work_item.call_args.args[0] + ) + + assert len(created_attachments) == 2 + assert created_attachments[0].title == created_attachments[1].title + assert ( + created_attachments[0].file_name[:3] + == created_attachments[0].file_name[:3] + ) + + assert ( + str(work_item.additional_attributes["context_diagram"]["value"]) + == '

' + ) From 6fc3fb7d5551b10dd9938d98bfde94c535433c13 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 2 Feb 2024 15:47:32 +0100 Subject: [PATCH 12/31] test: add tests for deleting attachments --- tests/test_workitem_attachments.py | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 9a4f8c2..3d9f603 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -277,3 +277,64 @@ def test_add_context_diagram( == '

' ) + + +def test_diagram_delete_attachments( + model: capellambse.MelodyModel, + worker: polarion_worker.CapellaPolarionWorker, +): + converter = model_converter.ModelConverter(model, "TEST") + worker.polarion_data_repo = polarion_repo.PolarionDataRepository( + [ + data_models.CapellaWorkItem( + "TEST-ID", + uuid_capella=TEST_DIAG_UUID, + checksum=json.dumps( + { + "__C2P__WORK_ITEM": DIAGRAM_WI_CHECKSUM, + "__C2P__diagram.png": DIAGRAM_PNG_CHECKSUM, + "delete_me.png": "123", + } + ), + ) + ] + ) + worker.client.get_all_work_item_attachments = mock.MagicMock() + worker.client.get_all_work_item_attachments.return_value = [ + polarion_api.WorkItemAttachment( + "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + ), + polarion_api.WorkItemAttachment( + "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + ), + polarion_api.WorkItemAttachment( + "TEST-ID", "SVG-DELETE", "test", file_name="delete_me.svg" + ), + polarion_api.WorkItemAttachment( + "TEST-ID", "PNG-DELETE", "test", file_name="delete_me.png" + ), + ] + + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + + converter.generate_work_items(worker.polarion_data_repo, False, True) + + worker.patch_work_item(TEST_DIAG_UUID, converter.converter_session) + + assert worker.client.update_work_item.call_count == 1 + assert worker.client.create_work_item_attachments.call_count == 0 + assert worker.client.update_work_item_attachment.call_count == 0 + assert worker.client.delete_work_item_attachment.call_count == 2 + + work_item: data_models.CapellaWorkItem = ( + worker.client.update_work_item.call_args.args[0] + ) + + assert work_item.description is None + assert work_item.additional_attributes == {} + assert work_item.title is None + assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM From ea59ea28d2e20abde70e324f73b8780082b296e7 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 5 Feb 2024 16:17:39 +0100 Subject: [PATCH 13/31] fix: Various minor bugs such as replacing attachment by workitemimg as attached images are using a different syntax in work items than in documents --- .../connectors/polarion_worker.py | 32 +++++++++++++++---- .../converters/element_converter.py | 4 +-- tests/test_elements.py | 4 +-- tests/test_workitem_attachments.py | 10 +++--- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index a5d46fb..dc79b75 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -195,7 +195,7 @@ def patch_work_item( ) except polarion_api.PolarionApiException as error: logger.error( - "Getting Attachments for WorkItem %r (%s %s) failed. %s", + "Updating attachments for WorkItem %r (%s %s) failed. %s", *log_args, error.args[0], ) @@ -241,8 +241,8 @@ def patch_work_item( ) else: new.additional_attributes = {} - del new.type - del new.status + new.type = None + new.status = None new.description = None new.description_type = None new.title = None @@ -281,13 +281,18 @@ def set_attachment_id(node: etree._Element) -> None: if node.tag != "img": return if img_src := node.attrib.get("src"): - if img_src.startswith("attachment:"): - file_name = img_src[11:] + if img_src.startswith("workitemimg:"): + file_name = img_src[12:] for attachment in new.attachments: if attachment.file_name == file_name: - node.attrib["src"] = f"attachment:{attachment.id}" + node.attrib["src"] = f"workitemimg:{attachment.id}" return + logger.error( + "Did not find attachment ID for file name %s", + file_name, + ) + if new.description: new.description = chelpers.process_html_fragments( new.description, set_attachment_id @@ -318,6 +323,21 @@ def update_attachments( attachment.file_name: attachment for attachment in old_attachments } + duplicate_attachments = [ + attachment + for attachment in old_attachments + if attachment not in old_attachment_dict.values() + ] + for attachment in duplicate_attachments: + logger.error( + "There are already multiple attachments named %s. " + "Attachment with ID %s will be deleted for that reason" + " - please report this as issue.", + attachment.file_name, + attachment.id, + ) + self.client.delete_work_item_attachment(attachment) + old_attachment_file_names = set(old_attachment_dict) new_attachment_file_names = set(new_attachment_dict) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 7b6908c..cf4bae5 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -112,7 +112,7 @@ def _generate_image_html(attachment_id: str) -> str: ) description = ( f'

' + f'src="workitemimg:{attachment_id}" />

' ) return description @@ -264,7 +264,7 @@ def repair_images(node: etree._Element) -> None: f"{file_name}.{file_path.suffix}", ) ) - node.attrib["src"] = f"attachment:{file_name}" + node.attrib["src"] = f"workitemimg:{file_name}" except FileNotFoundError: logger.error( diff --git a/tests/test_elements.py b/tests/test_elements.py index 45acd17..18ce3f6 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -53,7 +53,7 @@ TEST_SCENARIO = "afdaa095-e2cd-4230-b5d3-6cb771a90f51" TEST_CAP_REAL = "b80b3141-a7fc-48c7-84b2-1467dcef5fce" TEST_CONSTRAINT = "95cbd4af-7224-43fe-98cb-f13dda540b8e" -TEST_DIAG_DESCR = '

', + 'src="workitemimg:__C2P__diagram.svg" />

', status="open", linked_work_items=[], ) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 3d9f603..bb4c12f 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -19,7 +19,7 @@ from .test_elements import TEST_DIAG_UUID DIAGRAM_WI_CHECKSUM = ( - "e03b0295aacbc8211eb6c9801afefa036f080f56a2785331732ad5e23fd4cf4f" + "37121e4c32bfae03ab387051f676f976de3b5b8b92c22351d906534ddf0a3ee8" ) DIAGRAM_PNG_CHECKSUM = ( "c6d7880529ae26da4f1643740e235a44e71099dfd5e646849026445d9fb5024b" @@ -98,7 +98,7 @@ def test_diagram_attachments_new( assert ( work_item.description == '

' + 'src="workitemimg:1-__C2P__diagram.svg"/>' "

" ) assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM @@ -142,7 +142,7 @@ def test_diagram_attachments_updated( assert ( work_item.description == '

' + 'src="workitemimg:SVG-ATTACHMENT"/>' "

" ) @@ -196,7 +196,7 @@ def test_diagram_attachments_unchanged_work_item_changed( assert ( work_item.description == '

' + 'src="workitemimg:SVG-ATTACHMENT"/>' "

" ) @@ -275,7 +275,7 @@ def test_add_context_diagram( assert ( str(work_item.additional_attributes["context_diagram"]["value"]) == '

' + 'src="workitemimg:1-__C2P__context_diagram.svg"/>

' ) From c32053222560dfd00c76eaa6d9854ff0d1a50926 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 5 Feb 2024 17:01:44 +0100 Subject: [PATCH 14/31] fix: minor change for pylint --- capella2polarion/data_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/capella2polarion/data_models.py b/capella2polarion/data_models.py index ca9c0f0..55f8ab3 100644 --- a/capella2polarion/data_models.py +++ b/capella2polarion/data_models.py @@ -54,6 +54,7 @@ def calculate_checksum(self) -> str: data = dict(sorted(data.items())) converted = json.dumps(data).encode("utf8") + # pylint: disable-next=attribute-defined-outside-init self._checksum = json.dumps( {"__C2P__WORK_ITEM": hashlib.sha256(converted).hexdigest()} | dict(sorted(attachment_checksums.items())) From d4860f242c70d98f8892c92295af0d002cdf5740 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 5 Feb 2024 18:23:45 +0100 Subject: [PATCH 15/31] fix: tests --- tests/test_workitem_attachments.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index bb4c12f..be160f3 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -1,9 +1,11 @@ # Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 - +import base64 +import hashlib import json from unittest import mock +import cairosvg import capellambse import polarion_rest_api_client as polarion_api import pytest @@ -16,14 +18,32 @@ model_converter, ) -from .test_elements import TEST_DIAG_UUID +from .conftest import TEST_DIAGRAM_CACHE DIAGRAM_WI_CHECKSUM = ( "37121e4c32bfae03ab387051f676f976de3b5b8b92c22351d906534ddf0a3ee8" ) -DIAGRAM_PNG_CHECKSUM = ( - "c6d7880529ae26da4f1643740e235a44e71099dfd5e646849026445d9fb5024b" -) + +TEST_DIAG_UUID = "_APMboAPhEeynfbzU12yy7w" + +with open( + TEST_DIAGRAM_CACHE / "_APMboAPhEeynfbzU12yy7w.svg", "r", encoding="utf8" +) as f: + diagram_svg = f.read() + +wia_dict = { + "work_item_id": "", + "title": "Diagram", + "content_bytes": base64.b64encode(cairosvg.svg2png(diagram_svg)).decode( + "utf8" + ), + "mime_type": "image/png", + "file_name": "__C2P__diagram.png", +} + +DIAGRAM_PNG_CHECKSUM = hashlib.sha256( + json.dumps(wia_dict).encode("utf8") +).hexdigest() DIAGRAM_CHECKSUM = json.dumps( { "__C2P__WORK_ITEM": DIAGRAM_WI_CHECKSUM, From 44f41a88326ad82f8fb346d9063c6b24a3700646 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 5 Feb 2024 18:29:22 +0100 Subject: [PATCH 16/31] fix: black --- capella2polarion/converters/element_converter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index cf4bae5..51854ba 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -206,8 +206,7 @@ def _generic_work_item( obj = converter_data.capella_element raw_description = getattr(obj, "description", None) uuids, value, attachments = self._sanitize_description( - obj, raw_description - or markupsafe.Markup("") + obj, raw_description or markupsafe.Markup("") ) converter_data.description_references = uuids requirement_types = _get_requirement_types_text(obj) @@ -353,9 +352,10 @@ def _linked_text_as_description( """Return attributes for a ``Constraint``.""" # pylint: disable-next=attribute-defined-outside-init assert converter_data.work_item, "No work item set yet" - converter_data.work_item.description, attachments = self._get_linked_text( - converter_data - ) + ( + converter_data.work_item.description, + attachments, + ) = self._get_linked_text(converter_data) converter_data.work_item.attachments += attachments return converter_data.work_item From 6e8b63886054620eef51c02268e8860a453849d2 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 08:21:47 +0100 Subject: [PATCH 17/31] refactor: Move diagram drawing to a common method --- .../converters/element_converter.py | 111 ++++++++---------- 1 file changed, 52 insertions(+), 59 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 51854ba..7b7e433 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -117,6 +117,46 @@ def _generate_image_html(attachment_id: str) -> str: return description +def _draw_diagram( + diagram: capellambse.model.diagram.Diagram, file_name: str, title: str +) -> tuple[str, polarion_api.WorkItemAttachment]: + file_name = f"{C2P_IMAGE_PREFIX}{file_name}.svg" + + try: + diagram_svg = diagram.render("svg") + except Exception as error: + logger.exception("Failed to get diagram from cache. Error: %s", error) + diagram_svg = diagram.as_svg + + if isinstance(diagram_svg, str): + diagram_svg = diagram_svg.encode("utf8") + + attachment = polarion_api.WorkItemAttachment( + "", + "", + title, + diagram_svg, + "image/svg+xml", + file_name, + ) + + return _generate_image_html(file_name), attachment + + +def _draw_additional_attributes_diagram( + work_item: data_models.CapellaWorkItem, + diagram: capellambse.model.diagram.Diagram, + attribute: str, + title: str, +): + diagram_html, attachment = _draw_diagram(diagram, attribute, title) + work_item.attachments.append(attachment) + work_item.additional_attributes[attribute] = { + "type": "text/html", + "value": diagram_html, + } + + class CapellaWorkItemSerializer: """The general serializer class for CapellaWorkItems.""" @@ -167,36 +207,16 @@ def _diagram( """Serialize a diagram for Polarion.""" diagram = converter_data.capella_element - try: - diagram_svg = diagram.render("svg") - except Exception as error: - logger.exception( - "Failed to get diagram from cache. Error: %s", error - ) - diagram_svg = diagram.as_svg - - if isinstance(diagram_svg, str): - diagram_svg = diagram_svg.encode("utf8") - - file_name = f"{C2P_IMAGE_PREFIX}diagram.svg" + diagram_html, attachment = _draw_diagram(diagram, "diagram", "Diagram") converter_data.work_item = data_models.CapellaWorkItem( type=converter_data.type_config.p_type, title=diagram.name, description_type="text/html", - description=_generate_image_html(file_name), + description=diagram_html, status="open", uuid_capella=diagram.uuid, - attachments=[ - polarion_api.WorkItemAttachment( - "", - "", - "Diagram", - diagram_svg, - "image/svg+xml", - file_name, - ) - ], + attachments=[attachment], ) return converter_data.work_item @@ -365,26 +385,14 @@ def _add_context_diagram( """Add a new custom field context diagram.""" assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.context_diagram - file_name = f"{C2P_IMAGE_PREFIX}context_diagram.svg" - diagram_svg = diagram.as_svg - if isinstance(diagram_svg, str): - diagram_svg = diagram_svg.encode("utf8") - - converter_data.work_item.attachments.append( - polarion_api.WorkItemAttachment( - "", - "", - "Context Diagram", - diagram_svg, - "image/svg+xml", - file_name, - ) + _draw_additional_attributes_diagram( + converter_data.work_item, + diagram, + "context_diagram", + "Context Diagram", ) - converter_data.work_item.additional_attributes["context_diagram"] = { - "type": "text/html", - "value": _generate_image_html(file_name), - } + return converter_data.work_item def _add_tree_diagram( @@ -393,24 +401,9 @@ def _add_tree_diagram( """Add a new custom field tree diagram.""" assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.tree_view - file_name = f"{C2P_IMAGE_PREFIX}_tree_view.svg" - diagram_svg = diagram.as_svg - if isinstance(diagram_svg, str): - diagram_svg = diagram_svg.encode("utf8") - - converter_data.work_item.attachments.append( - polarion_api.WorkItemAttachment( - "", - "", - "Tree View", - diagram_svg, - "image/svg+xml", - file_name, - ) + _draw_additional_attributes_diagram( + converter_data.work_item, diagram, "tree_view", "Tree View" ) - converter_data.work_item.additional_attributes["tree_view"] = { - "type": "text/html", - "value": _generate_image_html(file_name), - } + return converter_data.work_item From d7085d828aeccdf5e318e860cb120918d43a366a Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 08:32:48 +0100 Subject: [PATCH 18/31] fix: windows tests --- .github/workflows/build-test-publish.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 8c4bf81..65bf1e8 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -30,6 +30,17 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{matrix.python_version}} + - name: Download GTK Runtime Installer (Windows only) + if: matrix.os == 'windows-latest' + run: | + choco install wget -y + wget https://github.com/tschoonj/GTK-for-Windows-Runtime-Environment-Installer/releases/download/2022-01-04/gtk3-runtime-3.24.31-2022-01-04-ts-win64.exe + - name: Install GTK Runtime (Windows only) + if: matrix.os == 'windows-latest' + shell: powershell + run: | + Start-Process -FilePath "gtk3-runtime-3.24.31-2022-01-04-ts-win64.exe" -ArgumentList "/SILENT /LOG=install.log" -Wait -NoNewWindow + Get-Content "install.log" - uses: actions/cache@v3 with: path: ~/.cache/pip From 9219a7e9259451aaa561fda225fbcf1bd9c9afd1 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 09:40:20 +0100 Subject: [PATCH 19/31] feat: increase png quality to 400dpi --- capella2polarion/converters/model_converter.py | 2 +- tests/test_workitem_attachments.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index 0ce4299..9799685 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -115,7 +115,7 @@ def generate_pngs_for_svgs(self): attachment.work_item_id, "", attachment.title, - cairosvg.svg2png(attachment.content_bytes), + cairosvg.svg2png(attachment.content_bytes, dpi=400), "image/png", attachment.file_name[:-3] + "png", ) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index be160f3..892f1f4 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -34,9 +34,9 @@ wia_dict = { "work_item_id": "", "title": "Diagram", - "content_bytes": base64.b64encode(cairosvg.svg2png(diagram_svg)).decode( - "utf8" - ), + "content_bytes": base64.b64encode( + cairosvg.svg2png(diagram_svg, dpi=400) + ).decode("utf8"), "mime_type": "image/png", "file_name": "__C2P__diagram.png", } From 5e4296c85afa33d03933a6beb997706baa75cd40 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 10:25:20 +0100 Subject: [PATCH 20/31] fix: add missing workitem id --- capella2polarion/converters/element_converter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 7b7e433..0fb34c8 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -199,6 +199,8 @@ def serialize(self, uuid: str) -> data_models.CapellaWorkItem | None: old = self.capella_polarion_mapping.get_work_item_by_capella_uuid(uuid) if old: converter_data.work_item.id = old.id + for attachment in converter_data.work_item.attachments: + attachment.work_item_id = old.id return converter_data.work_item def _diagram( From 18801e7730d688c3df997808c3e1eaa0a3501e82 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 11:47:53 +0100 Subject: [PATCH 21/31] fix: context diagrams png conversion does not work properly using custom DPI values, so we reset it to default --- .../converters/model_converter.py | 2 +- tests/test_workitem_attachments.py | 55 +++++++++++++------ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index 9799685..0ce4299 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -115,7 +115,7 @@ def generate_pngs_for_svgs(self): attachment.work_item_id, "", attachment.title, - cairosvg.svg2png(attachment.content_bytes, dpi=400), + cairosvg.svg2png(attachment.content_bytes), "image/png", attachment.file_name[:-3] + "png", ) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 892f1f4..165260c 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -25,6 +25,7 @@ ) TEST_DIAG_UUID = "_APMboAPhEeynfbzU12yy7w" +WORKITEM_ID = "TEST-ID" with open( TEST_DIAGRAM_CACHE / "_APMboAPhEeynfbzU12yy7w.svg", "r", encoding="utf8" @@ -32,11 +33,11 @@ diagram_svg = f.read() wia_dict = { - "work_item_id": "", + "work_item_id": WORKITEM_ID, "title": "Diagram", - "content_bytes": base64.b64encode( - cairosvg.svg2png(diagram_svg, dpi=400) - ).decode("utf8"), + "content_bytes": base64.b64encode(cairosvg.svg2png(diagram_svg)).decode( + "utf8" + ), "mime_type": "image/png", "file_name": "__C2P__diagram.png", } @@ -84,7 +85,7 @@ def test_diagram_attachments_new( ): converter = model_converter.ModelConverter(model, "TEST") worker.polarion_data_repo = polarion_repo.PolarionDataRepository( - [data_models.CapellaWorkItem("TEST-ID", uuid_capella=TEST_DIAG_UUID)] + [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID)] ) worker.client.create_work_item_attachments = mock.MagicMock() worker.client.create_work_item_attachments.side_effect = set_attachment_ids @@ -130,15 +131,21 @@ def test_diagram_attachments_updated( ): converter = model_converter.ModelConverter(model, "TEST") worker.polarion_data_repo = polarion_repo.PolarionDataRepository( - [data_models.CapellaWorkItem("TEST-ID", uuid_capella=TEST_DIAG_UUID)] + [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID)] ) worker.client.get_all_work_item_attachments = mock.MagicMock() worker.client.get_all_work_item_attachments.return_value = [ polarion_api.WorkItemAttachment( - "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + WORKITEM_ID, + "SVG-ATTACHMENT", + "test", + file_name="__C2P__diagram.svg", ), polarion_api.WorkItemAttachment( - "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + WORKITEM_ID, + "PNG-ATTACHMENT", + "test", + file_name="__C2P__diagram.png", ), ] @@ -175,7 +182,7 @@ def test_diagram_attachments_unchanged_work_item_changed( worker.polarion_data_repo = polarion_repo.PolarionDataRepository( [ data_models.CapellaWorkItem( - "TEST-ID", + WORKITEM_ID, uuid_capella=TEST_DIAG_UUID, checksum=json.dumps( { @@ -189,10 +196,16 @@ def test_diagram_attachments_unchanged_work_item_changed( worker.client.get_all_work_item_attachments = mock.MagicMock() worker.client.get_all_work_item_attachments.return_value = [ polarion_api.WorkItemAttachment( - "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + WORKITEM_ID, + "SVG-ATTACHMENT", + "test", + file_name="__C2P__diagram.svg", ), polarion_api.WorkItemAttachment( - "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + WORKITEM_ID, + "PNG-ATTACHMENT", + "test", + file_name="__C2P__diagram.png", ), ] @@ -229,7 +242,7 @@ def test_diagram_attachments_fully_unchanged( worker.polarion_data_repo = polarion_repo.PolarionDataRepository( [ data_models.CapellaWorkItem( - "TEST-ID", + WORKITEM_ID, uuid_capella=TEST_DIAG_UUID, checksum=DIAGRAM_CHECKSUM, ) @@ -259,7 +272,7 @@ def test_add_context_diagram( uuid = "11906f7b-3ae9-4343-b998-95b170be2e2b" converter = model_converter.ModelConverter(model, "TEST") worker.polarion_data_repo = polarion_repo.PolarionDataRepository( - [data_models.CapellaWorkItem("TEST-ID", uuid_capella=uuid)] + [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=uuid)] ) converter.converter_session[uuid] = data_session.ConverterData( @@ -307,7 +320,7 @@ def test_diagram_delete_attachments( worker.polarion_data_repo = polarion_repo.PolarionDataRepository( [ data_models.CapellaWorkItem( - "TEST-ID", + WORKITEM_ID, uuid_capella=TEST_DIAG_UUID, checksum=json.dumps( { @@ -322,16 +335,22 @@ def test_diagram_delete_attachments( worker.client.get_all_work_item_attachments = mock.MagicMock() worker.client.get_all_work_item_attachments.return_value = [ polarion_api.WorkItemAttachment( - "TEST-ID", "SVG-ATTACHMENT", "test", file_name="__C2P__diagram.svg" + WORKITEM_ID, + "SVG-ATTACHMENT", + "test", + file_name="__C2P__diagram.svg", ), polarion_api.WorkItemAttachment( - "TEST-ID", "PNG-ATTACHMENT", "test", file_name="__C2P__diagram.png" + WORKITEM_ID, + "PNG-ATTACHMENT", + "test", + file_name="__C2P__diagram.png", ), polarion_api.WorkItemAttachment( - "TEST-ID", "SVG-DELETE", "test", file_name="delete_me.svg" + WORKITEM_ID, "SVG-DELETE", "test", file_name="delete_me.svg" ), polarion_api.WorkItemAttachment( - "TEST-ID", "PNG-DELETE", "test", file_name="delete_me.png" + WORKITEM_ID, "PNG-DELETE", "test", file_name="delete_me.png" ), ] From fa06d08e8121d0325dccf7fa786597439973c53c Mon Sep 17 00:00:00 2001 From: ewuerger Date: Tue, 6 Feb 2024 12:32:13 +0100 Subject: [PATCH 22/31] refactor: Apply changes from code review --- capella2polarion/__main__.py | 4 +- .../connectors/polarion_worker.py | 50 +++++++++---------- .../converters/element_converter.py | 2 +- .../converters/model_converter.py | 21 ++++++-- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 1d6550f..8a2ce63 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -111,7 +111,9 @@ def synchronize(ctx: click.core.Context) -> None: # Create missing links for new work items converter.generate_work_items( - polarion_worker.polarion_data_repo, True, True + polarion_worker.polarion_data_repo, + generate_links=True, + generate_attachments=True, ) polarion_worker.patch_work_items(converter.converter_session) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index dc79b75..9c5d629 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -106,8 +106,8 @@ def delete_work_items( ) -> None: """Delete work items in a Polarion project. - If the delete flag is set to ``False`` in the context work items are - marked as ``to be deleted`` via the status attribute. + If the delete flag is set to ``False`` in the context work items + are marked as ``to be deleted`` via the status attribute. """ def serialize_for_delete(uuid: str) -> str: @@ -172,9 +172,11 @@ def patch_work_item( return log_args = (old.id, new.type, new.title) - logger.info("Update work item %r for model %s %r...", *log_args) + logger.info( + "Update work item %r for model element %s %r...", *log_args + ) - if old.get_current_checksum()[0] != "{": + if old.get_current_checksum()[0] != "{": # XXX: Remove in next release old_checksums = {"__C2P__WORK_ITEM": old.get_current_checksum()} else: old_checksums = json.loads(old.get_current_checksum()) @@ -297,12 +299,15 @@ def set_attachment_id(node: etree._Element) -> None: new.description = chelpers.process_html_fragments( new.description, set_attachment_id ) - for _, value in new.additional_attributes.items(): - if isinstance(value, dict): - if value.get("type") == "text/html" and value.get("value"): - value["value"] = chelpers.process_html_fragments( - value["value"], set_attachment_id - ) + for _, attributes in new.additional_attributes.items(): + if ( + isinstance(attributes, dict) + and attributes.get("type") == "text/html" + and attributes.get("value") is not None + ): + attributes["value"] = chelpers.process_html_fragments( + attributes["value"], set_attachment_id + ) def update_attachments( self, @@ -323,24 +328,19 @@ def update_attachments( attachment.file_name: attachment for attachment in old_attachments } - duplicate_attachments = [ - attachment - for attachment in old_attachments - if attachment not in old_attachment_dict.values() - ] - for attachment in duplicate_attachments: - logger.error( - "There are already multiple attachments named %s. " - "Attachment with ID %s will be deleted for that reason" - " - please report this as issue.", - attachment.file_name, - attachment.id, - ) - self.client.delete_work_item_attachment(attachment) + for attachment in old_attachments: + if attachment not in old_attachment_dict.values(): + logger.error( + "There are already multiple attachments named %s. " + "Attachment with ID %s will be deleted for that reason" + " - please report this as issue.", + attachment.file_name, + attachment.id, + ) + self.client.delete_work_item_attachment(attachment) old_attachment_file_names = set(old_attachment_dict) new_attachment_file_names = set(new_attachment_dict) - for file_name in old_attachment_file_names - new_attachment_file_names: self.client.delete_work_item_attachment( old_attachment_dict[file_name] diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 0fb34c8..90b2154 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -32,7 +32,7 @@ f"" ) RE_CAMEL_CASE_2ND_WORD_PATTERN = re.compile(r"([a-z]+)([A-Z][a-z]+)") -DIAGRAM_STYLES = {"max-width": "100%"} +DIAGRAM_STYLES = {"max-width": "100px"} POLARION_WORK_ITEM_URL = ( '' diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index 9799685..62f77fb 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -55,9 +55,9 @@ def read_model( if type_config := config.get_type_config( layer, c_type, **attributes ): - self.converter_session[ - obj.uuid - ] = data_session.ConverterData(layer, type_config, obj) + self.converter_session[obj.uuid] = ( + data_session.ConverterData(layer, type_config, obj) + ) else: missing_types.add((layer, c_type, attributes)) @@ -82,13 +82,24 @@ def generate_work_items( self, polarion_data_repo: polarion_repo.PolarionDataRepository, generate_links: bool = False, - pngs_for_svgs: bool = False, + generate_attachments: bool = False, ) -> dict[str, data_models.CapellaWorkItem]: """Return a work items mapping from model elements for Polarion. The dictionary maps Capella UUIDs to ``CapellaWorkItem`` s. In addition, it is ensured that neither title nor type are None, Links are not created in this step by default. + + Parameters + ---------- + polarion_data_repo + The PolarionDataRepository object storing current work item + data. + generate_links + A boolean flag to control linked work item generation. + generate_attachments + A boolean flag to control attachments generation. For SVG + attachments, PNGs are generated and attached automatically. """ serializer = element_converter.CapellaWorkItemSerializer( self.model, polarion_data_repo, self.converter_session @@ -101,7 +112,7 @@ def generate_work_items( if generate_links: self.generate_work_item_links(polarion_data_repo) - if pngs_for_svgs: + if generate_attachments: self.generate_pngs_for_svgs() return {wi.uuid_capella: wi for wi in work_items} From 8e159902cfdb0677f9d3afcededc52f40e079edf Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 16:26:36 +0100 Subject: [PATCH 23/31] refactor: Some optimizations for attachment creation --- .../converters/element_converter.py | 143 +++++++++++------- .../converters/model_converter.py | 32 +--- tests/test_elements.py | 6 +- 3 files changed, 103 insertions(+), 78 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 90b2154..939dba4 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -12,6 +12,7 @@ import typing as t from collections import abc as cabc +import cairosvg import capellambse import markupsafe import polarion_rest_api_client as polarion_api @@ -32,7 +33,7 @@ f"" ) RE_CAMEL_CASE_2ND_WORD_PATTERN = re.compile(r"([a-z]+)([A-Z][a-z]+)") -DIAGRAM_STYLES = {"max-width": "100px"} +DIAGRAM_STYLES = {"max-width": "100%"} POLARION_WORK_ITEM_URL = ( '' @@ -117,46 +118,6 @@ def _generate_image_html(attachment_id: str) -> str: return description -def _draw_diagram( - diagram: capellambse.model.diagram.Diagram, file_name: str, title: str -) -> tuple[str, polarion_api.WorkItemAttachment]: - file_name = f"{C2P_IMAGE_PREFIX}{file_name}.svg" - - try: - diagram_svg = diagram.render("svg") - except Exception as error: - logger.exception("Failed to get diagram from cache. Error: %s", error) - diagram_svg = diagram.as_svg - - if isinstance(diagram_svg, str): - diagram_svg = diagram_svg.encode("utf8") - - attachment = polarion_api.WorkItemAttachment( - "", - "", - title, - diagram_svg, - "image/svg+xml", - file_name, - ) - - return _generate_image_html(file_name), attachment - - -def _draw_additional_attributes_diagram( - work_item: data_models.CapellaWorkItem, - diagram: capellambse.model.diagram.Diagram, - attribute: str, - title: str, -): - diagram_html, attachment = _draw_diagram(diagram, attribute, title) - work_item.attachments.append(attachment) - work_item.additional_attributes[attribute] = { - "type": "text/html", - "value": diagram_html, - } - - class CapellaWorkItemSerializer: """The general serializer class for CapellaWorkItems.""" @@ -168,10 +129,12 @@ def __init__( model: capellambse.MelodyModel, capella_polarion_mapping: polarion_repo.PolarionDataRepository, converter_session: data_session.ConverterSession, + generate_attachments: bool, ): self.model = model self.capella_polarion_mapping = capella_polarion_mapping self.converter_session = converter_session + self.generate_attachments = generate_attachments def serialize_all(self) -> list[data_models.CapellaWorkItem]: """Serialize all items of the converter_session.""" @@ -182,6 +145,10 @@ def serialize(self, uuid: str) -> data_models.CapellaWorkItem | None: """Return a CapellaWorkItem for the given diagram or element.""" converter_data = self.converter_session[uuid] self._generic_work_item(converter_data) + old = self.capella_polarion_mapping.get_work_item_by_capella_uuid(uuid) + if old: + assert converter_data.work_item is not None + converter_data.work_item.id = old.id for converter in converter_data.type_config.converters or []: try: @@ -196,30 +163,100 @@ def serialize(self, uuid: str) -> data_models.CapellaWorkItem | None: converter_data.work_item = None return None # Force to not overwrite on failure assert converter_data.work_item is not None - old = self.capella_polarion_mapping.get_work_item_by_capella_uuid(uuid) - if old: - converter_data.work_item.id = old.id - for attachment in converter_data.work_item.attachments: - attachment.work_item_id = old.id + return converter_data.work_item + def _add_attachment( + self, + work_item: data_models.CapellaWorkItem, + attachment: polarion_api.WorkItemAttachment, + ): + work_item.attachments.append(attachment) + if attachment.mime_type == "image/svg+xml": + work_item.attachments.append( + polarion_api.WorkItemAttachment( + work_item.id, + "", + attachment.title, + cairosvg.svg2png(attachment.content_bytes), + "image/png", + attachment.file_name[:-3] + "png", + ) + ) + + def _draw_diagram_svg( + self, + diagram: capellambse.model.diagram.Diagram, + file_name: str, + title: str, + ) -> tuple[str, polarion_api.WorkItemAttachment | None]: + file_name = f"{C2P_IMAGE_PREFIX}{file_name}.svg" + + if self.generate_attachments: + try: + diagram_svg = diagram.render("svg") + except Exception as error: + logger.exception( + "Failed to get diagram from cache. Error: %s", error + ) + diagram_svg = diagram.as_svg + + if isinstance(diagram_svg, str): + diagram_svg = diagram_svg.encode("utf8") + + attachment = polarion_api.WorkItemAttachment( + "", + "", + title, + diagram_svg, + "image/svg+xml", + file_name, + ) + else: + attachment = None + + return _generate_image_html(file_name), attachment + + def _draw_additional_attributes_diagram( + self, + work_item: data_models.CapellaWorkItem, + diagram: capellambse.model.diagram.Diagram, + attribute: str, + title: str, + ): + diagram_html, attachment = self._draw_diagram_svg( + diagram, attribute, title + ) + if attachment: + self._add_attachment(work_item, attachment) + work_item.additional_attributes[attribute] = { + "type": "text/html", + "value": diagram_html, + } + def _diagram( self, converter_data: data_session.ConverterData ) -> data_models.CapellaWorkItem: """Serialize a diagram for Polarion.""" diagram = converter_data.capella_element + assert converter_data.work_item is not None + work_item_id = converter_data.work_item.id - diagram_html, attachment = _draw_diagram(diagram, "diagram", "Diagram") + diagram_html, attachment = self._draw_diagram_svg( + diagram, "diagram", "Diagram" + ) converter_data.work_item = data_models.CapellaWorkItem( + id=work_item_id, type=converter_data.type_config.p_type, title=diagram.name, description_type="text/html", description=diagram_html, status="open", uuid_capella=diagram.uuid, - attachments=[attachment], ) + if attachment: + self._add_attachment(converter_data.work_item, attachment) return converter_data.work_item def _generic_work_item( @@ -239,9 +276,11 @@ def _generic_work_item( description=value, status="open", uuid_capella=obj.uuid, - attachments=attachments, **requirement_types, ) + for attachment in attachments: + self._add_attachment(converter_data.work_item, attachment) + return converter_data.work_item def _sanitize_description( @@ -258,7 +297,7 @@ def _sanitize_description( attachments: list[polarion_api.WorkItemAttachment] = [] def repair_images(node: etree._Element) -> None: - if node.tag != "img": + if node.tag != "img" or not self.generate_attachments: return file_url = pathlib.PurePosixPath(node.get("src")) @@ -388,7 +427,7 @@ def _add_context_diagram( assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.context_diagram - _draw_additional_attributes_diagram( + self._draw_additional_attributes_diagram( converter_data.work_item, diagram, "context_diagram", @@ -404,7 +443,7 @@ def _add_tree_diagram( assert converter_data.work_item, "No work item set yet" diagram = converter_data.capella_element.tree_view - _draw_additional_attributes_diagram( + self._draw_additional_attributes_diagram( converter_data.work_item, diagram, "tree_view", "Tree View" ) diff --git a/capella2polarion/converters/model_converter.py b/capella2polarion/converters/model_converter.py index 2e72fb9..a1548f9 100644 --- a/capella2polarion/converters/model_converter.py +++ b/capella2polarion/converters/model_converter.py @@ -7,7 +7,6 @@ import logging import typing as t -import cairosvg import capellambse import polarion_rest_api_client as polarion_api @@ -55,9 +54,9 @@ def read_model( if type_config := config.get_type_config( layer, c_type, **attributes ): - self.converter_session[obj.uuid] = ( - data_session.ConverterData(layer, type_config, obj) - ) + self.converter_session[ + obj.uuid + ] = data_session.ConverterData(layer, type_config, obj) else: missing_types.add((layer, c_type, attributes)) @@ -102,7 +101,10 @@ def generate_work_items( attachments, PNGs are generated and attached automatically. """ serializer = element_converter.CapellaWorkItemSerializer( - self.model, polarion_data_repo, self.converter_session + self.model, + polarion_data_repo, + self.converter_session, + generate_attachments, ) work_items = serializer.serialize_all() for work_item in work_items: @@ -112,28 +114,8 @@ def generate_work_items( if generate_links: self.generate_work_item_links(polarion_data_repo) - if generate_attachments: - self.generate_pngs_for_svgs() - return {wi.uuid_capella: wi for wi in work_items} - def generate_pngs_for_svgs(self): - """Generate PNG files for all SVGs for all work items.""" - for converter_data in self.converter_session.values(): - if converter_data.work_item is not None: - converter_data.work_item.attachments += [ - polarion_api.WorkItemAttachment( - attachment.work_item_id, - "", - attachment.title, - cairosvg.svg2png(attachment.content_bytes), - "image/png", - attachment.file_name[:-3] + "png", - ) - for attachment in converter_data.work_item.attachments - if attachment.mime_type == "image/svg+xml" - ] - def generate_work_item_links( self, polarion_data_repo: polarion_repo.PolarionDataRepository ): diff --git a/tests/test_elements.py b/tests/test_elements.py index 18ce3f6..3ae59ce 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -197,7 +197,7 @@ def test_create_diagrams(base_object: BaseObjectContainer): pw = base_object.pw new_work_items: dict[str, data_models.CapellaWorkItem] new_work_items = base_object.mc.generate_work_items( - pw.polarion_data_repo + pw.polarion_data_repo, generate_attachments=True ) assert len(new_work_items) == 1 work_item = new_work_items[TEST_DIAG_UUID] @@ -1025,6 +1025,7 @@ def test_diagram(model: capellambse.MelodyModel): "", DIAGRAM_CONFIG, diag ) }, + True, ) serialized_diagram = serializer.serialize(TEST_DIAG_UUID) @@ -1217,6 +1218,7 @@ def test_generic_work_item( obj, ) }, + False, ) work_item = serializer.serialize(uuid) @@ -1242,6 +1244,7 @@ def test_add_context_diagram(self, model: capellambse.MelodyModel): model.by_uuid(uuid), ) }, + True, ) work_item = serializer.serialize(uuid) @@ -1279,6 +1282,7 @@ def test_multiple_serializers(self, model: capellambse.MelodyModel): "pa", type_config, cap ) }, + True, ) work_item = serializer.serialize(TEST_OCAP_UUID) From 8e063b98ddf5a1d0860584a4b9c0c0624b4a02c8 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 16:30:08 +0100 Subject: [PATCH 24/31] fix: remove windows pipeline --- .github/workflows/build-test-publish.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/build-test-publish.yml b/.github/workflows/build-test-publish.yml index 65bf1e8..19e9679 100644 --- a/.github/workflows/build-test-publish.yml +++ b/.github/workflows/build-test-publish.yml @@ -21,26 +21,12 @@ jobs: - "3.10" - "3.11" - "3.12" - include: - - os: windows-latest - python_version: "3.12" steps: - uses: actions/checkout@v3 - name: Set up Python ${{matrix.python_version}} uses: actions/setup-python@v3 with: python-version: ${{matrix.python_version}} - - name: Download GTK Runtime Installer (Windows only) - if: matrix.os == 'windows-latest' - run: | - choco install wget -y - wget https://github.com/tschoonj/GTK-for-Windows-Runtime-Environment-Installer/releases/download/2022-01-04/gtk3-runtime-3.24.31-2022-01-04-ts-win64.exe - - name: Install GTK Runtime (Windows only) - if: matrix.os == 'windows-latest' - shell: powershell - run: | - Start-Process -FilePath "gtk3-runtime-3.24.31-2022-01-04-ts-win64.exe" -ArgumentList "/SILENT /LOG=install.log" -Wait -NoNewWindow - Get-Content "install.log" - uses: actions/cache@v3 with: path: ~/.cache/pip From f57d675db26c38fc1712c8a9f01ed339c582e01e Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 18:11:17 +0100 Subject: [PATCH 25/31] fix: tests --- tests/test_elements.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_elements.py b/tests/test_elements.py index 3ae59ce..c409adb 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -204,6 +204,7 @@ def test_create_diagrams(base_object: BaseObjectContainer): assert isinstance(work_item, data_models.CapellaWorkItem) description = work_item.description work_item.description = None + work_item.attachments = [] assert work_item == data_models.CapellaWorkItem(**TEST_SER_DIAGRAM) assert isinstance(description, str) assert description.startswith(TEST_DIAG_DESCR) @@ -1030,6 +1031,17 @@ def test_diagram(model: capellambse.MelodyModel): serialized_diagram = serializer.serialize(TEST_DIAG_UUID) + assert serialized_diagram is not None + + attachment = serialized_diagram.attachments[0] + attachment.content_bytes = None + + assert attachment == polarion_api.WorkItemAttachment( + "", "", "Diagram", None, "image/svg+xml", "__C2P__diagram.svg" + ) + + serialized_diagram.attachments = [] + assert serialized_diagram == data_models.CapellaWorkItem( type="diagram", uuid_capella=TEST_DIAG_UUID, @@ -1041,13 +1053,6 @@ def test_diagram(model: capellambse.MelodyModel): linked_work_items=[], ) - attachment = serialized_diagram.attachments[0] - attachment.content_bytes = None - - assert attachment == polarion_api.WorkItemAttachment( - "", "", "Diagram", None, "image/svg+xml", "__C2P__diagram.svg" - ) - @staticmethod @pytest.mark.parametrize( "layer,uuid,expected", From 53ef2c89591dbb2a599f96e47c98b4f06328558c Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 6 Feb 2024 19:00:10 +0100 Subject: [PATCH 26/31] fix: tests and presence of work item id and remove default serialize config There is no need to define the generica_work_item serializer in the converter config as it is called anyway. --- .../converters/converter_config.py | 2 +- .../converters/element_converter.py | 25 +++++++++---------- tests/test_elements.py | 4 +-- tests/test_workitem_attachments.py | 10 ++++---- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/capella2polarion/converters/converter_config.py b/capella2polarion/converters/converter_config.py index 9308cd0..88e20ed 100644 --- a/capella2polarion/converters/converter_config.py +++ b/capella2polarion/converters/converter_config.py @@ -24,7 +24,7 @@ class CapellaTypeConfig: def __post_init__(self): """Post processing for the initialization.""" - self.converters = _force_list(self.converters) or ["generic_work_item"] + self.converters = _force_list(self.converters) def _default_type_conversion(c_type: str) -> str: diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 939dba4..51cd8f5 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -108,12 +108,8 @@ def _condition( def _generate_image_html(attachment_id: str) -> str: """Generate an image as HTMl with the given source.""" - style = "; ".join( - (f"{key}: {value}" for key, value in DIAGRAM_STYLES.items()) - ) description = ( - f'

' + f'

' ) return description @@ -144,11 +140,12 @@ def serialize_all(self) -> list[data_models.CapellaWorkItem]: def serialize(self, uuid: str) -> data_models.CapellaWorkItem | None: """Return a CapellaWorkItem for the given diagram or element.""" converter_data = self.converter_session[uuid] - self._generic_work_item(converter_data) - old = self.capella_polarion_mapping.get_work_item_by_capella_uuid(uuid) - if old: - assert converter_data.work_item is not None - converter_data.work_item.id = old.id + work_item_id = None + if old := self.capella_polarion_mapping.get_work_item_by_capella_uuid( + uuid + ): + work_item_id = old.id + self.__generic_work_item(converter_data, work_item_id) for converter in converter_data.type_config.converters or []: try: @@ -171,11 +168,12 @@ def _add_attachment( work_item: data_models.CapellaWorkItem, attachment: polarion_api.WorkItemAttachment, ): + attachment.work_item_id = work_item.id or "" work_item.attachments.append(attachment) if attachment.mime_type == "image/svg+xml": work_item.attachments.append( polarion_api.WorkItemAttachment( - work_item.id, + attachment.work_item_id, "", attachment.title, cairosvg.svg2png(attachment.content_bytes), @@ -259,8 +257,8 @@ def _diagram( self._add_attachment(converter_data.work_item, attachment) return converter_data.work_item - def _generic_work_item( - self, converter_data: data_session.ConverterData + def __generic_work_item( + self, converter_data: data_session.ConverterData, work_item_id ) -> data_models.CapellaWorkItem: obj = converter_data.capella_element raw_description = getattr(obj, "description", None) @@ -270,6 +268,7 @@ def _generic_work_item( converter_data.description_references = uuids requirement_types = _get_requirement_types_text(obj) converter_data.work_item = data_models.CapellaWorkItem( + id=work_item_id, type=converter_data.type_config.p_type, title=obj.name, description_type="text/html", diff --git a/tests/test_elements.py b/tests/test_elements.py index c409adb..1067a75 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -53,7 +53,7 @@ TEST_SCENARIO = "afdaa095-e2cd-4230-b5d3-6cb771a90f51" TEST_CAP_REAL = "b80b3141-a7fc-48c7-84b2-1467dcef5fce" TEST_CONSTRAINT = "95cbd4af-7224-43fe-98cb-f13dda540b8e" -TEST_DIAG_DESCR = '

', status="open", linked_work_items=[], diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 165260c..2a00ac8 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -21,7 +21,7 @@ from .conftest import TEST_DIAGRAM_CACHE DIAGRAM_WI_CHECKSUM = ( - "37121e4c32bfae03ab387051f676f976de3b5b8b92c22351d906534ddf0a3ee8" + "77ace573c3fb71db26b502f3cbb931b032432fe27290703c476707723a1e360d" ) TEST_DIAG_UUID = "_APMboAPhEeynfbzU12yy7w" @@ -118,7 +118,7 @@ def test_diagram_attachments_new( ) assert ( - work_item.description == '

' "

" ) @@ -168,7 +168,7 @@ def test_diagram_attachments_updated( ) assert ( - work_item.description == '

' "

" ) @@ -228,7 +228,7 @@ def test_diagram_attachments_unchanged_work_item_changed( ) assert ( - work_item.description == '

' "

" ) @@ -307,7 +307,7 @@ def test_add_context_diagram( assert ( str(work_item.additional_attributes["context_diagram"]["value"]) - == '

' ) From aba9ce06686dfff2b00e15c4c060852810325031 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 7 Feb 2024 08:41:56 +0100 Subject: [PATCH 27/31] feat: set diagram max-widths to 650px for additional attributes and 800px for diagrams in descriptions --- .../converters/element_converter.py | 15 ++++++--- tests/test_elements.py | 27 ++++++++++++---- tests/test_workitem_attachments.py | 31 +++++++++---------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 51cd8f5..9e128bf 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -106,10 +106,14 @@ def _condition( return {"type": _type, "value": value} -def _generate_image_html(attachment_id: str) -> str: +def _generate_image_html( + title: str, attachment_id: str, max_width: int +) -> str: """Generate an image as HTMl with the given source.""" description = ( - f'

' + f'' ) return description @@ -187,6 +191,7 @@ def _draw_diagram_svg( diagram: capellambse.model.diagram.Diagram, file_name: str, title: str, + max_width: int, ) -> tuple[str, polarion_api.WorkItemAttachment | None]: file_name = f"{C2P_IMAGE_PREFIX}{file_name}.svg" @@ -213,7 +218,7 @@ def _draw_diagram_svg( else: attachment = None - return _generate_image_html(file_name), attachment + return _generate_image_html(title, file_name, max_width), attachment def _draw_additional_attributes_diagram( self, @@ -223,7 +228,7 @@ def _draw_additional_attributes_diagram( title: str, ): diagram_html, attachment = self._draw_diagram_svg( - diagram, attribute, title + diagram, attribute, title, 650 ) if attachment: self._add_attachment(work_item, attachment) @@ -241,7 +246,7 @@ def _diagram( work_item_id = converter_data.work_item.id diagram_html, attachment = self._draw_diagram_svg( - diagram, "diagram", "Diagram" + diagram, "diagram", "Diagram", 800 ) converter_data.work_item = data_models.CapellaWorkItem( diff --git a/tests/test_elements.py b/tests/test_elements.py index 1067a75..4197c27 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -53,7 +53,11 @@ TEST_SCENARIO = "afdaa095-e2cd-4230-b5d3-6cb771a90f51" TEST_CAP_REAL = "b80b3141-a7fc-48c7-84b2-1467dcef5fce" TEST_CONSTRAINT = "95cbd4af-7224-43fe-98cb-f13dda540b8e" -TEST_DIAG_DESCR = '

' +) TEST_SER_DIAGRAM: dict[str, t.Any] = { "id": "Diag-1", "title": "[CC] Capability", @@ -207,7 +211,9 @@ def test_create_diagrams(base_object: BaseObjectContainer): work_item.attachments = [] assert work_item == data_models.CapellaWorkItem(**TEST_SER_DIAGRAM) assert isinstance(description, str) - assert description.startswith(TEST_DIAG_DESCR) + assert description == TEST_DIAG_DESCR.format( + title="Diagram", attachment_id="__C2P__diagram.svg", width=800 + ) @staticmethod def test_create_diagrams_filters_non_diagram_elements( @@ -1047,8 +1053,9 @@ def test_diagram(model: capellambse.MelodyModel): uuid_capella=TEST_DIAG_UUID, title="[CC] Capability", description_type="text/html", - description="

', + description=TEST_DIAG_DESCR.format( + title="Diagram", attachment_id="__C2P__diagram.svg", width=800 + ), status="open", linked_work_items=[], ) @@ -1258,7 +1265,11 @@ def test_add_context_diagram(self, model: capellambse.MelodyModel): assert "context_diagram" in work_item.additional_attributes assert str( work_item.additional_attributes["context_diagram"]["value"] - ).startswith(TEST_DIAG_DESCR) + ) == TEST_DIAG_DESCR.format( + title="Context Diagram", + attachment_id="__C2P__context_diagram.svg", + width=650, + ) attachment = work_item.attachments[0] attachment.content_bytes = None @@ -1298,4 +1309,8 @@ def test_multiple_serializers(self, model: capellambse.MelodyModel): assert "context_diagram" in work_item.additional_attributes assert str( work_item.additional_attributes["context_diagram"]["value"] - ).startswith(TEST_DIAG_DESCR) + ) == TEST_DIAG_DESCR.format( + title="Context Diagram", + attachment_id="__C2P__context_diagram.svg", + width=650, + ) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 2a00ac8..381cb5f 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -19,9 +19,10 @@ ) from .conftest import TEST_DIAGRAM_CACHE +from .test_elements import TEST_DIAG_DESCR DIAGRAM_WI_CHECKSUM = ( - "77ace573c3fb71db26b502f3cbb931b032432fe27290703c476707723a1e360d" + "9815d6136d0354dac455e4759d0fdb119a1c384e010b71d4d139472a5331f2fd" ) TEST_DIAG_UUID = "_APMboAPhEeynfbzU12yy7w" @@ -117,10 +118,8 @@ def test_diagram_attachments_new( == created_attachments[0].file_name[:3] ) - assert ( - work_item.description == "

' - "

" + assert work_item.description == TEST_DIAG_DESCR.format( + title="Diagram", attachment_id="1-__C2P__diagram.svg", width=800 ) assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM @@ -167,10 +166,8 @@ def test_diagram_attachments_updated( worker.client.update_work_item.call_args.args[0] ) - assert ( - work_item.description == "

' - "

" + assert work_item.description == TEST_DIAG_DESCR.format( + title="Diagram", attachment_id="SVG-ATTACHMENT", width=800 ) @@ -227,10 +224,8 @@ def test_diagram_attachments_unchanged_work_item_changed( worker.client.update_work_item.call_args.args[0] ) - assert ( - work_item.description == "

' - "

" + assert work_item.description == TEST_DIAG_DESCR.format( + title="Diagram", attachment_id="SVG-ATTACHMENT", width=800 ) @@ -305,10 +300,12 @@ def test_add_context_diagram( == created_attachments[0].file_name[:3] ) - assert ( - str(work_item.additional_attributes["context_diagram"]["value"]) - == "

' + assert str( + work_item.additional_attributes["context_diagram"]["value"] + ) == TEST_DIAG_DESCR.format( + title="Context Diagram", + attachment_id="1-__C2P__context_diagram.svg", + width=650, ) From 2a230c79e2ce94be51919d712f179b6dcfcbd40e Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 7 Feb 2024 09:36:06 +0100 Subject: [PATCH 28/31] fix: set diagrams to 750px instead --- capella2polarion/converters/element_converter.py | 2 +- tests/test_elements.py | 4 ++-- tests/test_workitem_attachments.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 9e128bf..9c27bdc 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -246,7 +246,7 @@ def _diagram( work_item_id = converter_data.work_item.id diagram_html, attachment = self._draw_diagram_svg( - diagram, "diagram", "Diagram", 800 + diagram, "diagram", "Diagram", 750 ) converter_data.work_item = data_models.CapellaWorkItem( diff --git a/tests/test_elements.py b/tests/test_elements.py index 4197c27..9dca77f 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -212,7 +212,7 @@ def test_create_diagrams(base_object: BaseObjectContainer): assert work_item == data_models.CapellaWorkItem(**TEST_SER_DIAGRAM) assert isinstance(description, str) assert description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="__C2P__diagram.svg", width=800 + title="Diagram", attachment_id="__C2P__diagram.svg", width=750 ) @staticmethod @@ -1054,7 +1054,7 @@ def test_diagram(model: capellambse.MelodyModel): title="[CC] Capability", description_type="text/html", description=TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="__C2P__diagram.svg", width=800 + title="Diagram", attachment_id="__C2P__diagram.svg", width=750 ), status="open", linked_work_items=[], diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 381cb5f..1298977 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -119,7 +119,7 @@ def test_diagram_attachments_new( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="1-__C2P__diagram.svg", width=800 + title="Diagram", attachment_id="1-__C2P__diagram.svg", width=750 ) assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM @@ -167,7 +167,7 @@ def test_diagram_attachments_updated( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="SVG-ATTACHMENT", width=800 + title="Diagram", attachment_id="SVG-ATTACHMENT", width=750 ) @@ -225,7 +225,7 @@ def test_diagram_attachments_unchanged_work_item_changed( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="SVG-ATTACHMENT", width=800 + title="Diagram", attachment_id="SVG-ATTACHMENT", width=750 ) From 9dd7e4d8169a20ea2152a661ee42e6f97b78a142 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 7 Feb 2024 13:10:01 +0100 Subject: [PATCH 29/31] feat: add html class diagram for diagrams and additional-attributes-diagram for diagrams in additional attributes --- .../converters/element_converter.py | 16 ++++++++++------ tests/test_elements.py | 14 +++++++++++--- tests/test_workitem_attachments.py | 18 ++++++++++++++---- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/capella2polarion/converters/element_converter.py b/capella2polarion/converters/element_converter.py index 9c27bdc..50bf53c 100644 --- a/capella2polarion/converters/element_converter.py +++ b/capella2polarion/converters/element_converter.py @@ -33,7 +33,7 @@ f"" ) RE_CAMEL_CASE_2ND_WORD_PATTERN = re.compile(r"([a-z]+)([A-Z][a-z]+)") -DIAGRAM_STYLES = {"max-width": "100%"} + POLARION_WORK_ITEM_URL = ( '' @@ -107,11 +107,11 @@ def _condition( def _generate_image_html( - title: str, attachment_id: str, max_width: int + title: str, attachment_id: str, max_width: int, cls: str ) -> str: """Generate an image as HTMl with the given source.""" description = ( - f'' ) @@ -192,6 +192,7 @@ def _draw_diagram_svg( file_name: str, title: str, max_width: int, + cls: str, ) -> tuple[str, polarion_api.WorkItemAttachment | None]: file_name = f"{C2P_IMAGE_PREFIX}{file_name}.svg" @@ -218,7 +219,10 @@ def _draw_diagram_svg( else: attachment = None - return _generate_image_html(title, file_name, max_width), attachment + return ( + _generate_image_html(title, file_name, max_width, cls), + attachment, + ) def _draw_additional_attributes_diagram( self, @@ -228,7 +232,7 @@ def _draw_additional_attributes_diagram( title: str, ): diagram_html, attachment = self._draw_diagram_svg( - diagram, attribute, title, 650 + diagram, attribute, title, 650, "additional-attributes-diagram" ) if attachment: self._add_attachment(work_item, attachment) @@ -246,7 +250,7 @@ def _diagram( work_item_id = converter_data.work_item.id diagram_html, attachment = self._draw_diagram_svg( - diagram, "diagram", "Diagram", 750 + diagram, "diagram", "Diagram", 750, "diagram" ) converter_data.work_item = data_models.CapellaWorkItem( diff --git a/tests/test_elements.py b/tests/test_elements.py index 9dca77f..43c67d8 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -54,7 +54,7 @@ TEST_CAP_REAL = "b80b3141-a7fc-48c7-84b2-1467dcef5fce" TEST_CONSTRAINT = "95cbd4af-7224-43fe-98cb-f13dda540b8e" TEST_DIAG_DESCR = ( - '' ) @@ -212,7 +212,10 @@ def test_create_diagrams(base_object: BaseObjectContainer): assert work_item == data_models.CapellaWorkItem(**TEST_SER_DIAGRAM) assert isinstance(description, str) assert description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="__C2P__diagram.svg", width=750 + title="Diagram", + attachment_id="__C2P__diagram.svg", + width=750, + cls="diagram", ) @staticmethod @@ -1054,7 +1057,10 @@ def test_diagram(model: capellambse.MelodyModel): title="[CC] Capability", description_type="text/html", description=TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="__C2P__diagram.svg", width=750 + title="Diagram", + attachment_id="__C2P__diagram.svg", + width=750, + cls="diagram", ), status="open", linked_work_items=[], @@ -1269,6 +1275,7 @@ def test_add_context_diagram(self, model: capellambse.MelodyModel): title="Context Diagram", attachment_id="__C2P__context_diagram.svg", width=650, + cls="additional-attributes-diagram", ) attachment = work_item.attachments[0] @@ -1313,4 +1320,5 @@ def test_multiple_serializers(self, model: capellambse.MelodyModel): title="Context Diagram", attachment_id="__C2P__context_diagram.svg", width=650, + cls="additional-attributes-diagram", ) diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 1298977..1852ad1 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -22,7 +22,7 @@ from .test_elements import TEST_DIAG_DESCR DIAGRAM_WI_CHECKSUM = ( - "9815d6136d0354dac455e4759d0fdb119a1c384e010b71d4d139472a5331f2fd" + "122dc8471ac3135a1af1e3b44ac76a9acd888a9e4add162e7433a94aa498598d" ) TEST_DIAG_UUID = "_APMboAPhEeynfbzU12yy7w" @@ -119,7 +119,10 @@ def test_diagram_attachments_new( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="1-__C2P__diagram.svg", width=750 + title="Diagram", + attachment_id="1-__C2P__diagram.svg", + width=750, + cls="diagram", ) assert work_item.get_current_checksum() == DIAGRAM_CHECKSUM @@ -167,7 +170,10 @@ def test_diagram_attachments_updated( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="SVG-ATTACHMENT", width=750 + title="Diagram", + attachment_id="SVG-ATTACHMENT", + width=750, + cls="diagram", ) @@ -225,7 +231,10 @@ def test_diagram_attachments_unchanged_work_item_changed( ) assert work_item.description == TEST_DIAG_DESCR.format( - title="Diagram", attachment_id="SVG-ATTACHMENT", width=750 + title="Diagram", + attachment_id="SVG-ATTACHMENT", + width=750, + cls="diagram", ) @@ -306,6 +315,7 @@ def test_add_context_diagram( title="Context Diagram", attachment_id="1-__C2P__context_diagram.svg", width=650, + cls="additional-attributes-diagram", ) From 4908298b6f67ee990df83c33e92d4b70bae3212e Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 8 Feb 2024 16:57:47 +0100 Subject: [PATCH 30/31] refactor: Reduce amount of unneeded get attachments API calls --- .../connectors/polarion_worker.py | 29 +++++++--- tests/test_elements.py | 3 + tests/test_workitem_attachments.py | 55 +++++++++++++++++-- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 9c5d629..8261866 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -187,14 +187,25 @@ def patch_work_item( old_work_item_check_sum = old_checksums.pop("__C2P__WORK_ITEM") work_item_changed = new_work_item_check_sum != old_work_item_check_sum - try: - old_attachments = self.client.get_all_work_item_attachments( - work_item_id=old.id - ) - self.update_attachments( - new, old_checksums, new_checksums, old_attachments - ) + if work_item_changed or self.force_update: + old = self.client.get_work_item(old.id) + if old.attachments: + old_attachments = ( + self.client.get_all_work_item_attachments( + work_item_id=old.id + ) + ) + else: + old_attachments = [] + else: + old_attachments = self.client.get_all_work_item_attachments( + work_item_id=old.id + ) + if old_attachments or new.attachments: + self.update_attachments( + new, old_checksums, new_checksums, old_attachments + ) except polarion_api.PolarionApiException as error: logger.error( "Updating attachments for WorkItem %r (%s %s) failed. %s", @@ -203,13 +214,13 @@ def patch_work_item( ) return - self._refactor_attached_images(new) assert new.id is not None delete_links = None create_links = None if work_item_changed or self.force_update: - old = self.client.get_work_item(old.id) + if new.attachments: + self._refactor_attached_images(new) del new.additional_attributes["uuid_capella"] del old.additional_attributes["uuid_capella"] diff --git a/tests/test_elements.py b/tests/test_elements.py index 43c67d8..abacb4e 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -686,6 +686,9 @@ def test_update_work_items( assert base_object.pw.client.create_work_item_links.call_count == 0 assert base_object.pw.client.update_work_item.call_count == 1 assert base_object.pw.client.get_work_item.call_count == 1 + assert ( + base_object.pw.client.get_all_work_item_attachments.call_count == 0 + ) work_item = base_object.pw.client.update_work_item.call_args[0][0] assert isinstance(work_item, data_models.CapellaWorkItem) assert work_item.id == "Obj-1" diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 1852ad1..ffc8a8c 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -59,7 +59,7 @@ def worker(monkeypatch: pytest.MonkeyPatch): mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) config = mock.Mock(converter_config.ConverterConfig) - worker = polarion_worker.CapellaPolarionWorker( + return polarion_worker.CapellaPolarionWorker( polarion_worker.PolarionWorkerParams( "TEST", "http://localhost", @@ -69,8 +69,6 @@ def worker(monkeypatch: pytest.MonkeyPatch): config, ) - return worker - def set_attachment_ids(attachments: list[polarion_api.WorkItemAttachment]): counter = 0 @@ -80,6 +78,37 @@ def set_attachment_ids(attachments: list[polarion_api.WorkItemAttachment]): counter += 1 +def test_diagram_no_attachments(model: capellambse.MelodyModel): + converter = model_converter.ModelConverter(model, "TEST") + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + converter.generate_work_items( + polarion_repo.PolarionDataRepository(), False, False + ) + work_item = converter.converter_session[TEST_DIAG_UUID].work_item + assert work_item is not None + assert work_item.attachments == [] + + +def test_diagram_has_attachments(model: capellambse.MelodyModel): + converter = model_converter.ModelConverter(model, "TEST") + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( + "", + converter_config.CapellaTypeConfig("diagram", "diagram", []), + model.diagrams.by_uuid(TEST_DIAG_UUID), + ) + converter.generate_work_items( + polarion_repo.PolarionDataRepository(), False, True + ) + + work_item = converter.converter_session[TEST_DIAG_UUID].work_item + assert work_item is not None + assert len(work_item.attachments) == 2 + + def test_diagram_attachments_new( model: capellambse.MelodyModel, worker: polarion_worker.CapellaPolarionWorker, @@ -88,6 +117,10 @@ def test_diagram_attachments_new( worker.polarion_data_repo = polarion_repo.PolarionDataRepository( [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID)] ) + + worker.client.get_work_item.return_value = data_models.CapellaWorkItem( + WORKITEM_ID, uuid_capella=TEST_DIAG_UUID + ) worker.client.create_work_item_attachments = mock.MagicMock() worker.client.create_work_item_attachments.side_effect = set_attachment_ids @@ -103,6 +136,7 @@ def test_diagram_attachments_new( assert worker.client.update_work_item.call_count == 1 assert worker.client.create_work_item_attachments.call_count == 1 + assert worker.client.get_all_work_item_attachments.call_count == 0 created_attachments: list[ polarion_api.WorkItemAttachment @@ -135,8 +169,7 @@ def test_diagram_attachments_updated( worker.polarion_data_repo = polarion_repo.PolarionDataRepository( [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID)] ) - worker.client.get_all_work_item_attachments = mock.MagicMock() - worker.client.get_all_work_item_attachments.return_value = [ + existing_attachments = [ polarion_api.WorkItemAttachment( WORKITEM_ID, "SVG-ATTACHMENT", @@ -151,6 +184,17 @@ def test_diagram_attachments_updated( ), ] + worker.client.get_work_item.return_value = data_models.CapellaWorkItem( + WORKITEM_ID, + uuid_capella=TEST_DIAG_UUID, + attachments=existing_attachments, + ) + + worker.client.get_all_work_item_attachments = mock.MagicMock() + worker.client.get_all_work_item_attachments.return_value = ( + existing_attachments + ) + converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( "", converter_config.CapellaTypeConfig("diagram", "diagram", []), @@ -164,6 +208,7 @@ def test_diagram_attachments_updated( assert worker.client.update_work_item.call_count == 1 assert worker.client.create_work_item_attachments.call_count == 0 assert worker.client.update_work_item_attachment.call_count == 2 + assert worker.client.get_all_work_item_attachments.call_count == 1 work_item: data_models.CapellaWorkItem = ( worker.client.update_work_item.call_args.args[0] From b9800aaca92d8ed8c5dffec887d916e85b829ce4 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 12 Feb 2024 16:38:04 +0100 Subject: [PATCH 31/31] docs: Add cairosvg dependency --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c25998f..6d4b6d2 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Synchronise Capella models with Polarion projects Read the [full documentation on GitLab pages](https://dsd-dbs.github.io/capella-polarion). # Installation +We have a dependency on [cairosvg](https://cairosvg.org/). Please check their +[documentation](https://cairosvg.org/documentation/) for OS specific dependencies. You can install the latest released version directly from PyPI (**Not yet**).