From 09fa463966aeaa06291af5dcefd8d46e5c5454d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Tue, 14 May 2024 17:03:14 +0200 Subject: [PATCH] add relevance to cp ninjs output (#239) SDCP-714 --- client/extensions/auto-tagger/src/adapter.ts | 22 ++++++++++++---- client/extensions/auto-tagger/src/types.ts | 2 ++ server/cp/ai/semaphore.py | 16 ++++++++---- .../cp/output/formatter/cp_ninjs_formatter.py | 4 +++ server/tests/conftest.py | 17 ++++++++++++ server/tests/orangelogic_test.py | 3 +-- server/tests/output/formatter/__init__.py | 6 ++++- .../formatter/cp_ninjs_formatter_test.py | 26 +++++++++++++++++++ server/tests/set_provice_on_publish_test.py | 7 ++++- 9 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 server/tests/conftest.py create mode 100644 server/tests/output/formatter/cp_ninjs_formatter_test.py diff --git a/client/extensions/auto-tagger/src/adapter.ts b/client/extensions/auto-tagger/src/adapter.ts index 841eac75..35ece73a 100644 --- a/client/extensions/auto-tagger/src/adapter.ts +++ b/client/extensions/auto-tagger/src/adapter.ts @@ -12,6 +12,8 @@ export interface ITagBase { altids: {[key: string]: string}; aliases?: Array; original_source?: string; + creator?: string; + relevance?: number; } export interface ISubjectTag extends ITagBase { @@ -44,11 +46,11 @@ export function toClientFormat(response: IServerResponse): OrderedMap(); response.subject?.forEach((item) => { - const {name, description, qcode, source, altids, aliases, original_source, parent, scheme} = item; + const {name, description, qcode, source, altids, aliases, original_source, parent, scheme, relevance, creator} = item; // Checking if the item has original_source to filter auto tagger tags if (original_source != null) { - if(scheme === 'http://cv.iptc.org/newscodes/mediatopic/' || scheme === 'subject') { + if (scheme === 'http://cv.iptc.org/newscodes/mediatopic/' || scheme === 'subject') { const tag: ITagUi = { name, description, @@ -62,6 +64,8 @@ export function toClientFormat(response: IServerResponse): OrderedMap { items.forEach((item) => { - const {name, description, qcode, source, altids, aliases, original_source, scheme} = item; + const {name, description, qcode, source, altids, aliases, original_source, scheme, relevance, creator} = item; const tag: ITagUi = { name, @@ -125,6 +131,8 @@ export function toClientFormat(response: IServerResponse): OrderedMap, superdesk: ISu result.subject = []; } - const {name, description, qcode, source, altids, aliases, original_source, parent} = item; + const {name, description, qcode, source, altids, aliases, original_source, parent, relevance, creator} = item; const subjectTag: ISubjectTag = { name, @@ -161,6 +169,8 @@ export function toServerFormat(items: OrderedMap, superdesk: ISu scheme: item.group.value, aliases, original_source, + relevance, + creator, }; result.subject.push(subjectTag); @@ -171,7 +181,7 @@ export function toServerFormat(items: OrderedMap, superdesk: ISu result[groupValue] = []; } - const {name, description, qcode, source, altids, aliases, original_source, scheme} = item; + const {name, description, qcode, source, altids, aliases, original_source, scheme, relevance, creator} = item; const tagBase: ITagBase = { name, @@ -182,6 +192,8 @@ export function toServerFormat(items: OrderedMap, superdesk: ISu aliases, original_source, scheme, + relevance, + creator, }; result[groupValue]!.push(tagBase); diff --git a/client/extensions/auto-tagger/src/types.ts b/client/extensions/auto-tagger/src/types.ts index 18397371..d74cd6e6 100644 --- a/client/extensions/auto-tagger/src/types.ts +++ b/client/extensions/auto-tagger/src/types.ts @@ -10,4 +10,6 @@ export interface ITagUi { /** The identifier of a controlled vocabulary which includes a code for the tag. */ scheme?: string; group: {kind: 'scheme' | 'visual'; value: string}; + creator?: string; + relevance?: number; } diff --git a/server/cp/ai/semaphore.py b/server/cp/ai/semaphore.py index 07194487..d972a365 100644 --- a/server/cp/ai/semaphore.py +++ b/server/cp/ai/semaphore.py @@ -23,6 +23,12 @@ TIMEOUT = (5, 30) +def format_relevance(value: str) -> int: + if value: + return int(float(value) * 100) + return 100 + + ResponseType = Mapping[str, Union[str, List[str]]] @@ -121,7 +127,7 @@ def fetch_parent_info(self, qcode): { "name": field.get("NAME"), "qcode": field.get("ID"), - "relevance": score, + "relevance": format_relevance(score), "parent": None, # Set to None initially } ) @@ -195,7 +201,7 @@ def transform_xml_response(api_response): "qcode": item["id"], "source": "Semaphore", "creator": "Human", - "relevance": score, + "relevance": int(score), "altids": {"source_name": "source_id"}, "original_source": "original_source_value", "scheme": scheme_url, @@ -238,7 +244,7 @@ def transform_xml_response(api_response): ), "creator": "Human", "source": "Semaphore", - "relevance": "100", + "relevance": format_relevance("100"), "altids": {"source_name": "source_id"}, "original_source": "original_source_value", "scheme": "http://cv.iptc.org/newscodes/mediatopic/", @@ -553,7 +559,7 @@ def adjust_score(score, existing_scores): "qcode": meta_id if meta_id else "", "creator": "Machine", "source": "Semaphore", - "relevance": meta_score, + "relevance": format_relevance(meta_score), "altids": f'{{"{meta_value}": "{meta_id}"}}', "original_source": "original_source_value", "scheme": scheme_url, @@ -574,7 +580,7 @@ def adjust_score(score, existing_scores): "parent": parent_qcode, "source": "Semaphore", "creator": "Machine", - "relevance": score, + "relevance": format_relevance(score), "altids": {"source_name": "source_id"}, "original_source": "original_source_value", "scheme": "http://cv.iptc.org/newscodes/mediatopic/", diff --git a/server/cp/output/formatter/cp_ninjs_formatter.py b/server/cp/output/formatter/cp_ninjs_formatter.py index 86db754f..425dacb4 100644 --- a/server/cp/output/formatter/cp_ninjs_formatter.py +++ b/server/cp/output/formatter/cp_ninjs_formatter.py @@ -13,6 +13,8 @@ def format_cv_item(item, language): "code": item.get("qcode"), "name": get_locale_name(item, language), "scheme": "http://cv.iptc.org/newscodes/mediatopic/", + "creator": item.get("creator", ""), + "relevance": item.get("relevance", 100), } ) else: @@ -21,6 +23,8 @@ def format_cv_item(item, language): "code": item.get("qcode"), "name": get_locale_name(item, language), "scheme": item.get("scheme"), + "creator": item.get("creator", ""), + "relevance": item.get("relevance", 100), } ) diff --git a/server/tests/conftest.py b/server/tests/conftest.py new file mode 100644 index 00000000..ff6212c6 --- /dev/null +++ b/server/tests/conftest.py @@ -0,0 +1,17 @@ +import flask +import pytest + + +@pytest.fixture(autouse=True) +def app(): + app = flask.Flask(__name__) + app.config.update( + { + "VERSION": "version", + "DEFAULT_LANGUAGE": "en", + } + ) + ctx = app.app_context() + ctx.push() + yield app + ctx.pop() diff --git a/server/tests/orangelogic_test.py b/server/tests/orangelogic_test.py index 23a3ba89..3ba12565 100644 --- a/server/tests/orangelogic_test.py +++ b/server/tests/orangelogic_test.py @@ -68,8 +68,7 @@ def setUp(self): self.ctx.push() def tearDown(self): - if hasattr(self, "ctx"): - self.ctx.pop() + self.ctx.pop() def test_instance(self): OrangelogicSearchProvider(self.provider) diff --git a/server/tests/output/formatter/__init__.py b/server/tests/output/formatter/__init__.py index b0325f15..e4e56196 100644 --- a/server/tests/output/formatter/__init__.py +++ b/server/tests/output/formatter/__init__.py @@ -20,7 +20,11 @@ def setUp(self): "DEFAULT_LANGUAGE": "en", } ) - self.app.app_context().push() + self.ctx = self.app.app_context() + self.ctx.push() + + def tearDown(self): + self.ctx.pop() def format(self, updates=None, _all=False): article = self.article.copy() diff --git a/server/tests/output/formatter/cp_ninjs_formatter_test.py b/server/tests/output/formatter/cp_ninjs_formatter_test.py new file mode 100644 index 00000000..96eb9fa5 --- /dev/null +++ b/server/tests/output/formatter/cp_ninjs_formatter_test.py @@ -0,0 +1,26 @@ +from unittest.mock import patch + +from cp.output.formatter.cp_ninjs_formatter import CPNINJSFormatter + + +@patch("superdesk.get_resource_service") +def test_subject_relevance(mock): + item = { + "type": "text", + "subject": [ + { + "name": "Society", + "qcode": "eef4a135-e188-4d1f-93f1-cf7af1f594a6", + "source": "Semaphore", + "altids": {"source_name": "source_id"}, + "scheme": "subject", + "original_source": "original_source_value", + "relevance": 54, + "creator": "Machine", + }, + ], + } + formatter = CPNINJSFormatter() + ninjs = formatter._transform_to_ninjs(item, {}) + assert ninjs["subject"][0]["relevance"] == 54 + assert ninjs["subject"][0]["creator"] == "Machine" diff --git a/server/tests/set_provice_on_publish_test.py b/server/tests/set_provice_on_publish_test.py index 12b329b0..81f151d8 100644 --- a/server/tests/set_provice_on_publish_test.py +++ b/server/tests/set_provice_on_publish_test.py @@ -16,7 +16,12 @@ def setUp(self) -> None: "DEFAULT_LANGUAGE": "en", } ) - self.app.app_context().push() + self.ctx = self.app.app_context() + self.ctx.push() + + def tearDown(self) -> None: + super().tearDown() + self.ctx.pop() def test_publish_signal(self): with patch.dict(superdesk.resources, resources):