From efb58a3c21340fa94ad8bbad7c6b4f3df86be711 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Tue, 26 Dec 2023 15:50:46 +0530 Subject: [PATCH 1/6] add NTBKilde attribute in ninjs output [SDNTB-861] --- server/ntb/publish/ntb_ninjs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/ntb/publish/ntb_ninjs.py b/server/ntb/publish/ntb_ninjs.py index b82c57ade..31e30e6c5 100644 --- a/server/ntb/publish/ntb_ninjs.py +++ b/server/ntb/publish/ntb_ninjs.py @@ -77,6 +77,9 @@ def _transform_to_ninjs(self, article, subscriber, recursive=True): if ninjs.get("guid"): ninjs.setdefault("uri", ninjs["guid"]) + if ninjs.get("extra", {}).get("ntb_pub_name"): + ninjs["NTBKilde"] = ninjs["extra"]["ntb_pub_name"] + # removed items which mapped according to Ninjs v2 properties ninjs_properties = [ "headlines", @@ -116,6 +119,7 @@ def _transform_to_ninjs(self, article, subscriber, recursive=True): "rightsinfo", "service", "infosources", + "NTBKilde", ] for key in list(ninjs.keys()): From 91139703624f3e0830ad415d3f3d51cdc3fe8f0d Mon Sep 17 00:00:00 2001 From: devketanpro Date: Tue, 26 Dec 2023 15:53:11 +0530 Subject: [PATCH 2/6] udpate tests --- server/ntb/tests/publish/ntb_ninjs_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/ntb/tests/publish/ntb_ninjs_test.py b/server/ntb/tests/publish/ntb_ninjs_test.py index b98685bd8..e3555b723 100644 --- a/server/ntb/tests/publish/ntb_ninjs_test.py +++ b/server/ntb/tests/publish/ntb_ninjs_test.py @@ -237,6 +237,9 @@ class Ninjs2FormatterTest(TestCase): ], }, }, + "extra":{ + "ntb_pub_name": "test ntb_pub_name", + } } def setUp(self): @@ -361,6 +364,7 @@ def test_format_item(self): "infosources": [{"name": "NTB"}], "copyrightholder": "NTB", "by": "byline", + "NTBKilde": "test ntb_pub_name" } self.assertEqual(ninjs, expected_item) From 94d0cbd9a43269d61ef2d49992cfe01cf594496e Mon Sep 17 00:00:00 2001 From: devketanpro Date: Tue, 26 Dec 2023 15:54:01 +0530 Subject: [PATCH 3/6] refactore code via Black --- server/ntb/publish/ntb_ninjs.py | 46 +++++--- server/ntb/tests/publish/ntb_ninjs_test.py | 122 +++++++++++++-------- 2 files changed, 102 insertions(+), 66 deletions(-) diff --git a/server/ntb/publish/ntb_ninjs.py b/server/ntb/publish/ntb_ninjs.py index 31e30e6c5..48f8efca2 100644 --- a/server/ntb/publish/ntb_ninjs.py +++ b/server/ntb/publish/ntb_ninjs.py @@ -22,9 +22,7 @@ def convert_dicts_to_lists(ninjs): for field in ("renditions", "associations"): if ninjs.get(field): ninjs[field] = [ - format_array_value(value, key) - for key, value - in ninjs[field].items() + format_array_value(value, key) for key, value in ninjs[field].items() ] elif field in ninjs: ninjs.pop(field) @@ -131,10 +129,12 @@ def _transform_to_ninjs(self, article, subscriber, recursive=True): ] if article.get("family_id"): - ninjs["altids"].extend([ - {"role": "NTB-ID", "value": utils.get_ntb_id(article)}, - {"role": "DOC-ID", "value": utils.get_doc_id(article)}, - ]) + ninjs["altids"].extend( + [ + {"role": "NTB-ID", "value": utils.get_ntb_id(article)}, + {"role": "DOC-ID", "value": utils.get_doc_id(article)}, + ] + ) if article.get("assignment_id"): self._format_planning_ids(ninjs, article) @@ -175,7 +175,9 @@ def _format_place(self, article) -> List[Dict]: ninjs_place["state-prov"] = cv_place["ntb_parent"] if place.get("altids") and place["altids"].get("wikidata"): - ninjs_place["uri"] = "http://www.wikidata.org/entity/{}".format(place["altids"]["wikidata"]) + ninjs_place["uri"] = "http://www.wikidata.org/entity/{}".format( + place["altids"]["wikidata"] + ) ninjs_place["literal"] = place["altids"]["wikidata"] places.append(ninjs_place) return places @@ -251,15 +253,23 @@ def places(self): return getattr(g, "_places") def _format_planning_ids(self, ninjs, article): - assignment = get_resource_service("assignments").find_one(req=None, _id=article["assignment_id"]) + assignment = get_resource_service("assignments").find_one( + req=None, _id=article["assignment_id"] + ) if assignment and assignment.get("planning_item"): - ninjs.setdefault("altids", []).append({ - "role": "PLANNING-ID", - "value": assignment["planning_item"], - }) - planning = get_resource_service("planning").find_one(req=None, _id=assignment["planning_item"]) + ninjs.setdefault("altids", []).append( + { + "role": "PLANNING-ID", + "value": assignment["planning_item"], + } + ) + planning = get_resource_service("planning").find_one( + req=None, _id=assignment["planning_item"] + ) if planning and planning.get("event_item"): - ninjs["altids"].append({ - "role": "EVENT-ID", - "value": planning["event_item"], - }) + ninjs["altids"].append( + { + "role": "EVENT-ID", + "value": planning["event_item"], + } + ) diff --git a/server/ntb/tests/publish/ntb_ninjs_test.py b/server/ntb/tests/publish/ntb_ninjs_test.py index e3555b723..3d874bc3d 100644 --- a/server/ntb/tests/publish/ntb_ninjs_test.py +++ b/server/ntb/tests/publish/ntb_ninjs_test.py @@ -22,7 +22,9 @@

testNTBMEDIA TO REMOVE

""".strip() -with open(pathlib.Path(__file__).parent.joinpath("fixtures", "text-item-with-table.json")) as f: +with open( + pathlib.Path(__file__).parent.joinpath("fixtures", "text-item-with-table.json") +) as f: text_item_with_table = json.load(f) @@ -132,15 +134,15 @@ class Ninjs2FormatterTest(TestCase): "qcode": "Global", }, { - 'aliases': [], - 'altids': { - 'imatrics': '34017822-d341-3826-ab94-71d226d639c4', + "aliases": [], + "altids": { + "imatrics": "34017822-d341-3826-ab94-71d226d639c4", }, - 'name': 'Genève', - 'original_source': None, - 'qcode': '34017822-d341-3826-ab94-71d226d639c4', - 'scheme': 'place_custom', - 'source': 'imatrics', + "name": "Genève", + "original_source": None, + "qcode": "34017822-d341-3826-ab94-71d226d639c4", + "scheme": "place_custom", + "source": "imatrics", }, ], "object": [ @@ -232,20 +234,24 @@ class Ninjs2FormatterTest(TestCase): "parent": "05000000", "scheme": None, "name": "further education", - "qcode": "05002000" + "qcode": "05002000", }, ], }, }, - "extra":{ + "extra": { "ntb_pub_name": "test ntb_pub_name", - } + }, } def setUp(self): super().setUp() self.formatter = NTBNINJSFormatter() - with open(pathlib.Path(__file__).parent.parent.parent.parent.joinpath("data/vocabularies.json")) as json_file: + with open( + pathlib.Path(__file__).parent.parent.parent.parent.joinpath( + "data/vocabularies.json" + ) + ) as json_file: json_cvs = json.load(json_file) for cv in json_cvs: if cv.get("_id") == "place_custom": @@ -346,8 +352,8 @@ def test_format_item(self): "literal": "Global", }, { - "name": 'Genève', - "literal": '34017822-d341-3826-ab94-71d226d639c4', + "name": "Genève", + "literal": "34017822-d341-3826-ab94-71d226d639c4", }, ], "taglines": [ @@ -364,41 +370,56 @@ def test_format_item(self): "infosources": [{"name": "NTB"}], "copyrightholder": "NTB", "by": "byline", - "NTBKilde": "test ntb_pub_name" + "NTBKilde": "test ntb_pub_name", } self.assertEqual(ninjs, expected_item) - self.assertEqual(assoc, [{ - "name": "featuremedia", - "altids": [ - {"role": "GUID", "value": "test_id"}, - ], - "descriptions": [ - {"contenttype": "text/plain", "value": "test feature media"}, - ], - "uri": "test_id", - "headlines": [ - {"contenttype": "text/plain", "value": "feature headline"}, - ], - "pubstatus": "usable", - "taglines": [], - "type": "picture", - "version": "1", - "versioncreated": "2023-03-01T12:10:00+0000", - "subjects": [ - {"name": "further education", "uri": "topics:05002000"}, + self.assertEqual( + assoc, + [ + { + "name": "featuremedia", + "altids": [ + {"role": "GUID", "value": "test_id"}, + ], + "descriptions": [ + {"contenttype": "text/plain", "value": "test feature media"}, + ], + "uri": "test_id", + "headlines": [ + {"contenttype": "text/plain", "value": "feature headline"}, + ], + "pubstatus": "usable", + "taglines": [], + "type": "picture", + "version": "1", + "versioncreated": "2023-03-01T12:10:00+0000", + "subjects": [ + {"name": "further education", "uri": "topics:05002000"}, + ], + "copyrightholder": "NTB", + } ], - "copyrightholder": "NTB", - }]) + ) def test_planning_ids(self): - self.app.data.insert("assignments", [ - {"_id": "assignment-id", "coverage_item": self.article["guid"], "planning_item": "planning-id"}, - ]) - self.app.data.insert("planning", [ - {"_id": "planning-id", "event_item": "event-id"}, - ]) + self.app.data.insert( + "assignments", + [ + { + "_id": "assignment-id", + "coverage_item": self.article["guid"], + "planning_item": "planning-id", + }, + ], + ) + self.app.data.insert( + "planning", + [ + {"_id": "planning-id", "event_item": "event-id"}, + ], + ) ninjs = self.format() @@ -417,9 +438,14 @@ def test_empty_assocations_renditions(self): assert "associations" not in ninjs, ninjs.get("associations") def test_publish_table(self): - ninjs = self.format({ - "fields_meta": text_item_with_table["fields_meta"], - "body_html": text_item_with_table["body_html"], - }) + ninjs = self.format( + { + "fields_meta": text_item_with_table["fields_meta"], + "body_html": text_item_with_table["body_html"], + } + ) assert "" in ninjs["bodies"][0]["value"] - assert text_item_with_table["body_html"].replace(" ", " ") == ninjs["bodies"][0]["value"] + assert ( + text_item_with_table["body_html"].replace(" ", " ") + == ninjs["bodies"][0]["value"] + ) From 9984151182beea0158fdd489eebca697694669f2 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Wed, 27 Dec 2023 16:56:06 +0530 Subject: [PATCH 4/6] update test for copyright text --- server/ntb/tests/publish/ntb_ninjs_test.py | 6 ++++-- server/ntb/tests/publish/ntb_nitf_test.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/ntb/tests/publish/ntb_ninjs_test.py b/server/ntb/tests/publish/ntb_ninjs_test.py index 3d874bc3d..b527604a5 100644 --- a/server/ntb/tests/publish/ntb_ninjs_test.py +++ b/server/ntb/tests/publish/ntb_ninjs_test.py @@ -20,6 +20,8 @@

intermediate line

this element should have a txt class

testNTBMEDIA TO REMOVE

+ +

(©MyCompany2023)

""".strip() with open( @@ -319,8 +321,8 @@ def test_format_item(self): ], "bodies": [ { - "charcount": 132, - "wordcount": 25, + "charcount": 148, + "wordcount": 26, "value": TEST_BODY_EXPECTED, "contenttype": "text/html", } diff --git a/server/ntb/tests/publish/ntb_nitf_test.py b/server/ntb/tests/publish/ntb_nitf_test.py index 07eea19eb..5e14ae7ad 100644 --- a/server/ntb/tests/publish/ntb_nitf_test.py +++ b/server/ntb/tests/publish/ntb_nitf_test.py @@ -51,6 +51,7 @@
SCRIPT TO FOLLOW
+

(©MyCompany2023)

""" ) ARTICLE = { @@ -442,7 +443,7 @@ def test_pubdata(self): pubdata = self.nitf_xml.find("head/pubdata") expected = NOW.astimezone(self.tz).strftime("%Y%m%dT%H%M%S") self.assertEqual(pubdata.get("date.publication"), expected) - self.assertEqual(pubdata.get("item-length"), "121") + self.assertEqual(pubdata.get("item-length"), "137") self.assertEqual(pubdata.get("unit-of-measure"), "character") def test_dateline(self): From db7013afb7cf8e17827fea7a91fb1cdc53c52ac8 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Wed, 27 Dec 2023 16:56:29 +0530 Subject: [PATCH 5/6] reformat code via black --- server/ntb/tests/publish/ntb_nitf_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/ntb/tests/publish/ntb_nitf_test.py b/server/ntb/tests/publish/ntb_nitf_test.py index 5e14ae7ad..dce016327 100644 --- a/server/ntb/tests/publish/ntb_nitf_test.py +++ b/server/ntb/tests/publish/ntb_nitf_test.py @@ -318,15 +318,15 @@ "original_source": "1013", }, { - 'aliases': [], - 'altids': { - 'imatrics': '34017822-d341-3826-ab94-71d226d639c4', + "aliases": [], + "altids": { + "imatrics": "34017822-d341-3826-ab94-71d226d639c4", }, - 'name': 'Genève', - 'original_source': None, - 'qcode': '34017822-d341-3826-ab94-71d226d639c4', - 'scheme': 'place_custom', - 'source': 'imatrics', + "name": "Genève", + "original_source": None, + "qcode": "34017822-d341-3826-ab94-71d226d639c4", + "scheme": "place_custom", + "source": "imatrics", }, ], "object": [ @@ -707,7 +707,7 @@ def test_pretty_formatting(self): formatter_output = self.formatter.format(article, {"name": "Test NTBNITF"}) doc = formatter_output[0]["encoded_item"] body_content = doc[ - doc.find(b"") - 4: doc.find(b"") + 15 + doc.find(b"") - 4 : doc.find(b"") + 15 ] expected = ( b"""\ From 51fc1be12fef3b3936a9dd8069cb890fb75193b6 Mon Sep 17 00:00:00 2001 From: devketanpro Date: Wed, 27 Dec 2023 17:12:11 +0530 Subject: [PATCH 6/6] fix flake --- server/ntb/tests/publish/ntb_nitf_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ntb/tests/publish/ntb_nitf_test.py b/server/ntb/tests/publish/ntb_nitf_test.py index dce016327..002dd9512 100644 --- a/server/ntb/tests/publish/ntb_nitf_test.py +++ b/server/ntb/tests/publish/ntb_nitf_test.py @@ -707,7 +707,7 @@ def test_pretty_formatting(self): formatter_output = self.formatter.format(article, {"name": "Test NTBNITF"}) doc = formatter_output[0]["encoded_item"] body_content = doc[ - doc.find(b"") - 4 : doc.find(b"") + 15 + doc.find(b"") - 4: doc.find(b"") + 15 ] expected = ( b"""\