Skip to content

Commit

Permalink
CP Newsroom ninjs formatter (#216)
Browse files Browse the repository at this point in the history 
* Updated to work for the bugs with qcodes in auto tagger.

Updated to work for the bugs with qcodes in auto tagger. Also removed the dependency to Index.js and will now use superdesk's library to fetch vocabularies.

* Update jimi_2.py

Fixed Bugs incoming on IndexCodes with non jimi tags.

* Update jimi_2.py

Fixed Issue with Categories.

* Updated to add JIMI tags in JSON.

* Update semaphore.py with updated code for scheme and qcodes.

* Updated to display proper JIMI Indexes and Categories.

* Updated to work for the bugs with qcodes in auto tagger.

Updated to work for the bugs with qcodes in auto tagger. Also removed the dependency to Index.js and will now use superdesk's library to fetch vocabularies.

* Update jimi_2.py

Fixed Bugs incoming on IndexCodes with non jimi tags.

* Updated to add JIMI tags in JSON.

* fix formatting

* Fixed the same key issue.

* Update ninjs_formatter_2.py

Removed Method defined twice issue.

* Update ninjs_formatter_2.py

Removing blank line issue

* Update jimi_2.py

Syntax issue solved

* Update ninjs_formatter_2.py

Removing White Spaces.

* Update ninjs_formatter_2.py

Removing trailing white spaces

* Update ninjs_formatter_2.py

Removing more white spaces

* Update ninjs_formatter_2.py

* Update semaphore.py

Fixed Mypy issues.

* Update semaphore.py

Removing White Spaces

* Update semaphore.py

Removed Final sets of error

* Update semaphore.py

Removing final mypy issues.

* Update semaphore.py

Fixed as per black formatter.

* Update ninjs_formatter_2.py

Black Formatted

* Update semaphore.py

Black formatted

* Update semaphore.py

Removing trailing white spaces

* Update ninjs_formatter_2.py

removing white spaces

* Update semaphore.py

Black formatting changes

* Add files via upload

* Update semaphore.py

Fixing Black Formatting

* Update scheme value in adapter.ts

* Add files via upload

Reformatted black file

* Add files via upload

Uploading Reformatted Ninjs_formatter.

* Update semaphore.py

Removing Env variables.

* Add files via upload

Black Formatted.

* Update ninjs_formatter_2.py

Updated formatter for NewsPro Output.

* Add files via upload

Black Formatting done.

* Add files via upload

* Black Formatted Formatters.

* Add files via upload

* Black Formatted

* Refactor auto-tagging component
Handle existing tags properly

* Update scheme value in adapter.ts

* use data_operation api for search and feedback

* Update and rename ninjs_newsroom_formatter.py to cp_ninjs_newsroom_formatter.py

renaming ninjs_newsroom_formatter.py to cp_ninjs_newsroom_formatter.py
changing the name of the Formatter to name = "CP Newsroom NINJS"
changing the type of the Formatter to type = "cp newsroom ninjs"

---------

Co-authored-by: tcp-bhargav <[email protected]>
Co-authored-by: Petr Jasek <[email protected]>
Co-authored-by: cmuldur <[email protected]>
  • Loading branch information
@cpefatimaabdillahi @tcp-bhargav
@petrjasek @cmuldur
4 people authored Mar 19, 2024
1 parent 6a65ab3 commit bf5c833
Showing 1 changed file with 159 additions and 159 deletions.
318 changes: 159 additions & 159 deletions ...put/formatter/ninjs_newsroom_formatter.py → .../formatter/cp_ninjs_newsroom_formatter.py
View file
Original file line number Diff line number Diff line change
@@ -1,159 +1,159 @@
# -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license


from .ninjs_formatter import NINJSFormatter
import superdesk
import elasticapm
import json
import logging

logger = logging.getLogger(__name__)


class NewsroomNinjsFormatter(NINJSFormatter):
name = "Newsroom NINJS"
type = "newsroom ninjs"

def __init__(self):
self.format_type = "newsroom ninjs"
self.can_preview = False
self.can_export = False
self.internal_renditions = ["original", "viewImage", "baseImage"]

def update_ninjs_subjects(self, ninjs, language="en-CA"):
try:
# Fetch the vocabulary
cv = superdesk.get_resource_service("vocabularies").find_one(
req=None, _id="subject_custom"
)
vocab_items = cv.get("items", [])
vocab_mapping = {}

vocab_mapping_all = {}

for item in vocab_items:
if item.get("in_jimi") is True:
name_in_vocab = item.get("name")
qcode = item.get("qcode")
translated_name = (
item.get("translations", {})
.get("name", {})
.get(language, name_in_vocab)
)
vocab_mapping[name_in_vocab.lower()] = (qcode, translated_name)

for item in vocab_items:
name_in_vocab = item.get("name")
qcode = item.get("qcode")
translated_name = (
item.get("translations", {})
.get("name", {})
.get(language, name_in_vocab)
)
vocab_mapping_all[name_in_vocab.lower()] = (qcode, translated_name)

updated_subjects = list(ninjs["subject"])

for subject in ninjs["subject"]:
subject_name = subject.get("name").lower()
if subject_name in vocab_mapping:
qcode, translated_name = vocab_mapping[subject_name]
updated_subjects.append(
{
"code": qcode,
"name": translated_name,
"scheme": "http://cv.cp.org/cp-subject-legacy/",
}
)
else:
if subject_name in vocab_mapping_all:
qcode, translated_name = vocab_mapping_all[subject_name]
updated_subjects.append(
{
"code": qcode,
"name": translated_name,
"scheme": "subject_custom",
}
)

ninjs["subject"] = [
{
**subject,
"name": (
vocab_mapping_all[subject["name"].lower()][1]
if subject["name"].lower() in vocab_mapping_all
and subject["scheme"] in ["subject"]
else subject["name"]
),
"scheme": (
"subject_custom"
if subject.get("scheme")
in ["http://cv.iptc.org/newscodes/mediatopic/", "subject"]
else subject.get("scheme")
),
}
for subject in updated_subjects
]

ninjs["subject"] = list(
{
json.dumps(subject, sort_keys=True): subject
for subject in ninjs["subject"]
}.values()
)

except Exception as e:
logger.error(
f"An error occurred. We are in NewsRoom Ninjs Formatter Ninjs Subjects exception: {str(e)}"
)

@elasticapm.capture_span()
def _format_products(self, article):
"""
Return a list of API product id's that the article matches.
:param article:
:return:
"""
result = superdesk.get_resource_service("product_tests").test_products(article)
return [
{"code": p["product_id"], "name": p.get("name")}
for p in result
if p.get("matched", False)
]

@elasticapm.capture_span()
def _transform_to_ninjs(self, article, subscriber, recursive=True):
ninjs = super()._transform_to_ninjs(article, subscriber, recursive)

if article.get("ingest_id") and article.get("auto_publish"):
ninjs["guid"] = article.get("ingest_id")
if article.get("ingest_version"):
ninjs["version"] = article["ingest_version"]

ninjs["products"] = self._format_products(article)

if article.get("assignment_id"):
assignment = superdesk.get_resource_service("assignments").find_one(
req=None, _id=article["assignment_id"]
)
if assignment is not None:
if assignment.get("coverage_item"):
ninjs.setdefault("coverage_id", assignment["coverage_item"])
if assignment.get("planning_item"):
ninjs.setdefault("planning_id", assignment["planning_item"])

if article.get("refs"):
ninjs["refs"] = article["refs"]

self.update_ninjs_subjects(ninjs, "en-CA")

return ninjs
# -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license


from .ninjs_formatter import NINJSFormatter
import superdesk
import elasticapm
import json
import logging

logger = logging.getLogger(__name__)


class NewsroomNinjsFormatter(NINJSFormatter):
name = "CP Newsroom NINJS"
type = "cp newsroom ninjs"

def __init__(self):
self.format_type = "cp newsroom ninjs"
self.can_preview = False
self.can_export = False
self.internal_renditions = ["original", "viewImage", "baseImage"]

def update_ninjs_subjects(self, ninjs, language="en-CA"):
try:
# Fetch the vocabulary
cv = superdesk.get_resource_service("vocabularies").find_one(
req=None, _id="subject_custom"
)
vocab_items = cv.get("items", [])
vocab_mapping = {}

vocab_mapping_all = {}

for item in vocab_items:
if item.get("in_jimi") is True:
name_in_vocab = item.get("name")
qcode = item.get("qcode")
translated_name = (
item.get("translations", {})
.get("name", {})
.get(language, name_in_vocab)
)
vocab_mapping[name_in_vocab.lower()] = (qcode, translated_name)

for item in vocab_items:
name_in_vocab = item.get("name")
qcode = item.get("qcode")
translated_name = (
item.get("translations", {})
.get("name", {})
.get(language, name_in_vocab)
)
vocab_mapping_all[name_in_vocab.lower()] = (qcode, translated_name)

updated_subjects = list(ninjs["subject"])

for subject in ninjs["subject"]:
subject_name = subject.get("name").lower()
if subject_name in vocab_mapping:
qcode, translated_name = vocab_mapping[subject_name]
updated_subjects.append(
{
"code": qcode,
"name": translated_name,
"scheme": "http://cv.cp.org/cp-subject-legacy/",
}
)
else:
if subject_name in vocab_mapping_all:
qcode, translated_name = vocab_mapping_all[subject_name]
updated_subjects.append(
{
"code": qcode,
"name": translated_name,
"scheme": "subject_custom",
}
)

ninjs["subject"] = [
{
**subject,
"name": (
vocab_mapping_all[subject["name"].lower()][1]
if subject["name"].lower() in vocab_mapping_all
and subject["scheme"] in ["subject"]
else subject["name"]
),
"scheme": (
"subject_custom"
if subject.get("scheme")
in ["http://cv.iptc.org/newscodes/mediatopic/", "subject"]
else subject.get("scheme")
),
}
for subject in updated_subjects
]

ninjs["subject"] = list(
{
json.dumps(subject, sort_keys=True): subject
for subject in ninjs["subject"]
}.values()
)

except Exception as e:
logger.error(
f"An error occurred. We are in CP NewsRoom Ninjs Formatter Ninjs Subjects exception: {str(e)}"
)

@elasticapm.capture_span()
def _format_products(self, article):
"""
Return a list of API product id's that the article matches.
:param article:
:return:
"""
result = superdesk.get_resource_service("product_tests").test_products(article)
return [
{"code": p["product_id"], "name": p.get("name")}
for p in result
if p.get("matched", False)
]

@elasticapm.capture_span()
def _transform_to_ninjs(self, article, subscriber, recursive=True):
ninjs = super()._transform_to_ninjs(article, subscriber, recursive)

if article.get("ingest_id") and article.get("auto_publish"):
ninjs["guid"] = article.get("ingest_id")
if article.get("ingest_version"):
ninjs["version"] = article["ingest_version"]

ninjs["products"] = self._format_products(article)

if article.get("assignment_id"):
assignment = superdesk.get_resource_service("assignments").find_one(
req=None, _id=article["assignment_id"]
)
if assignment is not None:
if assignment.get("coverage_item"):
ninjs.setdefault("coverage_id", assignment["coverage_item"])
if assignment.get("planning_item"):
ninjs.setdefault("planning_id", assignment["planning_item"])

if article.get("refs"):
ninjs["refs"] = article["refs"]

self.update_ninjs_subjects(ninjs, "en-CA")

return ninjs

0 comments on commit bf5c833

Please sign in to comment.