From 19d28c5d648c6fb5790a8904af01ba3bde716044 Mon Sep 17 00:00:00 2001 From: MarkLark86 Date: Wed, 18 Jan 2023 09:43:32 +1100 Subject: [PATCH] [TGA-39] capi: Embed AuthorProfiles to item resources (#8) * client: Add `exclude_from_content_api` to filter out AuthorProfile fields in ContentAPI * api: Populate authors field on item update * capi: Add AuthorProfiles resource * capi: Override items resource to exclude Author Profiles * Add capi tests * Add `URN_DOMAIN` to config and use with with AuthorProfiles --- .../src/components/profileText/config.tsx | 39 ++- .../src/components/profileText/interfaces.ts | 1 + .../src/components/vocab/config.tsx | 12 + .../src/components/vocab/interfaces.ts | 1 + server/settings.py | 11 + server/tests/author_profiles_capi_test.py | 143 +++++++++ server/tests/author_profiles_test.py | 281 +++++++++++------- server/tga/author_profiles.py | 13 + server/tga/content_api/__init__.py | 0 server/tga/content_api/author_profiles.py | 143 +++++++++ server/tga/content_api/items.py | 25 ++ 11 files changed, 553 insertions(+), 116 deletions(-) create mode 100644 server/tests/author_profiles_capi_test.py create mode 100644 server/tga/content_api/__init__.py create mode 100644 server/tga/content_api/author_profiles.py create mode 100644 server/tga/content_api/items.py diff --git a/client/extensions/tga-author-profile-fields/src/components/profileText/config.tsx b/client/extensions/tga-author-profile-fields/src/components/profileText/config.tsx index 5521505..31c3ea2 100644 --- a/client/extensions/tga-author-profile-fields/src/components/profileText/config.tsx +++ b/client/extensions/tga-author-profile-fields/src/components/profileText/config.tsx @@ -7,23 +7,40 @@ import {IProfileTextFieldConfig} from './interfaces'; type IProps = IConfigComponentProps; +const {Spacer} = superdesk.components; + export class ProfileTextFieldConfig extends React.PureComponent { render() { - const config = this.props.config ?? {use_editor_3: false}; + const config = this.props.config ?? { + use_editor_3: false, + exclude_from_content_api: false, + }; const {gettext} = superdesk.localization; return ( - { - this.props.onChange({ - ...config, - use_editor_3: use_editor_3, - }); - }} - /> + + { + this.props.onChange({ + ...config, + use_editor_3: use_editor_3, + }); + }} + /> + { + this.props.onChange({ + ...config, + exclude_from_content_api: exclude_from_content_api, + }) + }} + /> + ); } } diff --git a/client/extensions/tga-author-profile-fields/src/components/profileText/interfaces.ts b/client/extensions/tga-author-profile-fields/src/components/profileText/interfaces.ts index f1f6f88..a2be54a 100644 --- a/client/extensions/tga-author-profile-fields/src/components/profileText/interfaces.ts +++ b/client/extensions/tga-author-profile-fields/src/components/profileText/interfaces.ts @@ -2,6 +2,7 @@ import {IEditorComponentProps} from 'superdesk-api'; export interface IProfileTextFieldConfig { use_editor_3: boolean; + exclude_from_content_api: boolean; } export type IProfileTextFieldProps = IEditorComponentProps; diff --git a/client/extensions/tga-author-profile-fields/src/components/vocab/config.tsx b/client/extensions/tga-author-profile-fields/src/components/vocab/config.tsx index 2ef60ed..332a148 100644 --- a/client/extensions/tga-author-profile-fields/src/components/vocab/config.tsx +++ b/client/extensions/tga-author-profile-fields/src/components/vocab/config.tsx @@ -43,6 +43,7 @@ export class VocabularyFieldConfig extends React.Component { this.props.onChange({ vocabulary_name: cvs[0]._id, allow_freetext: this.props.config?.allow_freetext ?? false, + exclude_from_content_api: this.props.config?.exclude_from_content_api ?? false, }); } }); @@ -52,6 +53,7 @@ export class VocabularyFieldConfig extends React.Component { const config = this.props.config ?? { vocabulary_name: '', allow_freetext: false, + exclude_from_content_api: false, }; const {Spacer} = superdesk.components; const {gettext} = superdesk.localization; @@ -82,6 +84,16 @@ export class VocabularyFieldConfig extends React.Component { }); }} /> + { + this.props.onChange({ + ...config, + exclude_from_content_api: exclude_from_content_api, + }) + }} + /> ); } diff --git a/client/extensions/tga-author-profile-fields/src/components/vocab/interfaces.ts b/client/extensions/tga-author-profile-fields/src/components/vocab/interfaces.ts index cc28c21..66fa967 100644 --- a/client/extensions/tga-author-profile-fields/src/components/vocab/interfaces.ts +++ b/client/extensions/tga-author-profile-fields/src/components/vocab/interfaces.ts @@ -3,6 +3,7 @@ import {IEditorComponentProps, IVocabularyItem} from 'superdesk-api'; export interface IVocabularyFieldConfig { vocabulary_name: string; allow_freetext: boolean; + exclude_from_content_api: boolean; } export type IVocabularyFieldProps = IEditorComponentProps; diff --git a/server/settings.py b/server/settings.py index f6383db..e70a194 100644 --- a/server/settings.py +++ b/server/settings.py @@ -11,6 +11,7 @@ from pathlib import Path from superdesk.default_settings import strtobool, env +from content_api.app.settings import CONTENTAPI_INSTALLED_APPS ABS_PATH = str(Path(__file__).resolve().parent) @@ -162,3 +163,13 @@ SLACK_BOT_TOKEN = env('SLACK_BOT_TOKEN', '') APM_SERVICE_NAME = "360info" +URN_DOMAIN = "360info:superdesk" + +CONTENTAPI_INSTALLED_APPS = [ + module + for module in CONTENTAPI_INSTALLED_APPS + if module != "content_api.items" +] + [ + "tga.content_api.items", + "tga.content_api.author_profiles", +] diff --git a/server/tests/author_profiles_capi_test.py b/server/tests/author_profiles_capi_test.py new file mode 100644 index 0000000..97f2613 --- /dev/null +++ b/server/tests/author_profiles_capi_test.py @@ -0,0 +1,143 @@ +from copy import copy +from flask import json + +from superdesk import get_resource_service +from superdesk.tests import TestCase + +from content_api.app import get_app +from content_api.publish import MONGO_PREFIX + +from settings import CONTENTAPI_INSTALLED_APPS +from .author_profiles_test import VOCABULARIES, CONTENT_TYPES +from tga.author_profiles import AUTHOR_PROFILE_ROLE + +TEST_USER = { + "_id": "abcd123", + "username": "foobar", + "first_name": "Foo", + "last_name": "Bar", + "user_type": "user", + "display_name": "Foo Bar", + "is_enabled": True, + "is_active": True, +} + +TEST_SUBSCRIBER = {"_id": "sub1"} + + +class ContentAPITestCase(TestCase): + def setUp(self): + self.content_api = get_resource_service("content_api") + self.db = self.app.data.mongo.pymongo(prefix=MONGO_PREFIX).db + self.app.config["SECRET_KEY"] = "secret" + config = copy(self.app.config) + config["AMAZON_CONTAINER_NAME"] = None # force gridfs + config["URL_PREFIX"] = "" + config["MEDIA_PREFIX"] = "/assets" + config["CONTENTAPI_INSTALLED_APPS"] = CONTENTAPI_INSTALLED_APPS + self.capi = get_app(config) + self.capi.testing = True + self.subscriber = {"_id": "sub1"} + + self.app.data.insert("vocabularies", VOCABULARIES) + self.app.data.insert("content_types", CONTENT_TYPES) + self.app.data.insert("users", [TEST_USER]) + + def _auth_headers(self, sub=None): + if sub is None: + sub = self.subscriber + service = get_resource_service("subscriber_token") + payload = {"subscriber": sub.get("_id")} + service.create([payload]) + token = payload["_id"] + headers = {"Authorization": "Token " + token} + return headers + + def _publish_user_profile(self): + item = { + "guid": "foo", + "type": "text", + "authors": [{ + "code": [TEST_USER["_id"], "Author Profile"], + "role": AUTHOR_PROFILE_ROLE, + "name": TEST_USER["display_name"], + "parent": TEST_USER["_id"], + "sub_label": TEST_USER["display_name"], + }], + "extra": { + "profile_id": TEST_USER["_id"], + "profile_first_name": "Fooey", + "profile_last_name": "Barey", + "profile_job_title": { + "qcode": "DIRECTOR", + "name": "Director", + "is_active": True, + }, + "profile_private_text": "This should not be included in the ContentAPI", + }, + } + self.content_api.publish(item, [TEST_SUBSCRIBER]) + + def _publish_content_item(self): + item = { + "guid": "content_bar", + "type": "text", + "authors": [{ + "code": [TEST_USER["_id"], "Writer"], + "role": "writer", + "name": TEST_USER["display_name"], + "parent": TEST_USER["_id"], + "sub_label": TEST_USER["display_name"], + }], + "slugline": "test-content", + "headling": "Test Content", + "body_html": "

Test Content

", + } + self.content_api.publish(item, [TEST_SUBSCRIBER]) + + def test_author_profiles_endpoint(self): + headers = self._auth_headers(TEST_SUBSCRIBER) + self._publish_user_profile() + self._publish_content_item() + + def assertUser(data): + self.assertEqual(data["first_name"], "Fooey") + self.assertEqual(data["last_name"], "Barey") + self.assertEqual(data["job_title"], "Director") + self.assertEqual(data["profile_id"], TEST_USER["_id"]) + self.assertEqual(data["uri"], "http://localhost:5400/author_profiles/abcd123") + self.assertNotIn("private_text", data) + + with self.capi.test_client() as c: + response = c.get("author_profiles", headers=headers) + self.assertEqual(200, response.status_code) + data = json.loads(response.data) + self.assertEqual(1, data["_meta"]["total"]) + self.assertEqual("foo", data["_items"][0]["original_id"]) + assertUser(data["_items"][0]) + + response = c.get("author_profiles/abcd123", headers=headers) + self.assertEqual(200, response.status_code) + data = json.loads(response.data) + self.assertEqual("foo", data["original_id"]) + assertUser(data) + + # User Profiles not available through the Content Items endpoint + self.assertEqual(200, c.get("items/content_bar", headers=headers).status_code) + self.assertEqual(404, c.get("items/abcd123", headers=headers).status_code) + + response = c.get("items", headers=headers) + self.assertEqual(200, response.status_code) + data = json.loads(response.data) + self.assertEqual(1, data["_meta"]["total"]) + self.assertEqual("content_bar", data["_items"][0]["original_id"]) + + # Make sure the Authors metadata was enhanced using User Profiles + author = data["_items"][0]["authors"][0] + self.assertEqual("abcd123", author["code"]) + self.assertEqual("Fooey", author["first_name"]) + self.assertEqual("Barey", author["last_name"]) + self.assertEqual("Director", author["job_title"]) + self.assertEqual("writer", author["role"]) + self.assertEqual("urn:360info:superdesk:user:abcd123", author["uri"]) + self.assertNotIn("private_text", author) diff --git a/server/tests/author_profiles_test.py b/server/tests/author_profiles_test.py index 891bac7..83a37f3 100644 --- a/server/tests/author_profiles_test.py +++ b/server/tests/author_profiles_test.py @@ -5,113 +5,184 @@ from tga.author_profiles import update_author_profile_content +VOCABULARIES = [ + { + "_id": "profile_id", + "field_type": "custom", + "items": [], + "type": "manageable", + "schema": {}, + "service": {"all": 1}, + "custom_field_type": "profile-id", + "display_name": "Author", + "unique_field": "qcode", + }, + { + "_id": "profile_job_title", + "field_type": "custom", + "items": [], + "type": "manageable", + "schema": {}, + "service": {"all": 1}, + "custom_field_type": "vocabulary-typeahead-field", + "custom_field_config": { + "vocabulary_name": "job_titles", + "allow_freetext": True, + }, + "display_name": "Job Title", + "unique_field": "qcode", + }, + { + "_id": "job_titles", + "display_name": "Job Titles", + "type": "manageable", + "selection_type": "single selection", + "unique_field": "qcode", + "schema": {"qcode": {}, "name": {}}, + "items": [ + {"qcode": "ceo", "name": "CEO", "is_active": True}, + {"qcode": "director", "name": "Director", "is_active": True}, + {"qcode": "media_advisor", "name": "Media Advisor", "is_active": True}, + ], + }, + { + "_id": "profile_first_name", + "field_type": "custom", + "items": [], + "type": "manageable", + "schema": {}, + "service": {"all": 1}, + "custom_field_type": "profile-text", + "display_name": "First Name", + }, + { + "_id": "profile_last_name", + "field_type": "custom", + "items": [], + "type": "manageable", + "schema": {}, + "service": {"all": 1}, + "custom_field_type": "profile-text", + "display_name": "Last Name", + }, + { + "_id": "profile_private_text", + "field_type": "custom", + "items": [], + "type": "manageable", + "schema": {}, + "service": {"all": 1}, + "custom_field_type": "profile-text", + "custom_field_config": { + "exclude_from_content_api": True, + }, + "display_name": "Last Name", + }, +] + +CONTENT_TYPES = [ + { + "_id": "article", + "label": "Article", + "enabled": True, + "editor": { + "slugline": { + "order": 1, + "sdWidth": "full", + "enabled": True, + "required": True, + }, + }, + "schema": { + "slugline": { + "type": "string", + "required": True, + "maxlength": 24, + "nullable": False, + }, + }, + }, + { + "_id": "author_profile", + "label": "Author Profile", + "enabled": True, + "editor": { + "profile_id": { + "enabled": True, + "field_name": "Author", + "order": 1, + "section": "header", + "required": True, + }, + "profile_job_title": { + "enabled": True, + "field_name": "Job Title", + "order": 2, + "section": "content", + "required": False, + }, + "profile_first_name": { + "enabled": True, + "field_name": "First Name", + "order": 3, + "section": "content", + "required": True, + }, + "profile_last_name": { + "enabled": True, + "field_name": "Last Name", + "order": 4, + "section": "content", + "required": True, + }, + "profile_private_text": { + "enabled": True, + "field_name": "Private Text", + "order": 5, + "section": "content", + "required": False, + }, + }, + "schema": { + "profile_id": { + "type": "custom", + "required": True, + "enabled": True, + "nullable": False, + }, + "profile_job_title": { + "type": "custom", + "required": False, + "enabled": True, + "nullable": True, + }, + "profile_first_name": { + "type": "custom", + "required": False, + "enabled": True, + "nullable": True, + }, + "profile_last_name": { + "type": "custom", + "required": False, + "enabled": True, + "nullable": True, + }, + "profile_private_text": { + "type": "custom", + "required": False, + "enabled": True, + "nullable": True, + }, + }, + }, +] + + class AuthorProfilesTest(TestCase): def setUp(self) -> None: - self.app.data.insert( - "vocabularies", - [ - { - "_id": "profile_id", - "field_type": "custom", - "items": [], - "type": "manageable", - "schema": {}, - "service": {"all": 1}, - "custom_field_type": "profile-id", - "display_name": "Author", - "unique_field": "qcode", - }, - { - "_id": "profile_job_title", - "field_type": "custom", - "items": [], - "type": "manageable", - "schema": {}, - "service": {"all": 1}, - "custom_field_type": "vocabulary-typeahead-field", - "custom_field_config": { - "vocabulary_name": "job_titles", - "allow_freetext": True, - }, - "display_name": "Job Title", - "unique_field": "qcode", - }, - { - "_id": "job_titles", - "display_name": "Job Titles", - "type": "manageable", - "selection_type": "single selection", - "unique_field": "qcode", - "schema": {"qcode": {}, "name": {}}, - "items": [ - {"qcode": "ceo", "name": "CEO", "is_active": True}, - {"qcode": "director", "name": "Director", "is_active": True}, - {"qcode": "media_advisor", "name": "Media Advisor", "is_active": True}, - ], - }, - ], - ) - self.app.data.insert( - "content_types", - [ - { - "_id": "article", - "label": "Article", - "enabled": True, - "editor": { - "slugline": { - "order": 1, - "sdWidth": "full", - "enabled": True, - "required": True, - }, - }, - "schema": { - "slugline": { - "type": "string", - "required": True, - "maxlength": 24, - "nullable": False, - }, - }, - }, - { - "_id": "author_profile", - "label": "Author Profile", - "enabled": True, - "editor": { - "profile_id": { - "enabled": True, - "field_name": "Author", - "order": 1, - "section": "header", - "required": True, - }, - "profile_job_title": { - "enabled": True, - "field_name": "Job Title", - "order": 3, - "section": "content", - "required": False, - }, - }, - "schema": { - "profile_id": { - "type": "custom", - "required": True, - "enabled": True, - "nullable": False, - }, - "profile_job_title": { - "type": "custom", - "required": False, - "enabled": True, - "nullable": True, - }, - }, - }, - ], - ) + self.app.data.insert("vocabularies", VOCABULARIES) + self.app.data.insert("content_types", CONTENT_TYPES) @patch("tga.author_profiles._set_article_fields") @patch("tga.author_profiles._add_cv_item_on_update") diff --git a/server/tga/author_profiles.py b/server/tga/author_profiles.py index e3235d9..31d20a5 100644 --- a/server/tga/author_profiles.py +++ b/server/tga/author_profiles.py @@ -2,6 +2,9 @@ from superdesk import get_resource_service +AUTHOR_PROFILE_ROLE = "author_profile" + + def update_author_profile_content(_sender: Any, updates: Dict[str, Any], original: Dict[str, Any]): """Process the ``AuthorProfile`` content on update""" @@ -49,6 +52,16 @@ def _set_article_fields(updates: Dict[str, Any]): if not updates.get("headline"): updates["headline"] = "Author Profile" + if extra.get("profile_id"): + user_id = updates["extra"]["profile_id"] + updates["authors"] = [{ + "_id": [user_id, "Author Profile"], + "role": AUTHOR_PROFILE_ROLE, + "name": "Author Profile", + "parent": user_id, + "sub_label": updates["slugline"] + }] + def _add_cv_item_on_update(updates: Dict[str, Any], original: Dict[str, Any], custom_fields: List[Dict[str, Any]]): """Iterates over ``vocabulary-typeahead-field`` values and adds missing items to the CV""" diff --git a/server/tga/content_api/__init__.py b/server/tga/content_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tga/content_api/author_profiles.py b/server/tga/content_api/author_profiles.py new file mode 100644 index 0000000..6ccd125 --- /dev/null +++ b/server/tga/content_api/author_profiles.py @@ -0,0 +1,143 @@ +from typing import List, Dict, Any + +from urllib.parse import urljoin, quote +from flask import current_app as app + +import superdesk +from superdesk.vocabularies import VocabulariesService, VocabulariesResource + +from content_api.items.resource import ItemsResource +from content_api.items.service import ItemsService +from tga.author_profiles import AUTHOR_PROFILE_ROLE + + +class AuthorProfileResource(ItemsResource): + datasource = { + "source": "items", + "search_backend": "elastic", + "default_sort": [("versioncreated", -1)] + } + item_methods = ["GET"] + resource_methods = ["GET"] + + +def _get_content_profile_public_field_ids(): + return [ + field["_id"] + for field in superdesk.get_resource_service("vocabularies").get_extra_fields() + if not (field.get("custom_field_config") or {}).get("exclude_from_content_api") + ] + + +class AuthoringProfileService(ItemsService): + def _set_request_filters(self, req, filters: List[Any]): + filters.append({"term": {"authors.role": AUTHOR_PROFILE_ROLE}}) + super()._set_request_filters(req, filters) + + def _is_internal_api(self): + return False + + def _get_uri(self, document): + resource_url = "{api_url}/{endpoint}/".format( + api_url=app.config["CONTENTAPI_URL"], endpoint=app.config["URLS"]["author_profiles"] + ) + try: + user_id = document["authors"][0]["code"] + except (IndexError, KeyError): + user_id = (document.get("extra") or {}).get("profile_id") + return urljoin(resource_url, quote(user_id)) + + def find_one(self, req, **lookup): + if (req is None or not req.args) and len(lookup) == 1 and lookup["_id"]: + # Attempting to get a single item by ID, return based on authors.uri field + user_profiles = self.get_author_profiles_by_user_ids([lookup["_id"]]) + return user_profiles[0] if user_profiles.count() else None + + return super().find_one(req=req, **lookup) + + def _process_fetched_object(self, profile: Dict[str, Any]): + super()._process_fetched_object(profile) + KEYS_TO_KEEP = ["firstcreated", "versioncreated", "original_id", "firstpublished", "_type", "_links", "uri", + "extra", "guid"] + for key in list(profile.keys()): + if key not in KEYS_TO_KEEP: + profile.pop(key) + + profile_value = self.get_profile_value_enhanced(profile["extra"]) + profile.update(profile_value) + profile.pop("extra") + + def get_profile_value_enhanced(self, item): + field_names = _get_content_profile_public_field_ids() + profile = {} + for key, val in item.items(): + if key not in field_names: + continue + elif key == "profile_id": + profile["profile_id"] = val + else: + profile_key = key.replace("profile_", "") + if isinstance(val, dict): + profile[profile_key] = val.get("name") or val.get("qcode") + if profile == "country" and val.get("region"): + profile["region"] = val["region"] + else: + profile[profile_key] = val + + return profile + + def get_author_profiles_by_user_ids(self, user_ids) -> List[Dict[str, Any]]: + urn_domain = app.config["URN_DOMAIN"] + return self.search({ + "query": { + "bool": { + "must": [ + {"terms": {"authors.uri": [f"urn:{urn_domain}:user:{user_id}" for user_id in user_ids]}}, + {"term": {"authors.role": AUTHOR_PROFILE_ROLE}}, + ], + }, + }, + }) + + def ehance_embedded_item_authors(self, document): + if not document.get("authors") or document["authors"][0].get("role") == AUTHOR_PROFILE_ROLE: + return + + author_profiles = { + profile["extra"]["profile_id"]: self.get_profile_value_enhanced(profile["extra"]) + for profile in self.get_author_profiles_by_user_ids([author["code"] for author in document["authors"]]) + } + field_names = _get_content_profile_public_field_ids() + for author in document.get("authors"): + author_id = author.get("code") + + author_profile = author_profiles.get(author_id) + if not author_profile: + # Profile not found for this Author + continue + + for field in field_names: + profile_field = field.replace("profile_", "") + if author_profile.get(profile_field): + author[profile_field] = author_profile[profile_field] + + def on_item_fetched(self, document): + self.ehance_embedded_item_authors(document) + + def on_items_fetched(self, result): + for document in result["_items"]: + self.ehance_embedded_item_authors(document) + + +def init_app(app): + endpoint_name = "vocabularies" + VocabulariesResource.internal_resource = True + service = VocabulariesService(endpoint_name, backend=superdesk.get_backend()) + VocabulariesResource(endpoint_name, app=app, service=service) + + endpoint_name = "author_profiles" + service = AuthoringProfileService(endpoint_name, backend=superdesk.get_backend()) + AuthorProfileResource(endpoint_name, app=app, service=service) + + app.on_fetched_item_items += service.on_item_fetched + app.on_fetched_resource_items += service.on_items_fetched diff --git a/server/tga/content_api/items.py b/server/tga/content_api/items.py new file mode 100644 index 0000000..8a048dd --- /dev/null +++ b/server/tga/content_api/items.py @@ -0,0 +1,25 @@ +from typing import List, Any + +from flask import json + +import superdesk +from content_api.items.resource import ItemsResource +from content_api.items.service import ItemsService as _ItemsService + +from tga.author_profiles import AUTHOR_PROFILE_ROLE + + +class ItemsService(_ItemsService): + def _set_request_filters(self, req, filters: List[Any]): + req_filter = {"bool": {"must_not": [{"term": {"authors.role": AUTHOR_PROFILE_ROLE}}]}} + + if filters: + req_filter["bool"]["must"] = filters + + req.args["filter"] = json.dumps(req_filter) + + +def init_app(app): + endpoint_name = "items" + service = ItemsService(endpoint_name, backend=superdesk.get_backend()) + ItemsResource(endpoint_name, app=app, service=service)