diff --git a/doajtest/testbook/administrative_search/journals.yml b/doajtest/testbook/administrative_search/journals.yml index 9d249aff11..03b6e4123b 100644 --- a/doajtest/testbook/administrative_search/journals.yml +++ b/doajtest/testbook/administrative_search/journals.yml @@ -134,3 +134,86 @@ tests: results: - A new browser window opens containing the edit form for the journal - step: Click "Unlock and Close" + +- title: Test notes searchable for admin and not for public + context: + role: admin + steps: + - step: In the admin search, select Journal search, search for a journal and click on the "Edit Journal" button to edit the journal + path: /admin + results: + - The journal edit form is displayed + - step: Add a note to a journal record by clicking on the "Add" button in the "Notes" section of the journal record. Add a distinctive note + whose text is unlikely to appear in a journal title or other metadata. Then click on "save" button to save the new note + results: + - The note is added to the journal record + - step: Go to the admin journal search again and search for the journal you added the note with the words given in the note + path: /admin + results: + - The journal record is displayed in the search results + - step: Search for the journal from public search page with the distinctive words from the note + path: /search/journals + results: + - The journal record is not displayed in the search results + - step: Go to the API documentation page + path: /api/docs + - step: Scroll to the section "Search" > "Search journals" and click on the row to expand it. + - step: in the section "search_query" enter the distinctive words from the note you created, then click on the "Try it Out!" button at the bottom of the section + results: + - The journal record is not displayed in the search results + +- title: Test notes not searchable for public + context: + role: anonymous + steps: + - step: go to the public journal search + path: /search/journals + - step: Search for the distinctive word you entered into a note in the previous test. + results: No results displayed + - step: Modify the search url in the following way. Add the text '%2C"default_field"%3A"admin.notes.note"' in the URL bar immediately after where it says '"default_operator"%3A"AND"'. + This is to attempt to trick the public search into allowing you to search on notes. + results: An error message displayed + +- title: Test notes searchable for Editor + context: + role: Editor + steps: + - step: Make sure the editor group is assigned for the journal / application which is under test and assigned to the user + - step: Go to the tab 'Your group’s journals' + results: Group's journals page displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Your group’s applications' + results: Group's applications page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results + - step: Go to the tab 'Journals assigned to you' + results: Journals assigned to you page displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Applications assigned to you' + results: Applications assigned to you page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results + +- title: Test notes searchable for Associate Editor + context: + role: AssociateEditor + steps: + - step: Go to the tab 'Journals assigned to you' + results: 'Journals assigned to you page' displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Applications assigned to you' + results: 'Applications assigned to you' page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results + +- title: Test notes not searchable for fixed query widget (joint journal/article search) + context: + role: anonymous + steps: + - step: 'Do a url search for the keyword which is given in a notes for journal,article index. Ex: /query/journal,article/_search?&source={"query":{"query_string":{"query":"Test Note","default_operator":"AND"}},"size":"20"}&ref=fqw' + results: No results displayed + - step: Modify the search url to add the notes keyword and the default field 'admin.notes.note' and then hit the url + results: Error message displayed diff --git a/doajtest/testbook/public_site/public_search.yml b/doajtest/testbook/public_site/public_search.yml index 6b47834e85..43534a3e7c 100644 --- a/doajtest/testbook/public_site/public_search.yml +++ b/doajtest/testbook/public_site/public_search.yml @@ -58,15 +58,6 @@ tests: results: - The search/embed box is shown with a URL and an HTML/JS snippet displayed in each box - - step: Click "shorten url" (Note that this may not work in test, due to configuration - requirements, you may receive an error instead) - results: - - The long search url is replaced with a bit.ly url - - '"shorten url" is replaced by "original url"' - - step: Click "original url" - results: - - The bit.ly url is replaced by the original long search url - - '"original url" is replaced by "shorten url"' - step: Close the "Share | Embed" section by clicking the Close button at the bottom of the box, or clicking the "Share | Embed" button again results: diff --git a/doajtest/unit/test_query.py b/doajtest/unit/test_query.py index 99bf94976a..d1717341c7 100644 --- a/doajtest/unit/test_query.py +++ b/doajtest/unit/test_query.py @@ -1,6 +1,7 @@ from portality import models -from doajtest.fixtures import AccountFixtureFactory, ArticleFixtureFactory +from doajtest.fixtures import AccountFixtureFactory, ArticleFixtureFactory, EditorGroupFixtureFactory, \ + ApplicationFixtureFactory from doajtest.helpers import DoajTestCase, deep_sort from portality.bll.services.query import QueryService, Query @@ -30,7 +31,12 @@ "auth" : True, "role" : "admin", "dao" : "portality.models.Journal" - } + }, + "suggestion" : { + "auth" : True, + "role" : "admin", + "dao" : "portality.models.Application" + }, }, "api_query" : { "article" : { @@ -53,19 +59,89 @@ "query_filters" : ["owner", "private_source"], "dao" : "portality.models.Suggestion" } + }, + "editor_query" : { + "journal" : { + "auth" : True, + "role" : "editor", + "dao" : "portality.models.Journal" + }, + "suggestion" : { + "auth" : True, + "role" : "editor", + "dao" : "portality.models.Application" + } + }, + "associate_query": { + "journal": { + "auth": True, + "role": "associate_editor", + "dao": "portality.models.Journal" + }, + "suggestion" : { + "auth" : True, + "role" : "associate_editor", + "dao" : "portality.models.Application" + } + } +} + +SEARCH_ALL_QUERY_ROUTE = { + "query" : { + "journal" : { + "auth" : False, + "role" : None, + "query_filters" : ["search_all_meta"], + "dao" : "portality.models.Journal" + } + }, + "editor_query" : { + "journal" : { + "auth" : True, + "role" : "editor", + "query_filters" : ["search_all_meta"], + "dao" : "portality.models.Journal" + }, + "suggestion" : { + "auth" : False, + "role" : "editor", + "query_filters" : ["search_all_meta"], + "dao" : "portality.models.Application" + } + }, + "associate_query": { + "journal": { + "auth": False, + "role": "associate_editor", + "query_filters" : ["search_all_meta"], + "dao": "portality.models.Journal" + }, + "suggestion" : { + "auth" : False, + "role" : "associate_editor", + "query_filters" : ["search_all_meta"], + "dao" : "portality.models.Application" + } } } QUERY_FILTERS = { + "non_public_fields_validator" : "portality.lib.query_filters.non_public_fields_validator", + # query filters "only_in_doaj" : "portality.lib.query_filters.only_in_doaj", "owner" : "portality.lib.query_filters.owner", + "associate" : "portality.lib.query_filters.associate", + "editor" : "portality.lib.query_filters.editor", # result filters "public_result_filter" : "portality.lib.query_filters.public_result_filter", # source filter - "public_source": "portality.lib.query_filters.public_source" + "public_source": "portality.lib.query_filters.public_source", + + # search on all meta field + "search_all_meta" : "portality.lib.query_filters.search_all_meta", } def without_keys(d, keys): @@ -87,6 +163,29 @@ def tearDown(self): self.app_test.config['QUERY_ROUTE'] = self.OLD_QUERY_ROUTE self.app_test.config['QUERY_FILTERS'] = self.OLD_QUERY_FILTERS + def get_application_with_notes(self): + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + app.add_note("application test", "2015-01-01T00:00:00Z") + app.save() + return app + + def get_journal_with_notes(self): + j = models.Journal() + j.set_id("aabbccdd") + j.set_created("2010-01-01T00:00:00Z") + j.set_last_updated("2012-01-01T00:00:00Z") + j.set_last_manual_update("2014-01-01T00:00:00Z") + j.set_seal(True) + j.set_owner("rama") + j.set_editor_group("worldwide") + j.set_editor("eddie") + j.add_contact("rama", "rama@email.com") + j.add_note("testing", "2015-01-01T00:00:00Z") + j.set_bibjson({"title": "test"}) + j.save() + return j + def test_01_auth(self): with self.app_test.test_client() as t_client: response = t_client.get('/query/journal') # not in the settings above @@ -339,3 +438,156 @@ def test_10_scroll(self): am = models.Article(**res) assert am.publisher_record_id() is None, am.publisher_record_id() + def test_public_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = SEARCH_ALL_QUERY_ROUTE + + self.get_journal_with_notes() + + qsvc = QueryService() + + res = qsvc.search('query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=None, additional_parameters={}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] + + def test_admin_query_notes(self): + + self.get_journal_with_notes() + + maned = models.Account(**AccountFixtureFactory.make_managing_editor_source()) + maned.save(blocking=True) + qsvc = QueryService() + + res = qsvc.search('admin_query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=maned, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_editor_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = SEARCH_ALL_QUERY_ROUTE + + self.get_journal_with_notes() + + editor = models.Account(**AccountFixtureFactory.make_editor_source()) + editor.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=editor.id) + eg = models.EditorGroup(**eg_source) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('editor_query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=editor, additional_parameters={}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] + + def test_associate_editor_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = SEARCH_ALL_QUERY_ROUTE + + self.get_journal_with_notes() + + associate = models.Account(**AccountFixtureFactory.make_assed1_source()) + associate.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=associate.id) + eg = models.EditorGroup(**eg_source) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('associate_query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=associate, additional_parameters={}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] + + def test_associate_editor_application_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = SEARCH_ALL_QUERY_ROUTE + + app = self.get_application_with_notes() + + associate = models.Account(**AccountFixtureFactory.make_assed1_source()) + associate.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=associate.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('associate_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=associate, additional_parameters={}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] + + def test_editor_application_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = SEARCH_ALL_QUERY_ROUTE + + app = self.get_application_with_notes() + + editor = models.Account(**AccountFixtureFactory.make_editor_source()) + editor.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=editor.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('editor_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=editor, additional_parameters={}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] + + def test_admin_application_query_notes(self): + + app = self.get_application_with_notes() + + med = models.Account(**AccountFixtureFactory.make_managing_editor_source()) + med.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=med.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('admin_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=med, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_journal_article_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = self.OLD_QUERY_ROUTE + + self.app_test.config['QUERY_FILTERS'] = self.OLD_QUERY_FILTERS + + self.article11 = models.Article( + **ArticleFixtureFactory.make_article_source(pissn="1111-1111", eissn="2222-2222", doi="10.0000/article-11", + fulltext="https://www.article11.com")) + self.article11.save(blocking=True) + + self.get_journal_with_notes() + + qsvc = QueryService() + + res = qsvc.search('query', 'journal,article', {'query': {'query_string': + {'query': 'application test','default_operator': 'AND'}}, + 'size': 0, 'track_total_hits': True}, account=None, additional_parameters={"ref":"fqw"}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] diff --git a/portality/bll/services/query.py b/portality/bll/services/query.py index 9d2eac87cf..9595dfc801 100644 --- a/portality/bll/services/query.py +++ b/portality/bll/services/query.py @@ -43,18 +43,20 @@ def _get_config_for_search(self, domain, index_type, account): return cfg def _validate_query(self, cfg, query): - validator = cfg.get("query_validator") - if validator is None: - return True + validators = cfg.get("query_validators") + if validators: + for validator in validators: + filters = app.config.get("QUERY_FILTERS", {}) + validator_path = filters.get(validator) + fn = plugin.load_function(validator_path) + if fn is None: + msg = "Unable to load query validator for {x}".format(x=validator) + raise exceptions.ConfigurationException(msg) - filters = app.config.get("QUERY_FILTERS", {}) - validator_path = filters.get(validator) - fn = plugin.load_function(validator_path) - if fn is None: - msg = "Unable to load query validator for {x}".format(x=validator) - raise exceptions.ConfigurationException(msg) + if not fn(query): + return False + return True - return fn(query) def _pre_filter_search_query(self, cfg, query): # now run the query through the filters @@ -197,6 +199,43 @@ def add_must(self, filter): context["must"] = [] context["must"].append(filter) + def get_field_context(self): + """Get query string context""" + context = None + if "query_string" in self.q["query"]: + context = self.q["query"]["query_string"] + + elif "bool" in self.q["query"]: + if "must" in self.q["query"]["bool"]: + context = self.q["query"]["bool"]["must"] + return context + + def add_default_field(self, value: str): + """ Add a default field to the query string, if one is not already present""" + context = self.get_field_context() + + if context: + if isinstance(context, dict): + if "default_field" not in context: + context["default_field"] = value + elif isinstance(context, list): + for item in context: + if "query_string" in item: + if "default_field" not in item["query_string"]: + item["query_string"]["default_field"] = value + break + + def add_should(self, filter, minimum_should_match=1): + self.convert_to_bool() + context = self.q["query"]["bool"] + if "should" not in context: + context["should"] = [] + if isinstance(filter, list): + context["should"].extend(filter) + else: + context["should"].append(filter) + context["minimum_should_match"] = minimum_should_match + def add_must_filter(self, filter): self.convert_to_bool() context = self.q["query"]["bool"] diff --git a/portality/lib/es_data_mapping.py b/portality/lib/es_data_mapping.py index 2f8cf51f26..9e57d929ec 100644 --- a/portality/lib/es_data_mapping.py +++ b/portality/lib/es_data_mapping.py @@ -50,4 +50,9 @@ def create_mapping(struct, mapping_opts, path=()): for struct_name, struct_body in struct.get("structs", {}).items(): result["properties"][struct_name] = create_mapping(struct_body, mapping_opts, path + (struct_name,)) + dotpath = '.'.join(path) + for field, additional_mapping in mapping_opts.get("additional_mappings", {}).items(): + if field.startswith(dotpath) and "." not in field[len(dotpath) + 1:]: + result["properties"][field[len(dotpath):]] = additional_mapping + return result diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 13aed16f39..0bcaf9d916 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -29,18 +29,60 @@ def public_query_validator(q): return True +def non_public_fields_validator(q): + exclude_fields = app.config.get("PUBLIC_QUERY_VALIDATOR__EXCLUDED_FIELDS", {}) + context = q.get_field_context() + if context: + if "default_field" in context: + field_value = context["default_field"] + if field_value in exclude_fields: + return False + return True + # query filters ############### def remove_search_limits(query: dict): return remove_fields(query, ['size', 'from']) + def only_in_doaj(q): q.clear_match_all() q.add_must_filter({"term": {"admin.in_doaj": True}}) return q +def search_all_meta(q): + """Search by all_meta field, which is a concatenation of all the fields in the record""" + q.add_default_field("all_meta") + return q + + +def journal_article_filter(q): + """Search by all_meta field for journal and all fields for article""" + search_text = None + context = q.get_field_context() + if isinstance(context, list): + for item in context: + if "query_string" in item: + search_text = item["query_string"]["query"] + if "default_field" in item["query_string"]: + return q + elif isinstance(context, dict): + if "query_string" in context: + search_text = context["query_string"]["query"] + if "default_field" in context["query_string"]: + return q + q.convert_to_bool() + if search_text: + filter = [{"bool": {"must": [{"term": {"es_type.exact": "article"}}, + {"query_string": {"default_field": "*", "query": search_text}}]}}, + {"bool": {"must": [{"term": {"es_type.exact": "journal"}}, + {"query_string": {"default_field": "all_meta", "query": search_text}}]}}] + q.add_should(filter) + return q + + def owner(q): q.clear_match_all() q.add_must_filter({"term" : {"admin.owner.exact" : current_user.id}}) diff --git a/portality/models/v2/application.py b/portality/models/v2/application.py index 2b0d523720..58fff8d19d 100644 --- a/portality/models/v2/application.py +++ b/portality/models/v2/application.py @@ -244,14 +244,9 @@ class AllPublisherApplications(DomainObject): MAPPING_OPTS = { "dynamic": None, - "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], - "exceptions": { - "admin.notes.note": { - "type": "text", - "index": False, - # "include_in_all": False # Removed in es6 fixme: do we need to look at copy_to for the mapping? - } - } + "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]), + "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"], + "additional_mappings": app.config["ADMIN_NOTES_INDEX_ONLY_FIELDS"] } diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index efa6aa53e4..81e28d86dd 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -580,6 +580,15 @@ def delete_selected(cls, query, articles=False, snapshot_journals=True, snapshot # finally issue a delete request against the journals cls.delete_by_query(query) + @classmethod + def add_mapping_extensions(cls, default_mappings: dict): + default_mappings_copy = deepcopy(default_mappings) + mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS") + for key, value in mapping_extensions.items(): + if key in default_mappings_copy: + default_mappings_copy[key] = {**default_mappings_copy[key], **value} + return default_mappings_copy + def all_articles(self): from portality.models import Article return Article.find_by_issns(self.known_issns()) @@ -907,14 +916,9 @@ def _calculate_has_apc(self): MAPPING_OPTS = { "dynamic": None, - "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], - "exceptions": { - "admin.notes.note": { - "type": "text", - "index": False, - # "include_in_all": False # Removed in es6 fixme: do we need to look at copy_to for the mapping? - } - } + "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]), + "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"], + "additional_mappings": app.config["ADMIN_NOTES_INDEX_ONLY_FIELDS"] } diff --git a/portality/settings.py b/portality/settings.py index b52d33dd3f..d561e977d8 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.4.7" +DOAJ_VERSION = "6.5.0" API_VERSION = "3.0.1" ###################################### @@ -629,6 +629,19 @@ } } +# Extension Map from dataobj coercion declarations to ES mappings. +# This is useful when some extensions required for some objects additional to defaults. +# ~~->DataObj:Library~~ +# ~~->Seamless:Library~~ +DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS = { + "unicode": {"copy_to": ["all_meta"]}, + "str": {"copy_to": ["all_meta"]}, + "unicode_upper": {"copy_to": ["all_meta"]}, + "unicode_lower": {"copy_to": ["all_meta"]}, + "issn": {"copy_to": ["all_meta"]}, + "url": {"copy_to": ["all_meta"]} +} + # TODO: we may want a big-type and little-type setting DEFAULT_INDEX_SETTINGS = \ { @@ -699,8 +712,8 @@ "journal" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", - "query_filters" : ["only_in_doaj", "last_update_fallback"], + "query_validators" : ["non_public_fields_validator", "public_query_validator"], + "query_filters" : ["only_in_doaj", "last_update_fallback", "search_all_meta"], "result_filters" : ["public_result_filter"], "dao" : "portality.models.Journal", # ~~->Journal:Model~~ "required_parameters" : {"ref" : ["ssw", "public_journal", "subject_page"]} @@ -709,7 +722,7 @@ "article" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", + "query_validators" : ["non_public_fields_validator", "public_query_validator"], "query_filters" : ["only_in_doaj"], "result_filters" : ["public_result_filter"], "dao" : "portality.models.Article", # ~~->Article:Model~~ @@ -720,8 +733,8 @@ "journal,article" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", - "query_filters" : ["only_in_doaj", "strip_facets", "es_type_fix"], + "query_validators" : ["non_public_fields_validator", "public_query_validator"], + "query_filters" : ["only_in_doaj", "strip_facets", "es_type_fix", "journal_article_filter"], "result_filters" : ["public_result_filter", "add_fqw_facets", "fqw_back_compat"], "dao" : "portality.models.JournalArticle", # ~~->JournalArticle:Model~~ "required_parameters" : {"ref" : ["fqw"]} @@ -732,7 +745,8 @@ "journal" : { "auth" : True, "role" : "publisher", - "query_filters" : ["owner", "only_in_doaj"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["owner", "only_in_doaj", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.Journal" # ~~->Journal:Model~~ }, @@ -740,7 +754,8 @@ "applications" : { "auth" : True, "role" : "publisher", - "query_filters" : ["owner", "not_update_request"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["owner", "not_update_request", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.AllPublisherApplications" # ~~->AllPublisherApplications:Model~~ }, @@ -748,7 +763,8 @@ "update_requests" : { "auth" : True, "role" : "publisher", - "query_filters" : ["owner", "update_request"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["owner", "update_request", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.Application" # ~~->Application:Model~~ } @@ -811,14 +827,16 @@ "journal" : { "auth" : True, "role" : "associate_editor", - "query_filters" : ["associate"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["associate", "search_all_meta"], "dao" : "portality.models.Journal" # ~~->Journal:Model~~ }, # ~~->AssEdApplicationQuery:Endpoint~~ "suggestion" : { "auth" : True, "role" : "associate_editor", - "query_filters" : ["associate"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["associate", "search_all_meta"], "dao" : "portality.models.Application" # ~~->Application:Model~~ } }, @@ -827,14 +845,16 @@ "journal" : { "auth" : True, "role" : "editor", - "query_filters" : ["editor"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["editor", "search_all_meta"], "dao" : "portality.models.Journal" # ~~->Journal:Model~~ }, # ~~->EditorApplicationQuery:Endpoint~~ "suggestion" : { "auth" : True, "role" : "editor", - "query_filters" : ["editor"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["editor", "search_all_meta"], "dao" : "portality.models.Application" # ~~->Application:Model~~ } }, @@ -852,7 +872,8 @@ "journal" : { "auth" : False, "role" : None, - "query_filters" : ["only_in_doaj", "public_source"], + "query_validators": ["non_public_fields_validator"], + "query_filters" : ["only_in_doaj", "public_source", "search_all_meta"], "dao" : "portality.models.Journal", # ~~->Journal:Model~~ "required_parameters" : None }, @@ -860,7 +881,8 @@ "application" : { "auth" : True, "role" : None, - "query_filters" : ["owner", "private_source"], + "query_validators": ["non_public_fields_validator"], + "query_filters" : ["owner", "private_source", "search_all_meta"], "dao" : "portality.models.Suggestion", # ~~->Application:Model~~ "required_parameters" : None } @@ -880,6 +902,7 @@ QUERY_FILTERS = { # sanitisers "public_query_validator" : "portality.lib.query_filters.public_query_validator", + "non_public_fields_validator" : "portality.lib.query_filters.non_public_fields_validator", # query filters "only_in_doaj" : "portality.lib.query_filters.only_in_doaj", @@ -892,6 +915,8 @@ "last_update_fallback" : "portality.lib.query_filters.last_update_fallback", "not_update_request" : "portality.lib.query_filters.not_update_request", "who_current_user" : "portality.lib.query_filters.who_current_user", # ~~-> WhoCurrentUser:Query ~~ + "search_all_meta" : "portality.lib.query_filters.search_all_meta", # ~~-> SearchAllMeta:Query ~~ + "journal_article_filter" : "portality.lib.query_filters.journal_article_filter", # ~~-> JournalArticleFilter:Query ~~ # result filters "public_result_filter": "portality.lib.query_filters.public_result_filter", @@ -903,6 +928,55 @@ "private_source": "portality.lib.query_filters.private_source", "public_source": "portality.lib.query_filters.public_source", } +# Exclude the fields that doesn't want to be searched by public queries +# This is part of non_public_fields_validator. +PUBLIC_QUERY_VALIDATOR__EXCLUDED_FIELDS = [ + "admin.notes.note", + "admin.notes.id", + "admin.notes.author_id" +] + +ADMIN_NOTES_INDEX_ONLY_FIELDS = { + "all_meta" : { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + } +} + +ADMIN_NOTES_SEARCH_MAPPING = { + "admin.notes.id": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + }, + "admin.notes.note": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + }, + "admin.notes.author_id": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + } +} #################################################### # Autocomplete diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index 3fcdbc3ab7..c43bf55d5e 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -61,7 +61,8 @@ $.extend(true, doaj, { {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, - {'display':'Journal: alternative title','field':'bibjson.alternative_title'} + {'display':'Journal: alternative title','field':'bibjson.alternative_title'}, + {'display':'Notes','field':'admin.notes.note'}, ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index f6d6aef096..e612a2c7f5 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -307,7 +307,8 @@ $.extend(true, doaj, { {'display':'ISSN', 'field':'index.issn.exact'}, {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, - {'display':'Publisher','field':'bibjson.publisher.name'} + {'display':'Publisher','field':'bibjson.publisher.name'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index 22cc532911..cea56910fe 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -71,7 +71,8 @@ $.extend(true, doaj, { {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, - {'display':'Journal: Alternative Title','field':'bibjson.alternative_title'} + {'display':'Journal: Alternative Title','field':'bibjson.alternative_title'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/setup.py b/setup.py index 49da5a8c38..ce7e2121e4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='6.4.7', + version='6.5.0', packages=find_packages(), install_requires=[ "awscli==1.20.50",