From b5de043ed2c357967c4ca52ca978a2f04d7e113a Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Thu, 2 Feb 2023 16:25:40 +0530 Subject: [PATCH 001/184] Upload the journals as individual packages --- portality/models/preservation.py | 4 ++ portality/tasks/preservation.py | 62 +++++++++++++------ .../templates/publisher/preservation.html | 4 ++ 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/portality/models/preservation.py b/portality/models/preservation.py index c8042e1465..60c19e6e04 100644 --- a/portality/models/preservation.py +++ b/portality/models/preservation.py @@ -98,6 +98,10 @@ def no_files_articles(self, articles_list): if articles_list is not None and len(articles_list) > 0: self.data["articles_info"]["no_files_articles"] = ", ".join(articles_list) + def uploaded_journals(self, uploaded_journals): + if uploaded_journals is not None and len(uploaded_journals) > 0: + self.data["articles_info"]["uploaded_journals"] = ", ".join(uploaded_journals) + @classmethod def by_owner(cls, owner, size=10): q = OwnerFileQuery(owner) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index 727adc5ac7..00f9c644a6 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -126,11 +126,15 @@ def __init__(self): self.__unbagged_articles = [] self.__not_found_articles = [] self.__no_files_articles = [] + self.__uploaded_journals = [] self.has_errors = False def add_successful_article(self, article: ArticlePackage): self.__successful_articles.append(os.path.basename(article.article_dir)) + def add_uploaded_journal(self, journal_package): + self.__uploaded_journals.append(journal_package) + def add_unowned_articles(self, article: ArticlePackage): self.has_errors = True self.__unowned_articles.append(os.path.basename(article.article_dir)) @@ -168,6 +172,9 @@ def not_found_articles(self): def no_files_articles(self): return self.__no_files_articles + def uploaded_journals(self): + return self.__uploaded_journals + def get_count(self): return len(self.__successful_articles) + \ len(self.__unowned_articles) + \ @@ -243,24 +250,39 @@ def run(self): job.add_audit_message("Create Package structure") articles_list = preserv.create_package_structure() - self.save_articles_list(articles_list, preserve_model) + app.logger.debug("Created package structure") if len(articles_list.successful_articles()) > 0: - package = PreservationPackage(preserv.preservation_dir, job.user) - job.add_audit_message("Create preservation package") - tar_file = package.create_package() - app.logger.debug(f"Created tar file {tar_file}") - - job.add_audit_message("Create shasum") - sha256 = package.sha256() - - job.add_audit_message("Upload package") - response = package.upload_package(sha256) - app.logger.debug(f"Uploaded. Response{response.text}") + # Each subdirectory is a jornal and the directory name is ISSN of the journal + # iterate through the directories and upload each journal as an individual package + dirs = os.listdir(preserv.preservation_dir) + upload_failed = False + for sub_dir in dirs: + tar_file = os.path.join(preserv.preservation_dir, sub_dir) + package = PreservationPackage(tar_file, job.user) + job.add_audit_message("Create preservation package") + tar_file = package.create_package() + app.logger.debug(f"Created tar file {tar_file}") + + job.add_audit_message("Create shasum") + sha256 = package.sha256() + + job.add_audit_message("Upload package") + response = package.upload_package(sha256) + app.logger.debug(f"Uploaded. Response{response.text}") + + job.add_audit_message("Validate response") + self.validate_response(response, tar_file, sha256, preserve_model) + + if preserve_model.status == 'failed': + upload_failed = True + break + else: + articles_list.add_uploaded_journal(package.tar_file_name) - job.add_audit_message("Validate response") - self.validate_response(response, tar_file, sha256, preserve_model) + if not upload_failed: + preserve_model.uploaded_to_ia() # Check if the only few articles are successful if articles_list.is_partial_success(): @@ -278,6 +300,8 @@ def run(self): preserve_model.failed(FailedReasons.no_valid_article_available) preserve_model.save() + self.save_articles_list(articles_list, preserve_model) + except (PreservationException, Exception) as exp: # ~~-> PreservationException:Exception~~ preserve_model.failed(str(exp)) @@ -305,6 +329,8 @@ def save_articles_list(self, articles_list: ArticlesList, model: PreservationSta model.unbagged_articles(articles_list.unbagged_articles()) if len(articles_list.no_files_articles()) > 0: model.no_files_articles(articles_list.no_files_articles()) + if len(articles_list.uploaded_journals()) > 0: + model.uploaded_journals(articles_list.uploaded_journals()) model.save() def cleanup(self): @@ -345,8 +371,7 @@ def validate_response(self, response, tar_file, sha256, model): if res_filename and res_filename == tar_file: if res_shasum and res_shasum == sha256: - app.logger.info("successfully uploaded") - model.uploaded_to_ia() + app.logger.info("successfully uploaded " + tar_file) else: model.failed(FailedReasons.checksum_doesnot_match) else: @@ -379,7 +404,7 @@ def validate_response(self, response, tar_file, sha256, model): model.save() else: - app.logger.error(f"Upload failed {response.text}") + app.logger.error(f"Upload failed for {tar_file}. Reason - {response.text}") model.failed(response.text) model.save() @@ -680,7 +705,8 @@ class PreservationPackage: def __init__(self, directory, owner): self.package_dir = directory - self.tar_file = self.package_dir + ".tar.gz" + created_time = dates.format(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + self.tar_file = directory + "_" + created_time + ".tar.gz" self.tar_file_name = os.path.basename(self.tar_file) self.__owner = owner diff --git a/portality/templates/publisher/preservation.html b/portality/templates/publisher/preservation.html index b99c8e6c91..081f716f2a 100644 --- a/portality/templates/publisher/preservation.html +++ b/portality/templates/publisher/preservation.html @@ -144,6 +144,10 @@

History of uploads (showing last {{previous|length}})

{% endif %} {% if file.status == "uploaded" or file.status == "partial" and file.articles_info %} - - -
-

- Added {{journal.created_timestamp.strftime("%e %B %Y")}} - {% if journal.last_manual_update_timestamp and - dates.format(journal.last_manual_update_timestamp) != dates.DEFAULT_TIMESTAMP_VAL - %} - • Updated {{journal.last_manual_update_timestamp.strftime("%e %B %Y")}} + {% else %} + {{ policy }} + {% endif %} + + {% endfor %} + + + {% endif %} -

-
- - - {% include "includes/_hotjar.html" %} - -{% endblock %} -{% block extra_js_bottom %} + {% if bibjson.pid_scheme %} +
+ +
+

Permanent article identifier:

+
    + {% for identifier in bibjson.pid_scheme %} +
  • {{ identifier }}
    • + {% endfor %} +
+
+
+ {% endif %} + + +
+

Journal metadata

+ +
+ +
+ {% if bibjson.publisher_name %} +
Publisher
+
+ {% set source = search_query_source(terms=[{"bibjson.publisher.name.exact": [bibjson.publisher_name]}]) %} + {{ bibjson.publisher_name }} + {% if bibjson.publisher_country %}, {{ bibjson.publisher_country_name() }}{% endif %} +
+ {% endif %} - + {% if bibjson.language %} +
Manuscripts accepted in
+
{{ bibjson.language_name()|join(", ") }}
+ {% endif %} +
+
+ +
+ +
+ {% for path, code in bibjson.lcc_paths_and_codes() %} + {% if loop.index0 == 0 %} +
LCC subjects + + Look up the Library of Congress Classification Outline + +
{% endif %} +
+ {% set source = search_query_source(terms=[{"index.schema_codes_tree.exact": [code]}]) %} + + {{ path }} + +
+ {% endfor %} + + {% if bibjson.keywords %} +
Keywords
+
+ {% for keyword in bibjson.keywords %} + {% set source = search_query_source(query_string=keyword) %} + {{ keyword }} + {% endfor %} +
+ {% endif %} +
+
+
+ + - {% include "_edges_common_js.html" %} - +{% endblock %} -{% endblock extra_js_bottom %} diff --git a/portality/templates/doaj/toc_articles.html b/portality/templates/doaj/toc_articles.html new file mode 100644 index 0000000000..2446a355df --- /dev/null +++ b/portality/templates/doaj/toc_articles.html @@ -0,0 +1,44 @@ +{% extends "layouts/toc_base.html" %} + +{% block toc_content %} + +
+
+
+
+
+ +{% endblock %} + +{% block extra_js_bottom %} + + + + {% include "_edges_common_js.html" %} + + +{% endblock extra_js_bottom %} diff --git a/portality/templates/layouts/toc_base.html b/portality/templates/layouts/toc_base.html new file mode 100644 index 0000000000..0c27ac504d --- /dev/null +++ b/portality/templates/layouts/toc_base.html @@ -0,0 +1,147 @@ +{% extends "layouts/public_base.html" %} + +{% block body_class %} + journal-details +{% endblock %} + +{% block page_title %}{% include "doaj/includes/_journal_meta_title.html" %}{% endblock %} +{% block meta_og_title %}{% include "doaj/includes/_journal_meta_title.html" %}{% endblock %} +{% block meta_twitter_title %}{% include "doaj/includes/_journal_meta_title.html" %}{% endblock %} +{%- block meta_description -%}{% include "doaj/includes/_journal_meta_description.html" %}{%- endblock -%} +{%- block meta_og_description -%}{% include "doaj/includes/_journal_meta_description.html" %}{%- endblock -%} +{%- block meta_twitter_description -%}{% include "doaj/includes/_journal_meta_description.html" %}{%- endblock -%} + +{% block content %} + + +
+
+ {% if journal.last_manually_updated_since(days=30) %} + + + Updated recently + + {% endif %} +

+ + {{ bibjson.title }} + {% if bibjson.alternative_title %} + + {% endif %} + + {%- set seal = journal.has_seal() -%} + {%- if seal -%} +
+ + {% include "includes/svg/seal.svg" %} +

This journal has been awarded the DOAJ Seal.

+ +
+ {%- endif %} +

+

+ + + {# this next bit has to be all on one line so that the spacing is correct #} + {% if bibjson.pissn %}{{bibjson.pissn}} (Print){% endif %}{% if bibjson.eissn %}{% if bibjson.pissn %} / {% endif %}{{bibjson.eissn}} (Online){% endif %} + +

+ + {% if bibjson.discontinued_date %} +

Ceased publication on {{ bibjson.discontinued_datestamp.strftime("%d %B %Y") }}

+ {% endif %} + + {% set past = journal.get_past_continuations() %} + {% if past %} +

Continues + {% for p in past %} + {% set bibjson = p.bibjson() %} + {% if bibjson.issns()|length > 0 %} + {% if p.is_in_doaj() %} + {{ bibjson.title }} + {% else %} + {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in DOAJ) + {% endif %} + {% endif %} + {% if not loop.last %}; {% endif %} + {% endfor %} +

+ {% endif %} + + {% set future = journal.get_future_continuations() %} + {% if future %} +

Continued by + {% for f in future %} + {% set bibjson = f.bibjson() %} + {% if bibjson.issns()|length > 0 %} + {% if f.is_in_doaj() %} + {{ bibjson.title }} + {% else %} + {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in DOAJ) + {% endif %} + {% endif %} + {% if not loop.last %}; {% endif %} + {% endfor %} +

+ {% endif %} + + +
+ +
+ + +
+ {% block toc_content %} {% endblock %} +
+
+ +
+

+ Added {{journal.created_timestamp.strftime("%e %B %Y")}} + {% if journal.last_manual_update_timestamp and + dates.format(journal.last_manual_update_timestamp) != dates.DEFAULT_TIMESTAMP_VAL + %} + • Updated {{journal.last_manual_update_timestamp.strftime("%e %B %Y")}} + {% endif %} +

+
+ +
+ {% include "includes/_hotjar.html" %} + +{% endblock %} diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 5ac2f67f69..80ff0b2195 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -249,16 +249,7 @@ def autocomplete(doc_type, field_name): # http://flask.pocoo.org/docs/security/#json-security -@blueprint.route("/toc/") -@blueprint.route("/toc//") -@blueprint.route("/toc///") -def toc(identifier=None, volume=None, issue=None): - """ Table of Contents page for a journal. identifier may be the journal id or an issn """ - # If this route is changed, update JOURNAL_TOC_URL_FRAG in settings.py (partial ToC page link for journal CSV) - - journal = None - issn_ref = False - +def find_toc_journal_by_identifier(identifier): if identifier is None: abort(404) @@ -274,44 +265,40 @@ def toc(identifier=None, volume=None, issue=None): if journal is None: abort(400) - issn_ref = True # just a flag so we can check if we were requested via issn + return journal + elif len(identifier) == 32: js = models.Journal.pull(identifier) # Returns None on fail if js is None or not js.is_in_doaj(): abort(404) - journal = js - else: - abort(400) + return js + + abort(400) - # get the bibjson record that we're going to render - bibjson = journal.bibjson() - # The issn we are using to build the TOC - issn = bibjson.get_preferred_issn() +def is_issn_by_identifier(identifier): + return len(identifier) == 9 - # now redirect to the canonical E-ISSN if one is available +def find_correct_redirect_identifier(identifier, bibjson) -> str: + """ + return None if identifier is correct and no redirect is needed - if issn_ref: # the journal is referred to by an ISSN + :param identifier: + :param bibjson: + :return: + """ + if is_issn_by_identifier(identifier): # the journal is referred to by an ISSN # if there is an E-ISSN (and it's not the one in the request), redirect to it eissn = bibjson.get_one_identifier(bibjson.E_ISSN) if eissn and identifier != eissn: - return redirect(url_for('doaj.toc', identifier=eissn, volume=volume, issue=issue), 301) + return eissn # if there's no E-ISSN, but there is a P-ISSN (and it's not the one in the request), redirect to the P-ISSN if not eissn: pissn = bibjson.get_one_identifier(bibjson.P_ISSN) if pissn and identifier != pissn: - return redirect(url_for('doaj.toc', identifier=pissn, volume=volume, issue=issue), 301) - - # Add the volume and issue to query if present in path - if volume: - filters = [dao.Facetview2.make_term_filter('bibjson.journal.volume.exact', volume)] - if issue: - filters += [dao.Facetview2.make_term_filter('bibjson.journal.number.exact', issue)] - q = dao.Facetview2.make_query(filters=filters) - - return redirect(url_for('doaj.toc', identifier=issn) + '?source=' + dao.Facetview2.url_encode_query(q)) + return pissn # The journal has neither a PISSN or an EISSN. Yet somehow # issn_ref is True, the request was referring to the journal @@ -328,22 +315,52 @@ def toc(identifier=None, volume=None, issue=None): if not issn: issn = bibjson.get_one_identifier(bibjson.P_ISSN) if issn: - return redirect(url_for('doaj.toc', identifier=issn, volume=volume, issue=issue), 301) + return issn # let it continue loading if we only have the hex UUID for the journal (no ISSNs) # and the user is referring to the toc page via that ID - # get the continuations for this journal, future and past - future_journals = journal.get_future_continuations() - past_journals = journal.get_past_continuations() +@blueprint.route("/toc/") +def toc(identifier=None): + """ Table of Contents page for a journal. identifier may be the journal id or an issn """ + # If this route is changed, update JOURNAL_TOC_URL_FRAG in settings.py (partial ToC page link for journal CSV) + + journal = find_toc_journal_by_identifier(identifier) + bibjson = journal.bibjson() + real_identifier = find_correct_redirect_identifier(identifier, bibjson) + if real_identifier: + return redirect(url_for('doaj.toc', identifier=real_identifier), 301) + else: + # now render all that information + return render_template('doaj/toc.html', journal=journal, bibjson=bibjson ) + + +@blueprint.route("/toc/articles/") +@blueprint.route("/toc/articles//") +@blueprint.route("/toc/articles///") +def toc_articles(identifier=None, volume=None, issue=None): + journal = find_toc_journal_by_identifier(identifier) + bibjson = journal.bibjson() + real_identifier = find_correct_redirect_identifier(identifier, bibjson) + if real_identifier: + return redirect(url_for('doaj.toc_articles', identifier=real_identifier, + volume=volume, issue=issue), 301) + else: + + if is_issn_by_identifier(identifier) and volume: + filters = [dao.Facetview2.make_term_filter('bibjson.journal.volume.exact', volume)] + if issue: + filters += [dao.Facetview2.make_term_filter('bibjson.journal.number.exact', issue)] + q = dao.Facetview2.make_query(filters=filters) + + # The issn we are using to build the TOC + issn = bibjson.get_preferred_issn() + return redirect(url_for('doaj.toc', identifier=issn) + + '?source=' + dao.Facetview2.url_encode_query(q)) - # extract the bibjson, which is what the template is after, and whether the record is in doaj - #future = [j.bibjson() j for j in future_journals] - #past = [j.bibjson() for j in past_journals] + # now render all that information + return render_template('doaj/toc_articles.html', journal=journal, bibjson=bibjson ) - # now render all that information - return render_template('doaj/toc.html', journal=journal, bibjson=bibjson, future=future_journals, past=past_journals, - toc_issns=journal.bibjson().issns()) #~~->Article:Page~~ From 5f2c9f91009c44842e1c200fd9333d431e586605 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 2 Aug 2023 14:54:12 +0100 Subject: [PATCH 014/184] rename share toc test cases --- doajtest/unit/test_toc.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doajtest/unit/test_toc.py b/doajtest/unit/test_toc.py index b58b20445f..dafe64ac31 100644 --- a/doajtest/unit/test_toc.py +++ b/doajtest/unit/test_toc.py @@ -5,7 +5,7 @@ from portality.util import url_for -def test_toc_uses_both_issns_when_available(app_test, url_name): +def _test_toc_uses_both_issns_when_available(app_test, url_name): j = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) pissn = j.bibjson().first_pissn eissn = j.bibjson().first_eissn @@ -20,7 +20,7 @@ def test_toc_uses_both_issns_when_available(app_test, url_name): assert eissn in response.data.decode() -def toc_correctly_uses_pissn(app_test, url_name): +def _test_toc_correctly_uses_pissn(app_test, url_name): j = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) pissn = j.bibjson().first_pissn # remove eissn @@ -35,7 +35,7 @@ def toc_correctly_uses_pissn(app_test, url_name): assert pissn in response.data.decode() -def toc_correctly_uses_eissn(app_test, url_name): +def _test_toc_correctly_uses_eissn(app_test, url_name): j = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) eissn = j.bibjson().first_eissn # remove pissn @@ -131,21 +131,21 @@ def test_02_toc_requirements(self): assert a.data['index']['date_toc_fv_month'] == a.data['index']['date'] == "1991-01-01T00:00:00Z" def test_03_toc_uses_both_issns_when_available(self): - test_toc_uses_both_issns_when_available(self.app_test, 'doaj.toc') + _test_toc_uses_both_issns_when_available(self.app_test, 'doaj.toc') def test_04_toc_correctly_uses_pissn(self): - toc_correctly_uses_pissn(self.app_test, 'doaj.toc') + _test_toc_correctly_uses_pissn(self.app_test, 'doaj.toc') def test_05_toc_correctly_uses_eissn(self): - toc_correctly_uses_eissn(self.app_test, 'doaj.toc') + _test_toc_correctly_uses_eissn(self.app_test, 'doaj.toc') class TestTOCArticles(DoajTestCase): def test_03_toc_uses_both_issns_when_available(self): - test_toc_uses_both_issns_when_available(self.app_test, 'doaj.toc_articles') + _test_toc_uses_both_issns_when_available(self.app_test, 'doaj.toc_articles') def test_04_toc_correctly_uses_pissn(self): - toc_correctly_uses_pissn(self.app_test, 'doaj.toc_articles') + _test_toc_correctly_uses_pissn(self.app_test, 'doaj.toc_articles') def test_05_toc_correctly_uses_eissn(self): - toc_correctly_uses_eissn(self.app_test, 'doaj.toc_articles') + _test_toc_correctly_uses_eissn(self.app_test, 'doaj.toc_articles') From bddce5a206348d18f302318bfd61d2231e42a70d Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 4 Aug 2023 09:53:58 +0100 Subject: [PATCH 015/184] make differentTo validator case insensitive --- portality/static/js/application_form.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/static/js/application_form.js b/portality/static/js/application_form.js index 178aef09c4..3944845669 100644 --- a/portality/static/js/application_form.js +++ b/portality/static/js/application_form.js @@ -844,8 +844,8 @@ window.Parsley.addValidator("optionalIf", { }); window.Parsley.addValidator("differentTo", { - validateString : function(value, requirement) { - return (!value || ($("[name = " + requirement + "]")).val() !== value); + validateString : function(value, requirement, message) { + return (!value || ($("[name = " + requirement + "]")).val().toLowerCase() !== value.toLowerCase()); }, messages: { en: 'Value of this field and %s field must be different' From 05f67c761ae0d05f368803a058bbaa2f3046d5ce Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 4 Aug 2023 09:54:32 +0100 Subject: [PATCH 016/184] add DifferentTo validator to Publishers name and Institution name fields, make sure message is passed correctly --- portality/forms/application_forms.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py index 0065b765fc..b088a9786c 100644 --- a/portality/forms/application_forms.py +++ b/portality/forms/application_forms.py @@ -444,7 +444,8 @@ class FieldDefinitions: "label": "Publisher’s name", "input": "text", "validate": [ - {"required": {"message": "Enter the name of the journal’s publisher"}} + {"required": {"message": "Enter the name of the journal’s publisher"}}, + {"different_to": {"field": "institution_name", "message": "This field must be different than 'Society or institution’s name'"}} # ~~^-> DifferetTo:FormValidator~~ ], "widgets": [ "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ @@ -505,6 +506,10 @@ class FieldDefinitions: "a society or other type of institution, enter that here."], "placeholder": "Type or select the society or institution’s name" }, + "validate": [ + {"different_to": {"field": "publisher_name", + "message": "This field must be different than 'Publisher’s name'"}} # ~~^-> DifferetTo:FormValidator~~ + ], "widgets": [ "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ {"autocomplete": {"type" : "journal", "field": "bibjson.institution.name.exact"}}, @@ -2743,6 +2748,7 @@ class DifferentToBuilder: @staticmethod def render(settings, html_attrs): html_attrs["data-parsley-different-to"] = settings.get("field") + html_attrs["data-parsley-different-to-message"] = "

" + settings.get("message") + "

" @staticmethod def wtforms(field, settings): From 5fce92245d3cf2aaa5de59e8918feeddb17fd15e Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 4 Aug 2023 10:00:16 +0100 Subject: [PATCH 017/184] add functional test --- doajtest/testbook/new_application_form/publishers_form.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doajtest/testbook/new_application_form/publishers_form.yml b/doajtest/testbook/new_application_form/publishers_form.yml index 07f1c2e40a..72b4de2316 100644 --- a/doajtest/testbook/new_application_form/publishers_form.yml +++ b/doajtest/testbook/new_application_form/publishers_form.yml @@ -114,6 +114,13 @@ tests: - step: Click the X next to it results: - It disappears + - step: Enter the same value in Publisher's Name and Publisher's Country + - step: Click Next + results: + - "You see the error message: The value of this field and the Publisher's Country field must be different." + - step: Repeat previous step checking if the validator is case insensitive (e.g. "Publisher's name" and "publisher's Name") + results: + - The error message is still displayed - step: Repeat steps X to X for the Publisher's Name, Country, Society name, Country - step: Delete the values from Society name and country, click Next results: From e417225cd3d7d22cf9164938b16ff5ba51c1e941 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Fri, 11 Aug 2023 13:43:28 +0530 Subject: [PATCH 018/184] Fixed some issues with admin notes --- .../administrative_search/journals.yml | 44 +++++++++ portality/bll/services/query.py | 32 ++++--- portality/lib/query_filters.py | 13 +++ portality/models/v1/journal.py | 28 +----- portality/models/v2/application.py | 32 +------ portality/models/v2/journal.py | 41 +++------ portality/models/v2/shared_structs.py | 3 +- portality/settings.py | 91 +++++++++++++++---- .../js/edges/admin.applications.edge.js | 2 +- .../js/edges/admin.update_requests.edge.js | 2 +- .../js/edges/associate.applications.edge.js | 3 +- .../js/edges/associate.journals.edge.js | 3 +- .../js/edges/editor.groupapplications.edge.js | 3 +- .../js/edges/editor.groupjournals.edge.js | 3 +- 14 files changed, 179 insertions(+), 121 deletions(-) diff --git a/doajtest/testbook/administrative_search/journals.yml b/doajtest/testbook/administrative_search/journals.yml index 065005e40b..e6882e5075 100644 --- a/doajtest/testbook/administrative_search/journals.yml +++ b/doajtest/testbook/administrative_search/journals.yml @@ -153,3 +153,47 @@ tests: - step: Search for the journal from api search with same words given in the note results: - The journal record is not displayed in the search results + +- title: Test notes not searchable for public + context: + role: anonymous + steps: + - step: Search for the keyword which is given in a notes. + results: No results displayed + - step: Modify the search url to add the notes keyword and the default field 'admin.notes.note' and then hit the url + results: Error message displayed + +- title: Test notes searchable for Editor + context: + role: Editor + steps: + - step: Make sure the editor group is assigned for the journal / application which is under test and assigned to the user + - step: Go to the tab 'Your group’s journals' + results: Group's journals page displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Your group’s applications' + results: Group's applications page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results + - step: Go to the tab 'Journals assigned to you' + results: Journals assigned to you page displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Applications assigned to you' + results: Applications assigned to you page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results + +- title: Test notes searchable for Associate Editor + context: + role: AssociateEditor + steps: + - step: Go to the tab 'Journals assigned to you' + results: 'Journals assigned to you page' displayed + - step: Search for the keyword which is given in a notes in a journal + results: The journal record is displayed in the search results + - step: Go to the tab 'Applications assigned to you' + results: 'Applications assigned to you' page displayed + - step: Search for the keyword which is given in a notes in an application + results: The application is displayed in the search results diff --git a/portality/bll/services/query.py b/portality/bll/services/query.py index 00ad129ae7..f8367e7844 100644 --- a/portality/bll/services/query.py +++ b/portality/bll/services/query.py @@ -43,18 +43,20 @@ def _get_config_for_search(self, domain, index_type, account): return cfg def _validate_query(self, cfg, query): - validator = cfg.get("query_validator") - if validator is None: - return True + validators = cfg.get("query_validators") + if validators: + for validator in validators: + filters = app.config.get("QUERY_FILTERS", {}) + validator_path = filters.get(validator) + fn = plugin.load_function(validator_path) + if fn is None: + msg = "Unable to load query validator for {x}".format(x=validator) + raise exceptions.ConfigurationException(msg) - filters = app.config.get("QUERY_FILTERS", {}) - validator_path = filters.get(validator) - fn = plugin.load_function(validator_path) - if fn is None: - msg = "Unable to load query validator for {x}".format(x=validator) - raise exceptions.ConfigurationException(msg) + if not fn(query): + return False + return True - return fn(query) def _pre_filter_search_query(self, cfg, query): # now run the query through the filters @@ -197,8 +199,8 @@ def add_must(self, filter): context["must"] = [] context["must"].append(filter) - def add_default_field(self, value: str): - """ Add a default field to the query string, if one is not already present""" + def get_field_context(self): + """Get query string context""" context = None if "query_string" in self.q["query"]: context = self.q["query"]["query_string"] @@ -206,6 +208,11 @@ def add_default_field(self, value: str): elif "bool" in self.q["query"]: if "must" in self.q["query"]["bool"]: context = self.q["query"]["bool"]["must"] + return context + + def add_default_field(self, value: str): + """ Add a default field to the query string, if one is not already present""" + context = self.get_field_context() if context: if isinstance(context, dict): @@ -216,6 +223,7 @@ def add_default_field(self, value: str): if "query_string" in item: if "default_field" not in item["query_string"]: item["query_string"]["default_field"] = value + break def add_must_filter(self, filter): self.convert_to_bool() diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 51ea3db5f7..92e21ca999 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -29,22 +29,35 @@ def public_query_validator(q): return True +def non_public_fields_validator(q): + exclude_fields = app.config.get("EXCLUDED_FIELDS", {}) + context = q.get_field_context() + if context: + if "default_field" in context: + field_value = context["default_field"] + if field_value in exclude_fields: + return False + return True + # query filters ############### def remove_search_limits(query: dict): return remove_fields(query, ['size', 'from']) + def only_in_doaj(q): q.clear_match_all() q.add_must_filter({"term": {"admin.in_doaj": True}}) return q + def search_all_meta(q): """Search by all_meta field, which is a concatenation of all the fields in the record""" q.add_default_field("all_meta") return q + def owner(q): q.clear_match_all() q.add_must_filter({"term" : {"admin.owner.exact" : current_user.id}}) diff --git a/portality/models/v1/journal.py b/portality/models/v1/journal.py index 10e007159e..dc0f98db92 100644 --- a/portality/models/v1/journal.py +++ b/portality/models/v1/journal.py @@ -1282,32 +1282,10 @@ def get_preferred_issn(self): "dynamic": None, "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], "exceptions": { - "admin.notes.id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, "admin.notes.note": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, - "admin.notes.author_id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } + "type": "text", + "index": False, + #"include_in_all": False # Removed in es6 fixme: do we need to look at copy_to for the mapping? } } } diff --git a/portality/models/v2/application.py b/portality/models/v2/application.py index d7cf846136..655489f9c2 100644 --- a/portality/models/v2/application.py +++ b/portality/models/v2/application.py @@ -244,36 +244,8 @@ class AllPublisherApplications(DomainObject): MAPPING_OPTS = { "dynamic": None, - "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], - "exceptions": { - "admin.notes.id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, - "admin.notes.note": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, - "admin.notes.author_id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - } - } + "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]), + "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"] } diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 2c40e1e8b8..37c18f8e03 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -564,6 +564,15 @@ def delete_selected(cls, query, articles=False, snapshot_journals=True, snapshot # finally issue a delete request against the journals cls.delete_by_query(query) + @classmethod + def add_mapping_extensions(cls, default_mappings: dict): + default_mappings_copy = deepcopy(default_mappings) + mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_EXTENSIONS") + for key, value in mapping_extensions.items(): + if key in default_mappings_copy: + default_mappings_copy[key] = {**default_mappings_copy[key], **value} + return default_mappings_copy + def all_articles(self): from portality.models import Article return Article.find_by_issns(self.known_issns()) @@ -891,36 +900,8 @@ def _calculate_has_apc(self): MAPPING_OPTS = { "dynamic": None, - "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], - "exceptions": { - "admin.notes.id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, - "admin.notes.note": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - }, - "admin.notes.author_id": { - "type": "text", - "fields": { - "exact": { - "type": "keyword", - "store": True - } - } - } - } + "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]), + "exceptions": app.config["ADMIN_NOTES_SEARCH_MAPPING"] } diff --git a/portality/models/v2/shared_structs.py b/portality/models/v2/shared_structs.py index e5bdbd2e07..a3bbcb3d86 100644 --- a/portality/models/v2/shared_structs.py +++ b/portality/models/v2/shared_structs.py @@ -175,8 +175,7 @@ "created_date" : {"coerce" : "utcdatetime"}, "last_updated" : {"coerce" : "utcdatetime"}, "last_manual_update" : {"coerce" : "utcdatetime"}, - "es_type": {"coerce": "unicode"}, - "all_meta" : {"coerce" : "unicode"} + "es_type": {"coerce": "unicode"} }, "objects" : [ "admin", diff --git a/portality/settings.py b/portality/settings.py index 8ebd1a0f25..30ec6c589f 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -484,8 +484,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "str": { "type": "text", @@ -495,8 +494,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "unicode_upper": { "type": "text", @@ -506,8 +504,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "unicode_lower": { "type": "text", @@ -517,8 +514,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "isolang": { "type": "text", @@ -578,8 +574,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "url": { "type": "text", @@ -589,8 +584,7 @@ # "index": False, "store": True } - }, - "copy_to": ["all_meta"] + } }, "utcdatetimemicros": { "type": "date", @@ -616,6 +610,19 @@ } } +# Extension Map from dataobj coercion declarations to ES mappings. +# This is useful when some extensions required for some objects additional to defaults. +# ~~->DataObj:Library~~ +# ~~->Seamless:Library~~ +DATAOBJ_TO_MAPPING_EXTENSIONS = { + "unicode": {"copy_to": ["all_meta"]}, + "str": {"copy_to": ["all_meta"]}, + "unicode_upper": {"copy_to": ["all_meta"]}, + "unicode_lower": {"copy_to": ["all_meta"]}, + "issn": {"copy_to": ["all_meta"]}, + "url": {"copy_to": ["all_meta"]} +} + # TODO: we may want a big-type and little-type setting DEFAULT_INDEX_SETTINGS = \ { @@ -686,7 +693,7 @@ "journal" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", + "query_validators" : ["non_public_fields_validator", "public_query_validator"], "query_filters" : ["only_in_doaj", "last_update_fallback", "search_all_meta"], "result_filters" : ["public_result_filter"], "dao" : "portality.models.Journal", # ~~->Journal:Model~~ @@ -696,7 +703,7 @@ "article" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", + "query_validators" : ["non_public_fields_validator", "public_query_validator"], "query_filters" : ["only_in_doaj"], "result_filters" : ["public_result_filter"], "dao" : "portality.models.Article", # ~~->Article:Model~~ @@ -707,7 +714,7 @@ "journal,article" : { "auth" : False, "role" : None, - "query_validator" : "public_query_validator", + "query_validators" : ["non_public_fields_validator", "public_query_validator"], "query_filters" : ["only_in_doaj", "strip_facets", "es_type_fix"], "result_filters" : ["public_result_filter", "add_fqw_facets", "fqw_back_compat"], "dao" : "portality.models.JournalArticle", # ~~->JournalArticle:Model~~ @@ -719,6 +726,7 @@ "journal" : { "auth" : True, "role" : "publisher", + "query_validators" : ["non_public_fields_validator"], "query_filters" : ["owner", "only_in_doaj", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.Journal" # ~~->Journal:Model~~ @@ -727,6 +735,7 @@ "applications" : { "auth" : True, "role" : "publisher", + "query_validators" : ["non_public_fields_validator"], "query_filters" : ["owner", "not_update_request", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.AllPublisherApplications" # ~~->AllPublisherApplications:Model~~ @@ -735,7 +744,8 @@ "update_requests" : { "auth" : True, "role" : "publisher", - "query_filters" : ["owner", "update_request"], + "query_validators" : ["non_public_fields_validator"], + "query_filters" : ["owner", "update_request", "search_all_meta"], "result_filters" : ["publisher_result_filter"], "dao" : "portality.models.Application" # ~~->Application:Model~~ } @@ -839,6 +849,7 @@ "journal" : { "auth" : False, "role" : None, + "query_validators": ["non_public_fields_validator"], "query_filters" : ["only_in_doaj", "public_source", "search_all_meta"], "dao" : "portality.models.Journal", # ~~->Journal:Model~~ "required_parameters" : None @@ -847,6 +858,7 @@ "application" : { "auth" : True, "role" : None, + "query_validators": ["non_public_fields_validator"], "query_filters" : ["owner", "private_source", "search_all_meta"], "dao" : "portality.models.Suggestion", # ~~->Application:Model~~ "required_parameters" : None @@ -867,6 +879,7 @@ QUERY_FILTERS = { # sanitisers "public_query_validator" : "portality.lib.query_filters.public_query_validator", + "non_public_fields_validator" : "portality.lib.query_filters.non_public_fields_validator", # query filters "only_in_doaj" : "portality.lib.query_filters.only_in_doaj", @@ -891,6 +904,52 @@ "private_source": "portality.lib.query_filters.private_source", "public_source": "portality.lib.query_filters.public_source", } +# Exclude the fields that doesn't want to be searched by public queries +# This is part of non_public_fields_validator. +EXCLUDED_FIELDS = [ + "admin.notes.note", + "admin.notes.id", + "admin.notes.author_id" +] + +ADMIN_NOTES_SEARCH_MAPPING = { + "all_meta" : { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + }, + "admin.notes.id": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + }, + "admin.notes.note": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + }, + "admin.notes.author_id": { + "type": "text", + "fields": { + "exact": { + "type": "keyword", + "store": True + } + } + } + } #################################################### # Autocomplete diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index e59415acb3..4f4ef6a072 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -62,7 +62,7 @@ $.extend(true, doaj, { {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, {'display':'Journal: alternative title','field':'bibjson.alternative_title'}, - {'display':'Notes','field':'admin.notes'}, + {'display':'Notes','field':'admin.notes.note'}, ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index dacbbde57c..0cb45d1fdf 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -72,7 +72,7 @@ $.extend(true, doaj, { {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, {'display':'Journal: Alternative Title','field':'bibjson.alternative_title'}, - {'display':'Notes','field':'admin.notes'} + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/associate.applications.edge.js b/portality/static/js/edges/associate.applications.edge.js index 7ff2a1b444..e178145d26 100644 --- a/portality/static/js/edges/associate.applications.edge.js +++ b/portality/static/js/edges/associate.applications.edge.js @@ -158,7 +158,8 @@ $.extend(true, doaj, { {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, - {'display':'Alternative Title','field':'bibjson.alternative_title'} + {'display':'Alternative Title','field':'bibjson.alternative_title'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/associate.journals.edge.js b/portality/static/js/edges/associate.journals.edge.js index 6077ce0cf6..512ee0187f 100644 --- a/portality/static/js/edges/associate.journals.edge.js +++ b/portality/static/js/edges/associate.journals.edge.js @@ -167,7 +167,8 @@ $.extend(true, doaj, { {'display':'ISSN', 'field':'index.issn.exact'}, {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, - {'display':'Publisher','field':'bibjson.publisher.name'} + {'display':'Publisher','field':'bibjson.publisher.name'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index 3faf1f843a..ade6598254 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -214,7 +214,8 @@ $.extend(true, doaj, { {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, {'display':'Publisher','field':'bibjson.publisher.name'}, - {'display':'Alternative Title','field':'bibjson.alternative_title'} + {'display':'Alternative Title','field':'bibjson.alternative_title'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index 632d04ca9c..e71dcc715c 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -209,7 +209,8 @@ $.extend(true, doaj, { {'display':'ISSN', 'field':'index.issn.exact'}, {'display':'Country of publisher','field':'index.country'}, {'display':'Journal language','field':'index.language'}, - {'display':'Publisher','field':'bibjson.publisher.name'} + {'display':'Publisher','field':'bibjson.publisher.name'}, + {'display':'Notes','field':'admin.notes.note'} ], defaultOperator: "AND", renderer: doaj.renderers.newFullSearchControllerRenderer({ From b44957078c39c4a304cf2489d97ccae904ce140c Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Mon, 14 Aug 2023 10:11:22 +0530 Subject: [PATCH 019/184] Added test cases for testing notes for journal and application --- doajtest/unit/test_query.py | 191 ++++++++++++++++++++++++++++++------ 1 file changed, 160 insertions(+), 31 deletions(-) diff --git a/doajtest/unit/test_query.py b/doajtest/unit/test_query.py index 3685d2927c..2dc145acac 100644 --- a/doajtest/unit/test_query.py +++ b/doajtest/unit/test_query.py @@ -1,6 +1,7 @@ from portality import models -from doajtest.fixtures import AccountFixtureFactory, ArticleFixtureFactory +from doajtest.fixtures import AccountFixtureFactory, ArticleFixtureFactory, EditorGroupFixtureFactory, \ + ApplicationFixtureFactory from doajtest.helpers import DoajTestCase, deep_sort from portality.bll.services.query import QueryService, Query @@ -30,7 +31,12 @@ "auth" : True, "role" : "admin", "dao" : "portality.models.Journal" - } + }, + "suggestion" : { + "auth" : True, + "role" : "admin", + "dao" : "portality.models.Application" + }, }, "api_query" : { "article" : { @@ -53,6 +59,30 @@ "query_filters" : ["owner", "private_source"], "dao" : "portality.models.Suggestion" } + }, + "editor_query" : { + "journal" : { + "auth" : True, + "role" : "editor", + "dao" : "portality.models.Journal" + }, + "suggestion" : { + "auth" : True, + "role" : "editor", + "dao" : "portality.models.Application" + } + }, + "associate_query": { + "journal": { + "auth": True, + "role": "associate_editor", + "dao": "portality.models.Journal" + }, + "suggestion" : { + "auth" : True, + "role" : "associate_editor", + "dao" : "portality.models.Application" + } } } @@ -68,9 +98,13 @@ } QUERY_FILTERS = { + "non_public_fields_validator" : "portality.lib.query_filters.non_public_fields_validator", + # query filters "only_in_doaj" : "portality.lib.query_filters.only_in_doaj", "owner" : "portality.lib.query_filters.owner", + "associate" : "portality.lib.query_filters.associate", + "editor" : "portality.lib.query_filters.editor", # result filters "public_result_filter" : "portality.lib.query_filters.public_result_filter", @@ -101,6 +135,29 @@ def tearDown(self): self.app_test.config['QUERY_ROUTE'] = self.OLD_QUERY_ROUTE self.app_test.config['QUERY_FILTERS'] = self.OLD_QUERY_FILTERS + def get_application_with_notes(self): + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + app.add_note("application test", "2015-01-01T00:00:00Z") + app.save() + return app + + def get_journal_with_notes(self): + j = models.Journal() + j.set_id("aabbccdd") + j.set_created("2010-01-01T00:00:00Z") + j.set_last_updated("2012-01-01T00:00:00Z") + j.set_last_manual_update("2014-01-01T00:00:00Z") + j.set_seal(True) + j.set_owner("rama") + j.set_editor_group("worldwide") + j.set_editor("eddie") + j.add_contact("rama", "rama@email.com") + j.add_note("testing", "2015-01-01T00:00:00Z") + j.set_bibjson({"title": "test"}) + j.save() + return j + def test_01_auth(self): with self.app_test.test_client() as t_client: response = t_client.get('/query/journal') # not in the settings above @@ -357,45 +414,19 @@ def test_public_query_notes(self): self.app_test.config['QUERY_ROUTE'] = JOURNAL_QUERY_ROUTE - j = models.Journal() - j.set_id("aabbccdd") - j.set_created("2010-01-01T00:00:00Z") - j.set_last_updated("2012-01-01T00:00:00Z") - j.set_last_manual_update("2014-01-01T00:00:00Z") - j.set_seal(True) - j.set_owner("rama") - j.set_editor_group("worldwide") - j.set_editor("eddie") - j.add_contact("rama", "rama@email.com") - j.add_note("testing", "2015-01-01T00:00:00Z") - j.set_bibjson({"title": "test"}) - j.save() + self.get_journal_with_notes() - maned = models.Account(**AccountFixtureFactory.make_managing_editor_source()) - maned.save(blocking=True) qsvc = QueryService() res = qsvc.search('query', 'journal', {'query': {'query_string': {'query': 'testing', 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, - 'track_total_hits': True}, account=maned, additional_parameters={}) + 'track_total_hits': True}, account=None, additional_parameters={}) assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] def test_admin_query_notes(self): - j = models.Journal() - j.set_id("aabbccdd") - j.set_created("2010-01-01T00:00:00Z") - j.set_last_updated("2012-01-01T00:00:00Z") - j.set_last_manual_update("2014-01-01T00:00:00Z") - j.set_seal(True) - j.set_owner("rama") - j.set_editor_group("worldwide") - j.set_editor("eddie") - j.add_contact("rama", "rama@email.com") - j.add_note("testing", "2015-01-01T00:00:00Z") - j.set_bibjson({"title": "test"}) - j.save() + self.get_journal_with_notes() maned = models.Account(**AccountFixtureFactory.make_managing_editor_source()) maned.save(blocking=True) @@ -406,3 +437,101 @@ def test_admin_query_notes(self): {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, 'track_total_hits': True}, account=maned, additional_parameters={}) assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_editor_query_notes(self): + + self.get_journal_with_notes() + + editor = models.Account(**AccountFixtureFactory.make_editor_source()) + editor.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=editor.id) + eg = models.EditorGroup(**eg_source) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('editor_query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=editor, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_associate_editor_query_notes(self): + + self.get_journal_with_notes() + + associate = models.Account(**AccountFixtureFactory.make_assed1_source()) + associate.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=associate.id) + eg = models.EditorGroup(**eg_source) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('associate_query', 'journal', {'query': {'query_string': {'query': 'testing', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=associate, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_associate_editor_application_query_notes(self): + + app = self.get_application_with_notes() + + associate = models.Account(**AccountFixtureFactory.make_assed1_source()) + associate.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=associate.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('associate_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=associate, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_editor_application_query_notes(self): + + app = self.get_application_with_notes() + + editor = models.Account(**AccountFixtureFactory.make_editor_source()) + editor.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=editor.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('editor_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=editor, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_editor_application_query_notes(self): + + app = self.get_application_with_notes() + + med = models.Account(**AccountFixtureFactory.make_managing_editor_source()) + med.save(blocking=True) + + eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=med.id) + eg = models.EditorGroup(**eg_source) + eg.set_name(app.editor_group) + eg.save(blocking=True) + + qsvc = QueryService() + + res = qsvc.search('admin_query', 'suggestion', {'query': {'query_string': {'query': 'application test', + 'default_operator': 'AND'}}, 'size': 0, 'aggs': {'country_publisher': + {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, + 'track_total_hits': True}, account=med, additional_parameters={}) + assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] From c73d7e6b56894ad372a66aafb3eea67c7a67c6e2 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Tue, 22 Aug 2023 13:09:44 +0530 Subject: [PATCH 020/184] Minor fixes related the field names --- portality/lib/query_filters.py | 2 +- portality/models/v2/journal.py | 2 +- portality/settings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 92e21ca999..593ec47943 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -30,7 +30,7 @@ def public_query_validator(q): def non_public_fields_validator(q): - exclude_fields = app.config.get("EXCLUDED_FIELDS", {}) + exclude_fields = app.config.get("PUBLIC_QUERY_VALIDATOR__EXCLUDED_FIELDS", {}) context = q.get_field_context() if context: if "default_field" in context: diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 6e2ce20bcd..9d8a797377 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -567,7 +567,7 @@ def delete_selected(cls, query, articles=False, snapshot_journals=True, snapshot @classmethod def add_mapping_extensions(cls, default_mappings: dict): default_mappings_copy = deepcopy(default_mappings) - mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_EXTENSIONS") + mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS") for key, value in mapping_extensions.items(): if key in default_mappings_copy: default_mappings_copy[key] = {**default_mappings_copy[key], **value} diff --git a/portality/settings.py b/portality/settings.py index ddd564b842..df76f0bcf4 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -630,7 +630,7 @@ # This is useful when some extensions required for some objects additional to defaults. # ~~->DataObj:Library~~ # ~~->Seamless:Library~~ -DATAOBJ_TO_MAPPING_EXTENSIONS = { +DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS = { "unicode": {"copy_to": ["all_meta"]}, "str": {"copy_to": ["all_meta"]}, "unicode_upper": {"copy_to": ["all_meta"]}, @@ -922,7 +922,7 @@ } # Exclude the fields that doesn't want to be searched by public queries # This is part of non_public_fields_validator. -EXCLUDED_FIELDS = [ +PUBLIC_QUERY_VALIDATOR__EXCLUDED_FIELDS = [ "admin.notes.note", "admin.notes.id", "admin.notes.author_id" From 01f270d99d1e8506caec4c580afc79dbdf9f5cc6 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Tue, 29 Aug 2023 16:27:45 +0530 Subject: [PATCH 021/184] Corrected the verification of the owner of the article --- portality/tasks/preservation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index c71a2f4e6e..b7f3d6e832 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -610,10 +610,9 @@ def __process_article(self, dir_path, files, articles_list): if article: article_data = article.data - if not self.owner_of_article(article): - articles_list.add_unowned_articles(package) + is_owner = self.owner_of_article(article) - else: + if not isinstance(is_owner, bool) and is_owner == True: issn, article_id, metadata_json = self.get_article_info(article_data) try: package = ArticlePackage(dir_path, files) @@ -633,6 +632,8 @@ def __process_article(self, dir_path, files, articles_list): except Exception: articles_list.add_unbagged_articles(package) app.logger.exception(f"Error while create article ( {article_id} ) package") + else: + articles_list.add_unowned_articles(package) else: # skip the article if not found From 77aea0ae6dd0adfd25343c71e69007d5a17a67a3 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Tue, 29 Aug 2023 17:36:48 +0530 Subject: [PATCH 022/184] corrected the conditional statement --- portality/tasks/preservation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index b7f3d6e832..a806174c16 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -612,7 +612,7 @@ def __process_article(self, dir_path, files, articles_list): is_owner = self.owner_of_article(article) - if not isinstance(is_owner, bool) and is_owner == True: + if isinstance(is_owner, bool) and is_owner == True: issn, article_id, metadata_json = self.get_article_info(article_data) try: package = ArticlePackage(dir_path, files) From 03778c3e569ba0406bb40a36bcc9d5b964f6b7d7 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Tue, 29 Aug 2023 18:05:35 +0530 Subject: [PATCH 023/184] Use dates library instead of datetime --- portality/tasks/preservation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index a806174c16..47e82107e7 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -5,7 +5,6 @@ import shutil import tarfile from copy import deepcopy -from datetime import datetime from zipfile import ZipFile import requests @@ -209,7 +208,7 @@ def prepare(cls, username, **kwargs): :return: background job """ - created_time = dates.format(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + created_time = dates.now() dir_name = username + "-" + created_time local_dir = os.path.join(Preservation.UPLOAD_DIR, dir_name) file = kwargs.get("upload_file") @@ -727,7 +726,7 @@ def __init__(self, preservation_dir, journal_dir, owner): self.preservation_dir = preservation_dir self.journal_dir = journal_dir self.package_dir = os.path.join(self.preservation_dir, journal_dir) - self.created_time = dates.format(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + self.created_time = dates.now() self.tar_file = self.package_dir + "_" + self.created_time + ".tar.gz" self.tar_file_name = os.path.basename(self.tar_file) self.__owner = owner From 626bc7720d0893bad17a95eb605ebda8d1429f76 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 31 Aug 2023 15:12:44 +0100 Subject: [PATCH 024/184] WIP; add appropriate classes and event listeners --- portality/static/js/tourist.js | 8 ++++++++ portality/templates/includes/_tourist_nav.html | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/portality/static/js/tourist.js b/portality/static/js/tourist.js index cc0f375f0a..1f93579e6b 100644 --- a/portality/static/js/tourist.js +++ b/portality/static/js/tourist.js @@ -15,6 +15,14 @@ doaj.tourist.init = function(params) { if (first) { doaj.tourist.start(first); } + + $(".dropdown--notifications").hoverIntent(doaj.notifications.showDropdown, doaj.notifications.hideDropdown); + doaj.notifications.showDropdown = function(e) { + $("#feature_tours").show(); + } + doaj.notifications.hideDropdown = function() { + $("#feature_tours").hide(); + } } doaj.tourist.findNextTour = function() { diff --git a/portality/templates/includes/_tourist_nav.html b/portality/templates/includes/_tourist_nav.html index f2fa255992..b141f89bbf 100644 --- a/portality/templates/includes/_tourist_nav.html +++ b/portality/templates/includes/_tourist_nav.html @@ -2,7 +2,7 @@ {% set tours = tourservice.activeTours(request.path, current_user) %} {% if tours|length > 0 %} -
  • +
  • From 293863a876c114851b912407c164853134a8a7a1 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 31 Aug 2023 16:51:16 +0100 Subject: [PATCH 025/184] A few minor fixes for preservation and its tests --- doajtest/unit/test_task_preservation.py | 5 ++--- portality/tasks/preservation.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doajtest/unit/test_task_preservation.py b/doajtest/unit/test_task_preservation.py index 2823c531d6..d8d62d87e8 100644 --- a/doajtest/unit/test_task_preservation.py +++ b/doajtest/unit/test_task_preservation.py @@ -53,8 +53,7 @@ class TestPreservationSetup(DoajTestCase): def initial_setup(self, package_name): super(TestPreservationSetup, self).setUp() - resources = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "unit", "resources") - articles_zip_path = test_constants.PATH_RESOURCES / "articles.zip" + articles_zip_path = test_constants.PATH_RESOURCES / package_name with open(articles_zip_path, 'rb') as zf: self.zip_file = FileStorage(BytesIO(zf.read()), filename=package_name) @@ -162,7 +161,7 @@ def tearDown(self): super(TestPreservationMultipleJournals, self).tearDown() @patch.object(Article, 'pull_by_key', mock_pull_by_key) - @patch.object(requests,"post", mock_requests_post) + @patch.object(requests, "post", mock_requests_post) @patch.object(preservation.Preservation, 'owner_of_article', mock_owner_of_article) def test_preservation_multiple_journals(self): self.preserve.save_file(self.zip_file) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index c1da7a675e..4fc1cf09b9 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -726,7 +726,7 @@ def __init__(self, preservation_dir, journal_dir, owner): self.preservation_dir = preservation_dir self.journal_dir = journal_dir self.package_dir = os.path.join(self.preservation_dir, journal_dir) - self.created_time = dates.now() + self.created_time = dates.now_str("%Y-%m-%d-%H-%M-%S") self.tar_file = self.package_dir + "_" + self.created_time + ".tar.gz" self.tar_file_name = os.path.basename(self.tar_file) self.__owner = owner From 315384915e68cc5d89f12383b252e13c2038f468 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Fri, 1 Sep 2023 11:13:34 +0100 Subject: [PATCH 026/184] symlinks to allow preservation tests to find the resources --- .gitignore | 4 ++++ doajtest/unit/resources/invalid_article.zip | 1 + doajtest/unit/resources/multi_journals.zip | 1 + doajtest/unit/resources/valid_article.zip | 1 + 4 files changed, 7 insertions(+) create mode 120000 doajtest/unit/resources/invalid_article.zip create mode 120000 doajtest/unit/resources/multi_journals.zip create mode 120000 doajtest/unit/resources/valid_article.zip diff --git a/.gitignore b/.gitignore index 5e73cfd20a..087f3ab5a4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,11 @@ src* build/* dist/* tmp/* + *.zip +!doajtest/unit/resources/*.zip +!doajtest/preservation_upload_test_package/*.zip + scratch.py .coverage htmlcov/* diff --git a/doajtest/unit/resources/invalid_article.zip b/doajtest/unit/resources/invalid_article.zip new file mode 120000 index 0000000000..e955a12f14 --- /dev/null +++ b/doajtest/unit/resources/invalid_article.zip @@ -0,0 +1 @@ +../../preservation_upload_test_package/invalid_article.zip \ No newline at end of file diff --git a/doajtest/unit/resources/multi_journals.zip b/doajtest/unit/resources/multi_journals.zip new file mode 120000 index 0000000000..ac64455e26 --- /dev/null +++ b/doajtest/unit/resources/multi_journals.zip @@ -0,0 +1 @@ +../../preservation_upload_test_package/multi_journals.zip \ No newline at end of file diff --git a/doajtest/unit/resources/valid_article.zip b/doajtest/unit/resources/valid_article.zip new file mode 120000 index 0000000000..ebf17a3099 --- /dev/null +++ b/doajtest/unit/resources/valid_article.zip @@ -0,0 +1 @@ +../../preservation_upload_test_package/valid_article.zip \ No newline at end of file From 319d819d5a0aed081c689604e1d05737f2c6a9d5 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 1 Sep 2023 12:51:55 +0100 Subject: [PATCH 027/184] modify styles so the new accessible radiobuttons and checkboxes are displayed correctly everywhere --- cms/sass/base/_general.scss | 17 ++++++++++++++++- cms/sass/components/_filters.scss | 1 + cms/sass/components/_form.scss | 5 ++++- doajtest/testbook/public_site/public_search.yml | 8 ++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cms/sass/base/_general.scss b/cms/sass/base/_general.scss index 29d2445e24..7272b938b3 100644 --- a/cms/sass/base/_general.scss +++ b/cms/sass/base/_general.scss @@ -261,7 +261,22 @@ select { input[type="checkbox"], input[type="radio"] { - display: none; + //display: none; + + opacity: 0; + width: 0.8em; + height: 0.8em; + margin-left: -0.8rem; + + &:focus + label { + outline: dashed 2px lightgrey; + outline-offset: 1px; + } + + &:focus:not(:focus-visible){ + outline: none; + } + + label { margin: 0 0 $spacing-03 0; diff --git a/cms/sass/components/_filters.scss b/cms/sass/components/_filters.scss index e82883841b..4bb137e862 100644 --- a/cms/sass/components/_filters.scss +++ b/cms/sass/components/_filters.scss @@ -50,6 +50,7 @@ max-height: $spacing-07; height: auto; overflow-y: auto; + padding-top: $spacing-01; @include unstyled-list; li { diff --git a/cms/sass/components/_form.scss b/cms/sass/components/_form.scss index b203ff5988..0b5be1a114 100644 --- a/cms/sass/components/_form.scss +++ b/cms/sass/components/_form.scss @@ -45,10 +45,13 @@ &[type="number"] { width: 25%; } + &[type="radio"], &[type="checkbox"] { + width: 0.8rem; + } } input[type="checkbox"] + label { - display: table; + display: unset; } input[type="checkbox"] + label, diff --git a/doajtest/testbook/public_site/public_search.yml b/doajtest/testbook/public_site/public_search.yml index 1bce101f8c..fdc9415f79 100644 --- a/doajtest/testbook/public_site/public_search.yml +++ b/doajtest/testbook/public_site/public_search.yml @@ -166,3 +166,11 @@ tests: results: - You are taken to the full text of this article on the Web. It opens in a new tab + - step: Click tab + results: + - focus is on the list of checkboxes + results: + - focus is clearly marked by the outline + - step: click spacebar to check the filter + results: + - filter is applied From 6e2152554b2ff71267a16f3225b418c3ae3b0ce9 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Fri, 1 Sep 2023 19:59:00 +0530 Subject: [PATCH 028/184] journal,article query not searchable for journal notes --- .../administrative_search/journals.yml | 9 +++++++ doajtest/unit/test_query.py | 20 +++++++++++++++ portality/bll/services/query.py | 11 ++++++++ portality/lib/query_filters.py | 25 +++++++++++++++++++ portality/settings.py | 3 ++- 5 files changed, 67 insertions(+), 1 deletion(-) diff --git a/doajtest/testbook/administrative_search/journals.yml b/doajtest/testbook/administrative_search/journals.yml index e6882e5075..0d11716340 100644 --- a/doajtest/testbook/administrative_search/journals.yml +++ b/doajtest/testbook/administrative_search/journals.yml @@ -197,3 +197,12 @@ tests: results: 'Applications assigned to you' page displayed - step: Search for the keyword which is given in a notes in an application results: The application is displayed in the search results + +- title: Test notes not searchable for public journal,article + context: + role: anonymous + steps: + - step: Do a url search for the keyword which is given in a notes for journal,article index. Ex: /query/journal,article/_search?&source={"query":{"query_string":{"query":"Test Note","default_operator":"AND"}},"size":"20"}&ref=fqw + results: No results displayed + - step: Modify the search url to add the notes keyword and the default field 'admin.notes.note' and then hit the url + results: Error message displayed diff --git a/doajtest/unit/test_query.py b/doajtest/unit/test_query.py index 2dc145acac..6ab470754d 100644 --- a/doajtest/unit/test_query.py +++ b/doajtest/unit/test_query.py @@ -535,3 +535,23 @@ def test_editor_application_query_notes(self): {'terms': {'field': 'index.country.exact', 'size': 100, 'order': {'_count': 'desc'}}}}, 'track_total_hits': True}, account=med, additional_parameters={}) assert res['hits']['total']["value"] == 1, res['hits']['total']["value"] + + def test_journal_article_query_notes(self): + + self.app_test.config['QUERY_ROUTE'] = self.OLD_QUERY_ROUTE + + self.app_test.config['QUERY_FILTERS'] = self.OLD_QUERY_FILTERS + + self.article11 = models.Article( + **ArticleFixtureFactory.make_article_source(pissn="1111-1111", eissn="2222-2222", doi="10.0000/article-11", + fulltext="https://www.article11.com")) + self.article11.save(blocking=True) + + self.get_journal_with_notes() + + qsvc = QueryService() + + res = qsvc.search('query', 'journal,article', {'query': {'query_string': + {'query': 'application test','default_operator': 'AND'}}, + 'size': 0, 'track_total_hits': True}, account=None, additional_parameters={"ref":"fqw"}) + assert res['hits']['total']["value"] == 0, res['hits']['total']["value"] diff --git a/portality/bll/services/query.py b/portality/bll/services/query.py index f8367e7844..9595dfc801 100644 --- a/portality/bll/services/query.py +++ b/portality/bll/services/query.py @@ -225,6 +225,17 @@ def add_default_field(self, value: str): item["query_string"]["default_field"] = value break + def add_should(self, filter, minimum_should_match=1): + self.convert_to_bool() + context = self.q["query"]["bool"] + if "should" not in context: + context["should"] = [] + if isinstance(filter, list): + context["should"].extend(filter) + else: + context["should"].append(filter) + context["minimum_should_match"] = minimum_should_match + def add_must_filter(self, filter): self.convert_to_bool() context = self.q["query"]["bool"] diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 593ec47943..0bcaf9d916 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -58,6 +58,31 @@ def search_all_meta(q): return q +def journal_article_filter(q): + """Search by all_meta field for journal and all fields for article""" + search_text = None + context = q.get_field_context() + if isinstance(context, list): + for item in context: + if "query_string" in item: + search_text = item["query_string"]["query"] + if "default_field" in item["query_string"]: + return q + elif isinstance(context, dict): + if "query_string" in context: + search_text = context["query_string"]["query"] + if "default_field" in context["query_string"]: + return q + q.convert_to_bool() + if search_text: + filter = [{"bool": {"must": [{"term": {"es_type.exact": "article"}}, + {"query_string": {"default_field": "*", "query": search_text}}]}}, + {"bool": {"must": [{"term": {"es_type.exact": "journal"}}, + {"query_string": {"default_field": "all_meta", "query": search_text}}]}}] + q.add_should(filter) + return q + + def owner(q): q.clear_match_all() q.add_must_filter({"term" : {"admin.owner.exact" : current_user.id}}) diff --git a/portality/settings.py b/portality/settings.py index df76f0bcf4..72f4a18455 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -731,7 +731,7 @@ "auth" : False, "role" : None, "query_validators" : ["non_public_fields_validator", "public_query_validator"], - "query_filters" : ["only_in_doaj", "strip_facets", "es_type_fix"], + "query_filters" : ["only_in_doaj", "strip_facets", "es_type_fix", "journal_article_filter"], "result_filters" : ["public_result_filter", "add_fqw_facets", "fqw_back_compat"], "dao" : "portality.models.JournalArticle", # ~~->JournalArticle:Model~~ "required_parameters" : {"ref" : ["fqw"]} @@ -909,6 +909,7 @@ "not_update_request" : "portality.lib.query_filters.not_update_request", "who_current_user" : "portality.lib.query_filters.who_current_user", # ~~-> WhoCurrentUser:Query ~~ "search_all_meta" : "portality.lib.query_filters.search_all_meta", # ~~-> SearchAllMeta:Query ~~ + "journal_article_filter" : "portality.lib.query_filters.journal_article_filter", # ~~-> JournalArticleFilter:Query ~~ # result filters "public_result_filter": "portality.lib.query_filters.public_result_filter", From 015a0c8ecfc49b52ed016918076049c8f4f1fedf Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Mon, 4 Sep 2023 09:56:14 +0530 Subject: [PATCH 029/184] Check if Preservation collection is empty --- portality/view/publisher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/view/publisher.py b/portality/view/publisher.py index 2410a8323a..f2b3713ddc 100644 --- a/portality/view/publisher.py +++ b/portality/view/publisher.py @@ -290,8 +290,8 @@ def preservation(): # check if collection has been assigned for the user # collection must be in the format {"user_id1",["collection_name1","collection_id1"], # "user_id2",["collection_name2","collection_id2"]} - collection_available = True - collection_dict = app.config.get("PRESERVATION_COLLECTION") + collection_dict = app.config.get("PRESERVATION_COLLECTION", {}) + collection_available = True if collection_dict else False if collection_dict and not current_user.id in collection_dict: collection_available = False elif collection_dict: From d95d0d92b7530b0f9988a9ca7582d12d7bcb6492 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 27 Sep 2023 10:21:42 +0100 Subject: [PATCH 030/184] add hover intent to tourist navigation --- portality/static/js/tourist.js | 17 ++++++++++------- portality/templates/includes/_tourist_nav.html | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/portality/static/js/tourist.js b/portality/static/js/tourist.js index 1f93579e6b..b2931f6dcf 100644 --- a/portality/static/js/tourist.js +++ b/portality/static/js/tourist.js @@ -16,13 +16,16 @@ doaj.tourist.init = function(params) { doaj.tourist.start(first); } - $(".dropdown--notifications").hoverIntent(doaj.notifications.showDropdown, doaj.notifications.hideDropdown); - doaj.notifications.showDropdown = function(e) { - $("#feature_tours").show(); - } - doaj.notifications.hideDropdown = function() { - $("#feature_tours").hide(); - } + $("#dropdown--tour_nav").hoverIntent(doaj.tourist.showDropdown, doaj.tourist.hideDropdown); +} + +doaj.tourist.showDropdown = function(e) { + console.log("showDropdown` called") + $("#feature_tours").show(); +} +doaj.tourist.hideDropdown = function() { + console.log("showDropdown` called") + $("#feature_tours").hide(); } doaj.tourist.findNextTour = function() { diff --git a/portality/templates/includes/_tourist_nav.html b/portality/templates/includes/_tourist_nav.html index b141f89bbf..ef8d33090a 100644 --- a/portality/templates/includes/_tourist_nav.html +++ b/portality/templates/includes/_tourist_nav.html @@ -2,10 +2,10 @@ {% set tours = tourservice.activeTours(request.path, current_user) %} {% if tours|length > 0 %} -
  • -
  • +
      {% for tour in tours %} {% set seen = namespace(val="[new] ") %} From 64452b47e500a137196037fea1c0e7a8b37943f4 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 27 Sep 2023 10:22:12 +0100 Subject: [PATCH 031/184] change classes to ids where appropriate --- portality/static/js/notifications.js | 2 +- portality/templates/layouts/dashboard_base.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/static/js/notifications.js b/portality/static/js/notifications.js index c564aff81e..9492402824 100644 --- a/portality/static/js/notifications.js +++ b/portality/static/js/notifications.js @@ -58,7 +58,7 @@ doaj.notifications.notificationsReceived = function(data) { } $(".notification_action_link").on("click", doaj.notifications.notificationClicked); - $(".dropdown--notifications").hoverIntent(doaj.notifications.showDropdown, doaj.notifications.hideDropdown); + $("#notifications_nav").hoverIntent(doaj.notifications.showDropdown, doaj.notifications.hideDropdown); } doaj.notifications.showDropdown = function(e) { diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index 355463e4e2..3e4bc13972 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -26,7 +26,7 @@

      DOAJ Dashboard

        {% include "includes/_tourist_nav.html" %} {# ~~->Notifications:Feature~~ #} -
      • +
      + +--> From 85886060d6e8996ed92ebd737923d92e455e6dcd Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 31 Oct 2023 09:20:22 +0100 Subject: [PATCH 055/184] Removed 2023 information 2023 information no longer needed after 31st October 2023 --- cms/pages/support/publisher-supporters.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/cms/pages/support/publisher-supporters.md b/cms/pages/support/publisher-supporters.md index f010872aba..5bf4509652 100644 --- a/cms/pages/support/publisher-supporters.md +++ b/cms/pages/support/publisher-supporters.md @@ -65,21 +65,6 @@ We are particularly grateful to those publishers who can contribute over and abo Please get in touch to discuss. -## 2023 benefits for publisher supporters - -([A downloadable version](https://docs.google.com/document/d/1xTVxUvqLkh2-r53cYlWdSIHsPGSnhcE7gi7bRFCaJik/edit?usp=sharing) of these benefits is available.) - -| Basic contribution | Sustaining contribution (25% higher than Basic) | Premier contribution (50% higher than Basic) | -|--------------------------|-------------------------------------------------|----------------------------------------------| -| Your logo on our website | Your logo on our website | Your logo on our website | -| A Tweet from the DOAJ account (21,900 followers) acknowledging your organisation as a Supporter | An interview-style blog post on [blog.doaj.org](https://blog.doaj.org/) (12,000 views per month) acknowledging your organisation as a Contributor, with questions about the state of open access and other current developments in open access scholarly publishing | An interview-style blog post on [blog.doaj.org](https://blog.doaj.org/) (12,000 views per month) acknowledging your organisation as a Contributor with questions about the state of open access and other current developments in open access scholarly publishing| -| | A one-pager for journal editors explaining, in general terms, how article metadata is ingested, and then pulled downstream by discovery services. | A one-pager for journal editors explaining, in general terms, how article metadata is ingested, and then pulled downstream by discovery services.| -| | General presentation or documentation, for raising awareness among your stakeholders, about the benefits of DOAJ, the work we do and how DOAJ benefits individual journal titles. | Dedicated presentation or documentation, tailored to your organisation, about the benefits of DOAJ, the work we do and how DOAJ benefits individual journal titles. Written as a joint document with the publisher. | -| | A report, (a Google Sheet), collecting metadata errors, as reported to us by discovery services. | A report, (a Google Sheet), collecting metadata errors, as reported to us by discovery services. | -| | | A yearly account review between the customer and DOAJ. The agenda can be determined by the publisher contact and a DOAJ representative. | -| | | A CSV file, generated annually, for recording changes in and which DOAJ updates your live records with. | -| | | Exposure across all our social media channels: Twitter, Instagram, LinkedIn, Facebook, WeChat. (Stats available.) | -
      {% include '/data/sponsors.html' %}
      ## Other publisher supporters From 0ddd67a46a1e6044de4dbef151944b88a96f2901 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 31 Oct 2023 09:22:00 +0100 Subject: [PATCH 056/184] Update contribution_rates.html --- portality/templates/includes/contribution_rates.html | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/portality/templates/includes/contribution_rates.html b/portality/templates/includes/contribution_rates.html index 6b2e09c080..303492e406 100644 --- a/portality/templates/includes/contribution_rates.html +++ b/portality/templates/includes/contribution_rates.html @@ -1,5 +1,6 @@ - + +

    • 2023 pricing for academic institutions, including:

      @@ -261,7 +262,7 @@

      2023 pricing for institutions that are members of:

      The definition as to Large/Small is at the discretion of the consortium.

      • - -
    - --> From da188ee76def8ba7c4f5bd5d62d3ec6ed47afd40 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 31 Oct 2023 09:23:34 +0100 Subject: [PATCH 057/184] Deleted code --- .../includes/contribution_rates.html | 301 ------------------ 1 file changed, 301 deletions(-) diff --git a/portality/templates/includes/contribution_rates.html b/portality/templates/includes/contribution_rates.html index 303492e406..9c2e3da277 100644 --- a/portality/templates/includes/contribution_rates.html +++ b/portality/templates/includes/contribution_rates.html @@ -1,304 +1,3 @@ - -
  • -

    2023 pricing for academic institutions, including:

    -
      -
    • Individual institutions
      • -
    • Academic libraries
      • -
    • A university or universities
      • -
    • Colleges
      • -
    -

    - -

    - -
    • - - -
  • -

    2023 pricing for institutions that are members of:

    -
      -
    • A billing consortium
      • -
    -

    - -

    - -
    • - -
  • -

    2023 pricing for organisations that are:

    -
      -
    • Consortia
      • -
    • Library networks
      • -
    -

    - -

    - -
    • ---> From 8de555413f4d0c00228127db79415fba3a352274 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 31 Oct 2023 09:24:56 +0100 Subject: [PATCH 058/184] Added l. 68 --- cms/pages/support/publisher-supporters.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cms/pages/support/publisher-supporters.md b/cms/pages/support/publisher-supporters.md index 5bf4509652..debf2afa71 100644 --- a/cms/pages/support/publisher-supporters.md +++ b/cms/pages/support/publisher-supporters.md @@ -65,6 +65,8 @@ We are particularly grateful to those publishers who can contribute over and abo Please get in touch to discuss. +## Publisher supporters +
    {% include '/data/sponsors.html' %}
    ## Other publisher supporters From b06e05535bc3379a9adaa8ddc0b67e276267a85f Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 31 Oct 2023 09:30:46 +0100 Subject: [PATCH 059/184] Deleted heading l. 1 --- portality/templates/data/sponsors.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/data/sponsors.html b/portality/templates/data/sponsors.html index a9723ad4d5..4cffdbf57b 100644 --- a/portality/templates/data/sponsors.html +++ b/portality/templates/data/sponsors.html @@ -1,4 +1,4 @@ -

    Contributors

    +
    {% for i in data.sponsors %} "; } else { return false; } }, authorPays : function(val, resultobj, renderer) { - var field = ""; - if (edges.hasProp(resultobj, "bibjson.apc.max") && resultobj.bibjson.apc.max.length > 0) { - field += 'Has charges'; - } else if (edges.hasProp(resultobj, "bibjson.other_charges.has_other_charges") && resultobj.bibjson.other_charges.has_other_charges) { - field += 'Has charges'; - } - if (field === "") { - field = 'No charges'; - } + if (resultobj.es_type === "journal") { + var field = ""; + if (edges.hasProp(resultobj, "bibjson.apc.max") && resultobj.bibjson.apc.max.length > 0) { + field += 'Has charges'; + } else if (edges.hasProp(resultobj, "bibjson.other_charges.has_other_charges") && resultobj.bibjson.other_charges.has_other_charges) { + field += 'Has charges'; + } + if (field === "") { + field = 'No charges'; + } - var urls = []; - if (edges.hasProp(resultobj, "bibjson.apc.url")) { - urls.push(resultobj.bibjson.apc.url); - } - if (edges.hasProp(resultobj, "bibjson.has_other_charges.url")) { - urls.push(resultobj.bibjson.has_other_charges.url) - } + var urls = []; + if (edges.hasProp(resultobj, "bibjson.apc.url")) { + urls.push(resultobj.bibjson.apc.url); + } + if (edges.hasProp(resultobj, "bibjson.has_other_charges.url")) { + urls.push(resultobj.bibjson.has_other_charges.url) + } - if (urls.length > 0) { - field += ' (see '; - for (var i = 0; i < urls.length; i++) { - field += '' + urls[i] + ''; + if (urls.length > 0) { + field += ' (see '; + for (var i = 0; i < urls.length; i++) { + field += '' + urls[i] + ''; + } + field += ')'; } - field += ')'; - } - return field ? field : false; + return field ? field : false; + } + else { + return false; + } }, abstract : function (val, resultobj, renderer) { @@ -8006,20 +8392,25 @@ $.extend(true, doaj, { if (resultobj.es_type === "application") { // determine the link name var linkName = "Review application"; - if (resultobj.admin.application_status === 'accepted' || resultobj.admin.application_status === 'rejected') { - linkName = "View finished application"; - if (resultobj.admin.related_journal) { - linkName = "View finished update"; + if (resultobj.admin.application_type === "new_application") { + if (resultobj.admin.application_status === 'accepted' || resultobj.admin.application_status === 'rejected') { + linkName = "View application (finished)" + } else { + linkName = "Review application" + } + } else { + if (resultobj.admin.application_status === 'accepted' || resultobj.admin.application_status === 'rejected') { + linkName = "View update (finished)" + } else { + linkName = "Review update" } - } else if (resultobj.admin.current_journal) { - linkName = "Review update"; } var result = '

    '; + result += ' style="margin-bottom: .75em;">' + linkName + '

    '; return result; } return false; @@ -8046,16 +8437,81 @@ $.extend(true, doaj, { // if it's not a suggestion or an article .. (it's a // journal!) // we really need to expose _type ... - var result = '

    ", methods=["GET"]) From e1e23875428a7f9d78531443a7919722588d7563 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Mon, 13 Nov 2023 11:11:49 +0530 Subject: [PATCH 083/184] Added timeout to the connection --- portality/scripts/es_reindex.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/portality/scripts/es_reindex.py b/portality/scripts/es_reindex.py index 093ed70cef..70206b013e 100644 --- a/portality/scripts/es_reindex.py +++ b/portality/scripts/es_reindex.py @@ -4,14 +4,19 @@ import json import time +import elasticsearch from elasticsearch import helpers from elasticsearch.exceptions import NotFoundError, RequestError, ConnectionError, AuthorizationException -from portality.core import app, es_connection, put_mappings +from portality.core import app from portality.lib import es_data_mapping def do_import(config): + # create a connection with timeout + es_connection = elasticsearch.Elasticsearch(app.config['ELASTICSEARCH_HOSTS'], + verify_certs=app.config.get("ELASTIC_SEARCH_VERIFY_CERTS", True), + timeout=60*10) # get the versions version = config.get("new_version") From 0e6b25f694a00d193fcf4e49fd59b91b0166c6c4 Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Mon, 13 Nov 2023 11:38:24 +0530 Subject: [PATCH 084/184] Added more documentation --- portality/scripts/es_reindex.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/portality/scripts/es_reindex.py b/portality/scripts/es_reindex.py index 70206b013e..920ce5c938 100644 --- a/portality/scripts/es_reindex.py +++ b/portality/scripts/es_reindex.py @@ -1,5 +1,28 @@ +# ~~ESReindex:CLI~~ """ This script is useful to create new index with any new mapping changes if applicable and copy the content from old index to new index +run the script: +portality/scripts/es_reindex.py +ex: +portality/scripts/es_reindex.py -u /portality/migrate/3575_make_notes_searchable/migrate.json + +example json file: +{ + "new_version": "-20231109", #new index version + "old_version": "-20230901", #old index version + "types": [ + { + "type" : "application", #model type + "migrate": true, #if migration required true or false + "set_alias": false #set to true if alias has to be set with base name ex: doaj-application + }, + { + "type": "journal", + "migrate": true, + "set_alias": false + } + ] +} """ import json From 4fcd326468fe2de9652fb6d1d4e501d6be7daf30 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 13 Nov 2023 11:52:35 +0000 Subject: [PATCH 085/184] add blank target to contact button --- portality/templates/application_form/_contact.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/application_form/_contact.html b/portality/templates/application_form/_contact.html index b312055c22..b1140a6e87 100644 --- a/portality/templates/application_form/_contact.html +++ b/portality/templates/application_form/_contact.html @@ -6,7 +6,7 @@ {% else %}

    {% endif %} From 04b5d46caefdbbb8486ebc98a5e20a204fa50731 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 13 Nov 2023 15:58:08 +0000 Subject: [PATCH 086/184] remove unneeded functional test steps from public search script --- doajtest/testbook/public_site/public_search.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/doajtest/testbook/public_site/public_search.yml b/doajtest/testbook/public_site/public_search.yml index 6b47834e85..43534a3e7c 100644 --- a/doajtest/testbook/public_site/public_search.yml +++ b/doajtest/testbook/public_site/public_search.yml @@ -58,15 +58,6 @@ tests: results: - The search/embed box is shown with a URL and an HTML/JS snippet displayed in each box - - step: Click "shorten url" (Note that this may not work in test, due to configuration - requirements, you may receive an error instead) - results: - - The long search url is replaced with a bit.ly url - - '"shorten url" is replaced by "original url"' - - step: Click "original url" - results: - - The bit.ly url is replaced by the original long search url - - '"original url" is replaced by "shorten url"' - step: Close the "Share | Embed" section by clicking the Close button at the bottom of the box, or clicking the "Share | Embed" button again results: From 30af27ed77efadfa95a5e1487efc027c720eeac7 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Mon, 13 Nov 2023 16:14:53 +0000 Subject: [PATCH 087/184] Version bump for mail button UI change --- portality/settings.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 48149cc5ef..df3b31fc57 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.4.6" +DOAJ_VERSION = "6.4.7" API_VERSION = "3.0.1" ###################################### diff --git a/setup.py b/setup.py index 8de500d659..49da5a8c38 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='6.4.6', + version='6.4.7', packages=find_packages(), install_requires=[ "awscli==1.20.50", From 8ab8ca15f5d8066d0209c30c0ad1afb7a065c334 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 13 Nov 2023 17:09:45 +0000 Subject: [PATCH 088/184] add click to copy widget to api field in the account page --- portality/templates/account/view.html | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/portality/templates/account/view.html b/portality/templates/account/view.html index ab4c8c0de6..dd1c2aa4a2 100644 --- a/portality/templates/account/view.html +++ b/portality/templates/account/view.html @@ -90,7 +90,9 @@

    Your API access

    {% if account.has_role("api") %}
    API Key
    - {{ account.api_key }} + {{ account.api_key }} + Copy value +
    {% endif %} @@ -121,6 +123,17 @@

    Your API access

    $(window).scrollTop($(firstError).offset().top - 100) } + {% if account.has_role("api") %} + // Copy API key to clipboard + $('#click-to-copy--api').click(function() { + let value_to_copy = $('#user-api').html(); + navigator.clipboard.writeText(value_to_copy); + var confirmation = $("#copy-confirmation--api"); + confirmation.text("Copied: " + value_to_copy); + confirmation.show().delay(3000).fadeOut(); + }); + {% endif %} + // If the user is editing their own account, make them confirm when email address is updated {% if current_user.id == account.id %} $('form#edit_user').submit(function () { From c0a663ff10d90754abd86f143eaf5add6b7fe0bf Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 13 Nov 2023 17:16:09 +0000 Subject: [PATCH 089/184] add functional test --- .../testbook/user_management/user_management.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doajtest/testbook/user_management/user_management.yml b/doajtest/testbook/user_management/user_management.yml index a19230dbd9..aeb95a3267 100644 --- a/doajtest/testbook/user_management/user_management.yml +++ b/doajtest/testbook/user_management/user_management.yml @@ -40,6 +40,10 @@ fragments: - Your API Key at the top of the page is updated - You receive a flash message saying your record is updated and the API key has changed + - step: Click "Copy to click" icon next to your API key + results: + - The API key is copied to your clipboard + - Confirmation message appears and dissapears after a moment - step: Enter mismatched passwords into the "Change your password" boxes and click "Update Account" results: @@ -205,3 +209,12 @@ tests: - step: Click on the user's journals results: - All the same journals are still associated with the user + +- title: User account without API role + context: + role: admin + steps: + - step: Find (or create) a user account which does not have the API role + - step: View the user account page at /account/[account name] + results: + - The API key is not displayed From 5ef1f3049f0e769d32bc631c375bd8b8f32b20f9 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 13 Nov 2023 17:23:59 +0000 Subject: [PATCH 090/184] fix the functional test --- doajtest/testbook/user_management/user_management.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doajtest/testbook/user_management/user_management.yml b/doajtest/testbook/user_management/user_management.yml index aeb95a3267..55795719bd 100644 --- a/doajtest/testbook/user_management/user_management.yml +++ b/doajtest/testbook/user_management/user_management.yml @@ -40,7 +40,7 @@ fragments: - Your API Key at the top of the page is updated - You receive a flash message saying your record is updated and the API key has changed - - step: Click "Copy to click" icon next to your API key + - step: Click "Click to Copy" icon next to your API key results: - The API key is copied to your clipboard - Confirmation message appears and dissapears after a moment From 2feeb4fe1a40f5d4b53da36360718e0c12162634 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 09:11:55 +0100 Subject: [PATCH 091/184] Added line 233 --- cms/pages/apply/guide.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cms/pages/apply/guide.md b/cms/pages/apply/guide.md index 6f037f0ad0..cd57b4a270 100644 --- a/cms/pages/apply/guide.md +++ b/cms/pages/apply/guide.md @@ -230,7 +230,9 @@ The DOAJ editorial team reviews update requests for accuracy and we aim to proce We are always looking for volunteers to translate our criteria. If you would like to volunteer to do this, please [contact us](mailto:helpdesk@doaj.org). -Our criteria are in: +Please note that these translations may not include the latest changes made to the Guide in November 2023. + +Our criteria are available in: - [Arabic](https://drive.google.com/file/d/1UrY8p8KV4hBaYlYuyH1MGUOsHhEABtSE/view?usp=sharing) - [Chinese (application form)](https://zenodo.org/record/4350096) @@ -252,7 +254,7 @@ Our criteria are in: This is Version 2.0 of the Guide to applying. -*Version 2.0 (November 2023 - addition of new criteria around special issues, smaller changes for clarity and plain English.)* +*Version 2.0 (November 2023 - addition of new criteria around special issues, smaller changes for clarity and plain English.)*
    Version: 1.3 (April 2023 - further clarification to endogeny, displaying journal rankings, information about author charges and reviewers, flipped journals and journals/publishers with multiple exclusions )
    Version: 1.2 (February 2022 - extra copyright and licensing information were moved to a separate page)
    Version: 1.1 (December 2021 - a clarification to endogeny criterion)
    From 2b146388480c0441c0572b1866e79ce4f64100d5 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 11:14:03 +0100 Subject: [PATCH 092/184] Added content --- cms/pages/apply/publisher-responsibilities.md | 148 +++++++++++++++++- 1 file changed, 142 insertions(+), 6 deletions(-) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 7980707816..6d6c7e5a30 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -1,6 +1,6 @@ --- layout: sidenav -title: Publisher responsibilities +title: Publisher information section: Apply toc: true sticky_sidenav: true @@ -8,12 +8,148 @@ featuremap: ~~PublisherResponsibilities:Fragment~~ --- -Lorem ipsum dolor sit amet. +## Working together -## First heading +### Our commitment to you -More content here. +- **No charge** - we will never charge a fee for reviewing your applications in DOAJ +- **Fairness** - we assess all journals using the same criteria. Our [Guide to applying](/apply/guide/) fully describes these criteria. +- **Confidentiality** - we will only share or discuss details and information collected during the review process with the applicant or an entity authorised to represent the journal. +- **Privacy** - we [will not share your personal data](https://doaj.org/privacy/) with third parties unless you have permitted us to do so. We will always ask you first. -### Subheading +### Your commitment to us -Some more content here. +- You will [respond to questions](https://doaj.org/apply/guide/#the-application-process) from our editorial team promptly. +- You will [keep your contact details up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.eieqb2mmewak). +- You will inform us of changes to your journals and [keep your journal records up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx)https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx. +- You will [upload article metadata](https://doaj.org/publisher/uploadfile) to us if you are able to. + +We have [a zero-tolerance policy](https://doaj.org/about/#zero-tolerance-policy). We will treat you with respect and courtesy. We expect you to do the same. + +### Emails from us + +When you register with us, we use your email address to send important information about your applications, updates and journals. To make sure that our emails reach you, consider the following: + +- Add helpdesk@doaj.org to your contacts. +- Check your Spam folder. If you see an email from us there, mark it as safe. + +**Using an institutional email address** +If you use an institutional email address, your institution may apply extra security measures, such as a firewall. Your IT department administers the firewall. If you aren’t receiving our emails, contact your IT department to see if they can find emails from helpdesk@doaj.org and if they will let them through. If they have concerns about doing this, ask them to contact us. + +## Applying to DOAJ + +### Before you submit an application + +1. Read our [Guide to applying](https://doaj.org/apply/guide/) (available in [several languages](https://doaj.org/apply/guide/#in-other-languages)) and make sure your journal meets all the basic criteria. +2. If you are applying for the first time, [register with us first](https://doaj.org/account/register). +3. If you already have a DOAJ account, [sign in](https://doaj.org/account/login)https://doaj.org/account/login to submit a new application. + +A [sample of the application form](https://doaj.org/static/doaj/docs/2023-07-04-DOAJQuestions.pdf) is available to download. Details of the [application process](https://doaj.org/apply/guide/#the-application-process) are in the guide. + +### Rejected applications + +If we reject your application, you will receive an email explaining why and how long you must wait before applying again. [Read the information](https://doaj.org/apply/guide/#if-your-application-is-rejected) in the guide. + +### Accepted applications + +If we accept your application, you will receive an automated email that includes: +- The URL of your journal record in DOAJ +- Information on how to log into your publisher dashboard +- Details on why it is helpful to upload article metadata to DOAJ + +If you found your journal in DOAJ but didn’t get an email from us, there are [steps you can take](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.afqabefrwof3) to ensure our emails are delivered. + +During the review process, we might correct some of the information in your application. + +You can [submit an update request](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx) if something about your journal has changed since you applied. + +## Your DOAJ account + +You can access your dashboard and account settings by logging into your account. + +### Your publisher dashboard + +On [your dashboard](https://doaj.org/publisher/), you can + +- See [a list of your journals](https://doaj.org/publisher/journal) indexed in DOAJ +- [Submit an update request](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx) when journal details change +- [Upload](https://doaj.org/publisher/uploadfile) or [enter](https://doaj.org/publisher/metadata) article metadata for your journals +- Download the Seal logo (only for journals awarded the DOAJ Seal). +- Upload your full-text content (only for journals preserved via JASPER). + +### Settings + +The Settings button allows you to: + +- Update your account details +- Change your password +- Find your account ID +- Find your API key + +### Contact details + +- You are responsible for keeping the name and email address on your DOAJ account up to date. +- If the contact person changes, log in to your account and edit the details so that DOAJ always knows how to contact you. + +### Problems logging in + +If you cannot log in, you can [reset your password](https://doaj.org/account/forgot). We will send an email containing a reset link. The link is valid for 24 hours. + +If you don’t get the email, check your Spam folder and check which address you registered with us. If you haven’t received the email after three hours, [contact us](mailto:helpdesk@doaj.org). + +## Your indexed journals + +### Displaying the DOAJ logo + +To show that your journal is indexed in DOAJ or has the DOAJ Seal: + +- You may use the DOAJ or the Seal logo on your journal’s website but not on individual articles +- You may only use the DOAJ or Seal logo on a platform or portal website if all of the platform’s journals are indexed or have been awarded the Seal +- You can [download the DOAJ logos](https://doaj-kit.netlify.app/building-blocks/logotype/) +- If awarded, you can download the Seal logo from [your publisher dashboard](https://doaj.org/publisher/journal) + +We have [guidelines](https://doaj-kit.netlify.app/building-blocks/logotype/) about how you can use our logo on your website or in printed material. + +### Keeping your journal records up to date +When something changes, you must keep your journal records in DOAJ up to date. + +- [Contact our Help Desk](mailto:helpdesk@doaj.org) if you want to change the Title or ISSN of your journal, but [read this section first](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.yjpqoy71fmiy). +- [Contact our Help Desk](mailto:helpdesk@doaj.org) if your journal qualifies for the Seal. +- For all other changes, log into your DOAJ account to [submit an update request](https://doaj.org/publisher/journal). Use the black ‘Update’ button. + +### When should I submit an update? + +- When the URLs of your journal change +- When you change the fees/APCs +- When journal information or policies change +- When your journal is included in a digital archive, starts using DOIs, or has an entry in Sherpa/RoMEO + +We aim to process update requests within one month. + +### Journal title changes, mergers or continuations + +- There are [specific rules](https://www.issn.org/understanding-the-issn/assignment-rules/issn-the-major-principles/) about what to do when a journal changes its title. +- You must submit a new application if your journal has a new ISSN. +- We will automatically reject the application if the new ISSN is not confirmed. +- We will review the two records alongside each other to determine if the new title meets our basic criteria. If not, we will reject the application and withdraw the previous title. + +### Ceased journals +If one of your indexed journals will be ceasing publication, let us know as soon as possible by contacting our Help Desk. When you contact us, include the ISSN and the title of your journal. +If another title is continuing the journal, you must submit a continuation. +Journal transfers +If a journal is moving to or from another publisher, let us know as soon as possible by contacting our Help Desk. We need the following information from you: +Journal title +ISSN +Transfer date +Name and email address of the old publisher +Name and email address of the new publisher +Journal queries +Journal is not listed in DOAJ +We might have withdrawn your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. Furthermore, we might have withdrawn the journal if it is inactive (ceased publishing) or the website is unavailable. +When we withdraw your journal, we will send you an email; remind yourself of the email address attached to your DOAJ account and check your Spam folder if you haven’t received anything. +Check our list of withdrawn journals. +You can ask for more information by contacting our Help Desk. +Journal is not listed on my dashboard +Contact our Help Desk with the journal title and ISSN; they will help you. +Unknown journal on my dashboard +Contact our Help Desk with the journal title and ISSN; they will help you. From 3de072300b9a9bdf1109e78de47b6f0ace646060 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 11:14:53 +0100 Subject: [PATCH 093/184] Update publisher-responsibilities.md --- cms/pages/apply/publisher-responsibilities.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 6d6c7e5a30..8745266159 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -21,7 +21,7 @@ featuremap: ~~PublisherResponsibilities:Fragment~~ - You will [respond to questions](https://doaj.org/apply/guide/#the-application-process) from our editorial team promptly. - You will [keep your contact details up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.eieqb2mmewak). -- You will inform us of changes to your journals and [keep your journal records up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx)https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx. +- You will inform us of changes to your journals and [keep your journal records up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx). - You will [upload article metadata](https://doaj.org/publisher/uploadfile) to us if you are able to. We have [a zero-tolerance policy](https://doaj.org/about/#zero-tolerance-policy). We will treat you with respect and courtesy. We expect you to do the same. From de8e28a07c19bd2ebaf320e66674c2e73512a63d Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 14 Nov 2023 10:41:47 +0000 Subject: [PATCH 094/184] reinstate web root for publisher-reponsibilities --- portality/view/doaj.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 1292728293..89ec506415 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -484,10 +484,10 @@ def transparency(): def why_index(): return render_template("layouts/static_page.html", page_frag="/apply/why-index.html") -# TODO: Uncomment when ready for public access - S.E. 2022-03-14 -# @blueprint.route("/apply/publisher-responsibilities/") -# def publisher_responsibilities(): -# return render_template("layouts/static_page.html", page_frag="/apply/publisher-responsibilities.html") + +@blueprint.route("/apply/publisher-responsibilities/") +def publisher_responsibilities(): + return render_template("layouts/static_page.html", page_frag="/apply/publisher-responsibilities.html") @blueprint.route("/apply/copyright-and-licensing/") From 113fe4703e128e0ff4f368f24dd7bd6b28ceb0b7 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 14:38:24 +0100 Subject: [PATCH 095/184] Added 136-167 --- cms/pages/apply/publisher-responsibilities.md | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 8745266159..4b5dfe1b2d 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -134,22 +134,34 @@ We aim to process update requests within one month. - We will review the two records alongside each other to determine if the new title meets our basic criteria. If not, we will reject the application and withdraw the previous title. ### Ceased journals -If one of your indexed journals will be ceasing publication, let us know as soon as possible by contacting our Help Desk. When you contact us, include the ISSN and the title of your journal. -If another title is continuing the journal, you must submit a continuation. -Journal transfers -If a journal is moving to or from another publisher, let us know as soon as possible by contacting our Help Desk. We need the following information from you: -Journal title -ISSN -Transfer date -Name and email address of the old publisher -Name and email address of the new publisher -Journal queries -Journal is not listed in DOAJ -We might have withdrawn your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. Furthermore, we might have withdrawn the journal if it is inactive (ceased publishing) or the website is unavailable. + +- If one of your indexed journals will be ceasing publication, let us know as soon as possible by [contacting our Help Desk](mailto:helpdesk@doaj.org). When you contact us, include the ISSN and the title of your journal. +- If another title is continuing the journal, you must [submit a continuation](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.yjpqoy71fmiy). + +### Journal transfers + +If a journal is moving to or from another publisher, let us know as soon as possible by [contacting our Help Desk](mailto:helpdesk@doaj.org). We need the following information from you: + +- Journal title +- ISSN +- Transfer date +- Name and email address of the old publisher +- Name and email address of the new publisher + +## Journal queries + +### Journal is not listed in DOAJ +We have to withdraw your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. Furthermore, we have to withdraw the journal if it is inactive (ceased publishing) or the website is unavailable. + When we withdraw your journal, we will send you an email; remind yourself of the email address attached to your DOAJ account and check your Spam folder if you haven’t received anything. -Check our list of withdrawn journals. -You can ask for more information by contacting our Help Desk. -Journal is not listed on my dashboard -Contact our Help Desk with the journal title and ISSN; they will help you. -Unknown journal on my dashboard -Contact our Help Desk with the journal title and ISSN; they will help you. + +- Check [our list of withdrawn journals](https://docs.google.com/spreadsheets/d/183mRBRqs2jOyP0qZWXN8dUd02D4vL0Mov_kgYF8HORM/edit#gid=1650882189). +- You can ask for more information by [contacting our Help Desk](mailto:helpdesk@doaj.org). + +### Journal is not listed on my dashboard + +- [Contact our Help Desk](mailto:helpdesk@doaj.org) with the journal title and ISSN; they will help you. + +### Unknown journal on my dashboard + +- [Contact our Help Desk](mailto:helpdesk@doaj.org) with the journal title and ISSN; they will help you. From 5d73ad03b4f87fd7795822e08a289c3b3b9e09ed Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 14:42:32 +0100 Subject: [PATCH 096/184] Update publisher-responsibilities.md --- cms/pages/apply/publisher-responsibilities.md | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 4b5dfe1b2d..b92428e614 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -19,12 +19,12 @@ featuremap: ~~PublisherResponsibilities:Fragment~~ ### Your commitment to us -- You will [respond to questions](https://doaj.org/apply/guide/#the-application-process) from our editorial team promptly. -- You will [keep your contact details up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.eieqb2mmewak). -- You will inform us of changes to your journals and [keep your journal records up to date](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx). -- You will [upload article metadata](https://doaj.org/publisher/uploadfile) to us if you are able to. +- You will [respond to questions](/apply/guide/#the-application-process) from our editorial team promptly. +- You will [keep your contact details up to date](). +- You will inform us of changes to your journals and [keep your journal records up to date](). +- You will [upload article metadata](/publisher/uploadfile) to us if you are able to. -We have [a zero-tolerance policy](https://doaj.org/about/#zero-tolerance-policy). We will treat you with respect and courtesy. We expect you to do the same. +We have [a zero-tolerance policy](/about/#zero-tolerance-policy). We will treat you with respect and courtesy. We expect you to do the same. ### Emails from us @@ -40,15 +40,15 @@ If you use an institutional email address, your institution may apply extra secu ### Before you submit an application -1. Read our [Guide to applying](https://doaj.org/apply/guide/) (available in [several languages](https://doaj.org/apply/guide/#in-other-languages)) and make sure your journal meets all the basic criteria. -2. If you are applying for the first time, [register with us first](https://doaj.org/account/register). -3. If you already have a DOAJ account, [sign in](https://doaj.org/account/login)https://doaj.org/account/login to submit a new application. +1. Read our [Guide to applying](/apply/guide/) (available in [several languages](/apply/guide/#in-other-languages)) and make sure your journal meets all the basic criteria. +2. If you are applying for the first time, [register with us first](/account/register). +3. If you already have a DOAJ account, [sign in](/account/login) to submit a new application. -A [sample of the application form](https://doaj.org/static/doaj/docs/2023-07-04-DOAJQuestions.pdf) is available to download. Details of the [application process](https://doaj.org/apply/guide/#the-application-process) are in the guide. +A [sample of the application form](/static/doaj/docs/2023-07-04-DOAJQuestions.pdf) (Downloads a PDF) is available. Details of the [application process](/apply/guide/#the-application-process) are in the guide. ### Rejected applications -If we reject your application, you will receive an email explaining why and how long you must wait before applying again. [Read the information](https://doaj.org/apply/guide/#if-your-application-is-rejected) in the guide. +If we reject your application, you will receive an email explaining why and how long you must wait before applying again. [Read the information](/apply/guide/#if-your-application-is-rejected) in the guide. ### Accepted applications @@ -57,11 +57,11 @@ If we accept your application, you will receive an automated email that includes - Information on how to log into your publisher dashboard - Details on why it is helpful to upload article metadata to DOAJ -If you found your journal in DOAJ but didn’t get an email from us, there are [steps you can take](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.afqabefrwof3) to ensure our emails are delivered. +If you found your journal in DOAJ but didn’t get an email from us, there are [steps you can take]() to ensure our emails are delivered. During the review process, we might correct some of the information in your application. -You can [submit an update request](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx) if something about your journal has changed since you applied. +You can [submit an update request]() if something about your journal has changed since you applied. ## Your DOAJ account @@ -69,11 +69,13 @@ You can access your dashboard and account settings by logging into your account. ### Your publisher dashboard -On [your dashboard](https://doaj.org/publisher/), you can +(You must be signed in to access these links.) -- See [a list of your journals](https://doaj.org/publisher/journal) indexed in DOAJ -- [Submit an update request](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.ydj8hbjlzlzx) when journal details change -- [Upload](https://doaj.org/publisher/uploadfile) or [enter](https://doaj.org/publisher/metadata) article metadata for your journals +On [your dashboard](/publisher/), you can + +- See [a list of your journals](/publisher/journal) indexed in DOAJ +- [Submit an update request]() when journal details change +- [Upload](/publisher/uploadfile) or [enter](/publisher/metadata) article metadata for your journals - Download the Seal logo (only for journals awarded the DOAJ Seal). - Upload your full-text content (only for journals preserved via JASPER). @@ -93,7 +95,7 @@ The Settings button allows you to: ### Problems logging in -If you cannot log in, you can [reset your password](https://doaj.org/account/forgot). We will send an email containing a reset link. The link is valid for 24 hours. +If you cannot log in, you can [reset your password](/account/forgot). We will send an email containing a reset link. The link is valid for 24 hours. If you don’t get the email, check your Spam folder and check which address you registered with us. If you haven’t received the email after three hours, [contact us](mailto:helpdesk@doaj.org). @@ -106,16 +108,16 @@ To show that your journal is indexed in DOAJ or has the DOAJ Seal: - You may use the DOAJ or the Seal logo on your journal’s website but not on individual articles - You may only use the DOAJ or Seal logo on a platform or portal website if all of the platform’s journals are indexed or have been awarded the Seal - You can [download the DOAJ logos](https://doaj-kit.netlify.app/building-blocks/logotype/) -- If awarded, you can download the Seal logo from [your publisher dashboard](https://doaj.org/publisher/journal) +- If awarded, you can download the Seal logo from [your publisher dashboard](/publisher/journal) We have [guidelines](https://doaj-kit.netlify.app/building-blocks/logotype/) about how you can use our logo on your website or in printed material. ### Keeping your journal records up to date When something changes, you must keep your journal records in DOAJ up to date. -- [Contact our Help Desk](mailto:helpdesk@doaj.org) if you want to change the Title or ISSN of your journal, but [read this section first](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.yjpqoy71fmiy). +- [Contact our Help Desk](mailto:helpdesk@doaj.org) if you want to change the Title or ISSN of your journal, but [read this section first](). - [Contact our Help Desk](mailto:helpdesk@doaj.org) if your journal qualifies for the Seal. -- For all other changes, log into your DOAJ account to [submit an update request](https://doaj.org/publisher/journal). Use the black ‘Update’ button. +- For all other changes, log into your DOAJ account to [submit an update request](/publisher/journal). Use the black ‘Update’ button. ### When should I submit an update? @@ -136,7 +138,7 @@ We aim to process update requests within one month. ### Ceased journals - If one of your indexed journals will be ceasing publication, let us know as soon as possible by [contacting our Help Desk](mailto:helpdesk@doaj.org). When you contact us, include the ISSN and the title of your journal. -- If another title is continuing the journal, you must [submit a continuation](https://docs.google.com/document/d/17hv4Wz7l_5jC0FwXEB8DlEQjkFxtd5ZOa_6GsfrUqPk/edit#heading=h.yjpqoy71fmiy). +- If another title is continuing the journal, you must [submit a continuation](). ### Journal transfers From d3366502201f52d74122be534d694627f6b5205a Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 14:43:45 +0100 Subject: [PATCH 097/184] Update publisher-responsibilities.md --- cms/pages/apply/publisher-responsibilities.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index b92428e614..8a9c3c16c2 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -53,6 +53,7 @@ If we reject your application, you will receive an email explaining why and how ### Accepted applications If we accept your application, you will receive an automated email that includes: + - The URL of your journal record in DOAJ - Information on how to log into your publisher dashboard - Details on why it is helpful to upload article metadata to DOAJ From 74a0c15355b18f8b8075dfe688d0c70a04690864 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 15:07:12 +0100 Subject: [PATCH 098/184] Update publisher-responsibilities.md --- cms/pages/apply/publisher-responsibilities.md | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 8a9c3c16c2..61698c9d8f 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -62,7 +62,7 @@ If you found your journal in DOAJ but didn’t get an email from us, there are [ During the review process, we might correct some of the information in your application. -You can [submit an update request]() if something about your journal has changed since you applied. +You can [submit an update request](/apply/guide/#updating-your-journal-record) if something about your journal has changed since you applied. ## Your DOAJ account @@ -75,7 +75,7 @@ You can access your dashboard and account settings by logging into your account. On [your dashboard](/publisher/), you can - See [a list of your journals](/publisher/journal) indexed in DOAJ -- [Submit an update request]() when journal details change +- [Submit an update request](/apply/guide/#updating-your-journal-record) when journal details change - [Upload](/publisher/uploadfile) or [enter](/publisher/metadata) article metadata for your journals - Download the Seal logo (only for journals awarded the DOAJ Seal). - Upload your full-text content (only for journals preserved via JASPER). @@ -98,7 +98,7 @@ The Settings button allows you to: If you cannot log in, you can [reset your password](/account/forgot). We will send an email containing a reset link. The link is valid for 24 hours. -If you don’t get the email, check your Spam folder and check which address you registered with us. If you haven’t received the email after three hours, [contact us](mailto:helpdesk@doaj.org). +If you don’t get the email, check your Spam folder and which address you registered with us. If you haven’t received the email after three hours, [contact us](mailto:helpdesk@doaj.org). ## Your indexed journals @@ -106,8 +106,8 @@ If you don’t get the email, check your Spam folder and check which address you To show that your journal is indexed in DOAJ or has the DOAJ Seal: -- You may use the DOAJ or the Seal logo on your journal’s website but not on individual articles -- You may only use the DOAJ or Seal logo on a platform or portal website if all of the platform’s journals are indexed or have been awarded the Seal +- You may use the DOAJ or the Seal logo on your journal's website but not on individual articles +- You may only use the DOAJ or Seal logo on a platform or portal website if all of the platform's journals are indexed or have been awarded the Seal - You can [download the DOAJ logos](https://doaj-kit.netlify.app/building-blocks/logotype/) - If awarded, you can download the Seal logo from [your publisher dashboard](/publisher/journal) @@ -153,13 +153,14 @@ If a journal is moving to or from another publisher, let us know as soon as poss ## Journal queries -### Journal is not listed in DOAJ -We have to withdraw your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. Furthermore, we have to withdraw the journal if it is inactive (ceased publishing) or the website is unavailable. +### Journal is not listed (is withdrawn) -When we withdraw your journal, we will send you an email; remind yourself of the email address attached to your DOAJ account and check your Spam folder if you haven’t received anything. +When we withdraw your journal, we will send you an email. Remind yourself of the email address attached to your DOAJ account and check your Spam folder if you haven’t received anything. The email may include information on how long you must wait before you can apply again. -- Check [our list of withdrawn journals](https://docs.google.com/spreadsheets/d/183mRBRqs2jOyP0qZWXN8dUd02D4vL0Mov_kgYF8HORM/edit#gid=1650882189). -- You can ask for more information by [contacting our Help Desk](mailto:helpdesk@doaj.org). +- We have to withdraw your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. +- We have to withdraw the journal if it is inactive (ceased publishing) or the website is unavailable. + +You can also check our [list of withdrawn journals](https://docs.google.com/spreadsheets/d/183mRBRqs2jOyP0qZWXN8dUd02D4vL0Mov_kgYF8HORM/edit#gid=1650882189) and ask for more information by [contacting our Help Desk](mailto:helpdesk@doaj.org). ### Journal is not listed on my dashboard From f47fceb23139b529ff8625dd81d06755b9d51258 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 15:11:32 +0100 Subject: [PATCH 099/184] Updated l. 80 --- cms/data/nav.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/data/nav.yml b/cms/data/nav.yml index e8ec84fc22..1e28511620 100644 --- a/cms/data/nav.yml +++ b/cms/data/nav.yml @@ -77,7 +77,7 @@ entries: route: doaj.seal # ~~->Seal:WebRoute~~ - label: Transparency & best practice route: doaj.transparency # ~~->Transparency:WebRoute~~ - - label: Why index your journal in DOAJ? + - label: Publisher information route: doaj.why_index # ~~->WhyIndex:WebRoute~~ - label: Licensing & copyright route: doaj.copyright_and_licensing # ~~->CopyrightAndLicensing:WebRoute~~ From 44bed1ebc5751d48454a227a24c5efa9366a7bfd Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 15:14:52 +0100 Subject: [PATCH 100/184] Edited l. 30 --- cms/data/nav.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/data/nav.yml b/cms/data/nav.yml index 1e28511620..bed7656aee 100644 --- a/cms/data/nav.yml +++ b/cms/data/nav.yml @@ -27,7 +27,7 @@ entries: route: doaj.openurl # ~~->OpenURLDocumentation:WebRoute~~ - label: XML route: doaj.xml # ~~->XMLDocumentation:WebRoute~~ - - label: FAQs + - label: Metadata help route: doaj.faq # ~~->FAQ:WebRoute~~ - id: about label: About From 8ca8825235587346920437fc6a3ebd92593f1e7d Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 14 Nov 2023 15:23:29 +0100 Subject: [PATCH 101/184] Text overhaul --- cms/pages/docs/faq.md | 117 +++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 76 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 2aee5f4006..8ea99c3ea9 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -1,6 +1,6 @@ --- layout: sidenav -title: FAQs +title: Metadata help section: About toc: true sticky_sidenav: true @@ -8,78 +8,43 @@ featuremap: ~~FAQ:Fragment~~ --- -## How can I contribute to DOAJ? - -1. [Support us](/support/) with **a financial contribution** - + Contributions help keep the DOAJ service running and allow us to develop it further. -2. If you have evidence that a journal in DOAJ **might be questionable**, [contact us](/contact/). - + All information shared with DOAJ is done so in confidence and is never published. - + Include the ISSN of the journal when you email us. -3. If you find **a broken link or something that is out of date** or incomplete, [contact us](/contact/). - + We apppreciate the community providing us with this type of feedback. - + For broken links in articles, include the journal’s ISSN, title and the title of the article. -4. Become **a volunteer** - + From time to time, we put out a call for volunteers. Follow us on [Twitter](https://twitter.com/doajplus) or [our blog](https://blog.doaj.org/) to find out when the next call is published. - -## How do I reset my password? - - If you cannot log in, you can [reset your password](/account/forgot). An email is sent to your email address with a rest link. That link is only valid for 24 hours. Check your Spam folder to make sure the link doesn't go in there. If you do not receive the email, [contact us](/contact/). - -## I haven't heard anything about the application or update I submitted - - Most applications are processed within three (3) months. Some applications take longer. We aim to have all applications processed within six (6) months. Please wait 3 months until you ask for a status update. Due to the volume of emails that we receive, we cannot answer status updates for applications which are less than three months old. - - We aim to process updates to journals already in DOAJ in three (3) weeks. - -## I know a journal which should be in DOAJ but isn’t. What should I do? - - Contact the journal and ask them to submit an application. You can also send us the details of the journal—title and ISSN—and we will contact them. - -## The journal I am looking for isn’t in DOAJ. Why? - - Maybe the journal hasn’t applied to us or its application is still in progress. Maybe the journal was [removed from DOAJ](https://docs.google.com/spreadsheets/d/183mRBRqs2jOyP0qZWXN8dUd02D4vL0Mov_kgYF8HORM/edit#gid=1650882189&range=A1). - -## How do I report a bug or ask for technical help? - - If you are a registered GitHub user, or you don’t mind becoming one, you can [log a GitHub issue](https://github.com/DOAJ/doaj/issues/new/choose) directly in our repository. Alternatively, you can send the information we ask for below to [our helpdesk](mailto:helpdesk@doaj.org). - - When you report a bug, please include the following information. (You may not be able to provide every detail but fill in as many as you can.) - -- The URL of the page -- The time of the error -- Whether the error happened once or repeatedly -- If you are uploading XML, was it DOAJ or Crossref XML? - - Whether you are uploading a file or uploading from a link - - The file(s) you had problems with -- The exact error message as it appeared on your screen, including the detail under the 'show error details' link if there is one -- A screenshot of the error message -- The DOAJ account ID that you are logged in with - -## How do I update the information about my journal? - - Log into your account and go to your [Publisher dashboard](/publisher/). Under the '[My journals](/publisher/journal)' tab, you will see all the journals connected to your account. You can use the 'Update' button to submit an update to us. You cannot update the title or ISSN of your journal this way. - -## How do I upload article metadata? - - There are three ways to do this: - - via [our API](/docs/api/) - you will need an API key to do this. - - [uploading XML](/publisher/uploadfile) - - [entering the metadata manually](/publisher/metadata) - - You must have a publisher account to upload metadata to us. Log into your account and go to your [Publisher dashboard](/publisher/) to begin. - - [Help on creating and uploading XML](/docs/xml/) is available. - -## How do I update the title or ISSN of my journal? - - Contact us with the following details: - - old title and new title - - old ISSN and new ISSN - - date of the change - - whether the old title and ISSN have ceased to exist or not - -## How do I update the account details you have? - - You can update your account details by going to 'My Account', 'Settings'. You can update your name, email address and password. If you cannot log in, you can [reset your password](/account/forgot). - - If the account holder has changed and you need to add a new name and email address, please [contact us](/contact/). You should be able to provide the account name or the name of the previous account holder. +After your journal is indexed in DOAJ and you start to upload article metadata to us, we generate journal and article metadata. We make these publicly and freely available via different methods: + +- Our [Atom feed](https://staticdoaj.cottagelabs.com/feed) +- Our [OAI-PMH service](https://staticdoaj.cottagelabs.com/docs/oai-pmh/) +- [A journal CSV](https://staticdoaj.cottagelabs.com/csv) file (updates every 60 minutes) +- Our [API](https://staticdoaj.cottagelabs.com/docs/api/) +- On our website + +Our metadata is collected and incorporated into commercial discovery systems, library discovery portals and search engines around the world. Here are some of them: +- OCLC +- EBSCO products +- Clarivate's Proquest and Ex Libris products +- Clarivate's Web of Science +- SCOPUS +- Google Scholar +- Google +- Dimensions +- CONSER's MARC records + +## Uploading article metadata + +We are one of the most trusted and reliable providers of metadata about open access journals and articles. When publishers upload their article metadata to us, it increases the visibility of the journal and the articles. + +Choose how you want to upload article metadata to us. + +### API + +- **Speed and efficiency**: high +- **Level**: difficult +- **Formats accepted**: JSON +- **Maximum upload limit**: 50 MB +- **Requirements**: + - An API that will connect to ours + - An API key from your DOAJ account [link] +- **Help available?** Yes, via one of our API groups. [link] +- **Testing available**: only for Premier Supporters [link] +- **Documentation** Yes [link] +- **FAQS** Yes [link] +- **OJS plugin available?** Yes. Refer to the admin area of your journal. +- **OJS support**: for help, refer to the OJS Technical Forum From 941c895b5a6bcb24d696e37c1c8267e968266324 Mon Sep 17 00:00:00 2001 From: Sophy Date: Tue, 14 Nov 2023 14:33:40 +0000 Subject: [PATCH 102/184] Link to PublisherResponsibilities --- cms/data/nav.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/data/nav.yml b/cms/data/nav.yml index bed7656aee..07e09ce2ab 100644 --- a/cms/data/nav.yml +++ b/cms/data/nav.yml @@ -78,7 +78,7 @@ entries: - label: Transparency & best practice route: doaj.transparency # ~~->Transparency:WebRoute~~ - label: Publisher information - route: doaj.why_index # ~~->WhyIndex:WebRoute~~ + route: doaj.publisher_responsibilities # ~~->PublisherResponsibilities:WebRoute~~ - label: Licensing & copyright route: doaj.copyright_and_licensing # ~~->CopyrightAndLicensing:WebRoute~~ # Uncomment once this page is ready to be published From 1b18457f78440b9069014fb9158325fd42af3f4b Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 14 Nov 2023 16:52:44 +0000 Subject: [PATCH 103/184] reapply correct edges version --- portality/static/vendor/edges | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges index 7350e42008..990f422016 160000 --- a/portality/static/vendor/edges +++ b/portality/static/vendor/edges @@ -1 +1 @@ -Subproject commit 7350e420087f89364e71b4431e3a3130d4d00c69 +Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd From 52de78ee5c5c2bd8eb25c152f227a1fbba9c1c1e Mon Sep 17 00:00:00 2001 From: Zhan4i <122616204+Zhan4i@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:26:15 +0100 Subject: [PATCH 104/184] Update volunteers.yml --- cms/data/volunteers.yml | 120 +++++++--------------------------------- 1 file changed, 21 insertions(+), 99 deletions(-) diff --git a/cms/data/volunteers.yml b/cms/data/volunteers.yml index 9e09159ad7..3c1aa00afe 100644 --- a/cms/data/volunteers.yml +++ b/cms/data/volunteers.yml @@ -4,23 +4,13 @@ ## All editors ed: - -- name: Alain Chaple - area: Medical Sciences - year_since: - city: La Habana - country: Cuba - language: Spanish, English - photo: "alainchaple.png" - -- name: Alicia Aparicio - area: Humanities, Social Sciences +- name: Christine Davidian + area: Technology year_since: - city: Buenos Aires - country: Argentina - language: Spanish - featured: true - + city: Glassboro, NJ + country: USA + language: English, French + - name: Fei Yu area: Librarianship, Scholarly Publishing, Data Management year_since: @@ -29,14 +19,6 @@ ed: language: Chinese, English featured: true -- name: Geoff Husic - area: Humanities, Social Sciences - year_since: - city: Lawrence, KS - country: USA - language: Russian, German, English - featured: true - - name: Ikhwan Arief area: Science, Technology year_since: @@ -58,6 +40,13 @@ ed: country: Türkiye language: Turkish, English featured: true + +- name: Muhamad Ratodi + area: Public Health + year_since: + city: Surabaya + country: Indonesia + language: Indonesian, English - name: Natalia Pamuła-Cieślak area: Humanities, Social Sciences @@ -110,7 +99,14 @@ ed: ## All associate editors ass_ed: - +- name: Alain Chaple + area: Medical Sciences + year_since: + city: La Habana + country: Cuba + language: Spanish, English + photo: "alainchaple.png" + - name: Aleksandra Zawadzka area: Librarianship year_since: @@ -134,13 +130,6 @@ ass_ed: language: Persian (Farsi), English photo: "Amir Reza Asnafi.jpg" -- name: Ana Fresco-Santalla - area: Humanities - year_since: - city: Granada - country: Spain - language: Galician, Portuguese, Spanish, English - - name: Andista Candra Yusro area: Social Sciences year_since: @@ -198,13 +187,6 @@ ass_ed: language: Indonesian, English photo: "azhar.jpg" -- name: Barbara Łuszczek - area: Librarianship - year_since: - city: Bydgoszcz - country: Poland - language: Polish, English - - name: Ben Catt area: Librarianship, Research Support year_since: @@ -219,14 +201,6 @@ ass_ed: country: Indonesia language: Indonesian, English -- name: Cara Forster - area: Health Sciences and Library Science - year_since: - city: Germantown, Maryland - country: USA - language: English, American Sign Language - photo: "cara.jpg" - - name: Carlos Alegre area: Social Sciences year_since: @@ -243,13 +217,6 @@ ass_ed: language: Polish, English photo: "CezaryBorkowicz.jpg" -- name: Christine Davidian - area: Technology - year_since: - city: Glassboro, NJ - country: USA - language: English, French - - name: Constancy Aleru area: Medical Sciences year_since: @@ -265,13 +232,6 @@ ass_ed: country: Indonesia language: Indonesian, English photo: - -- name: Dania Deroy Dominguez - area: Social Sciences - year_since: - city: La Habana - country: Cuba - language: English, French - name: Daria Chrześcijańska area: Computer Science, Social Sciences @@ -620,13 +580,6 @@ ass_ed: language: Spanish, English photo: "MiltonRamirez.jpg" -- name: Muhamad Ratodi - area: Public Health - year_since: - city: Surabaya - country: Indonesia - language: Indonesian, English - - name: Muhamad Taufik Hidayat area: Humanities, Social Sciences year_since: @@ -734,14 +687,6 @@ ass_ed: country: Indonesia language: Indonesian, English -- name: Ramiro Doyenart - area: Health Sciences - year_since: - city: Rio Grande do Sul - country: Brazil - language: Spanish, Portuguese, English - photo: "RamiroDoyenart.jpeg" - - name: Ran Dang area: Biology, Medical Sciences year_since: @@ -771,14 +716,6 @@ ass_ed: country: USA language: English -- name: Richard James - area: Medicine - year_since: - city: Philadelphia, PA - country: USA - language: English - photo: "rjames.png" - - name: Roberto Sarmiento area: Engineering year_since: @@ -860,13 +797,6 @@ ass_ed: country: Romania language: Romanian, English, French -- name: Umar Khasan - area: Social Sciences - year_since: - city: Bangkalang - country: Indonesia - language: Bahasa Indonesian, English - - name: Utiya Hikmah area: Material Physics year_since: @@ -883,14 +813,6 @@ ass_ed: language: Kannada, English photo: "Vasanth.jpg" -- name: Weldy St-Fleur - area: Social Sciences - year_since: - city: São Paulo - country: Brazil - language: Portuguese, English - photo: "weldy.jpeg" - - name: Yalçın Tükel area: Sports recreation year_since: From e798ada20da82d53dd94c85714d0d2cdbc8ee7ec Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 08:15:07 +0100 Subject: [PATCH 105/184] Adding new text --- cms/pages/docs/faq.md | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 8ea99c3ea9..d896f76fcc 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -41,10 +41,28 @@ Choose how you want to upload article metadata to us. - **Maximum upload limit**: 50 MB - **Requirements**: - An API that will connect to ours - - An API key from your DOAJ account [link] -- **Help available?** Yes, via one of our API groups. [link] -- **Testing available**: only for Premier Supporters [link] -- **Documentation** Yes [link] -- **FAQS** Yes [link] -- **OJS plugin available?** Yes. Refer to the admin area of your journal. -- **OJS support**: for help, refer to the OJS Technical Forum + - The API key from [your DOAJ account](/account/login) +- **Help available?** Yes, via one of our API groups; search for 'Google Group DOAJ API' in your browser. +- **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) +- **Documentation** [Yes](/docs/api/) +- **FAQS** [Yes](/api/v3/docs#api-faq) +- **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. +- **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 + +### XML + +- **Speed and efficiency**: medium +- **Level**: medium +- **Formats accepted**: JSON +- **Maximum upload limit**: [DOAJ XML](/docs/xml/), [Crossref 5.3.1 XML](https://www.crossref.org/documentation/schema-library/metadata-deposit-schema-5-3-1/), [Crossref 4.4.2 XML](https://www.crossref.org/documentation/schema-library/resource-only-deposit-schema-4-4-2/) +- **Requirements**: + - A way to generate structured XML and validate it against the required XSD file. + - The XSD files: [DOAJ](/static/doaj/doajArticles.xsd), [Crossref 5.3.1](/static/crossref/crossref5.3.1.xsd), [Crossref 4.4.2](/static/crossref/crossref4.4.2.xsd) +- **Help available?** + - For [DOAJ XML](/docs/xml/) + - For Crossref XML, contact [Crossref support](mailto:support@crossref.org) +- **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) +- **Documentation** [Yes](/docs/xml/) +- **FAQS** No +- **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. +- **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 From 54a4a424a343ba3e4d01d70198f5b5b2e779f7ff Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 08:22:33 +0100 Subject: [PATCH 106/184] Update faq.md --- cms/pages/docs/faq.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index d896f76fcc..188ceab209 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -38,7 +38,7 @@ Choose how you want to upload article metadata to us. - **Speed and efficiency**: high - **Level**: difficult - **Formats accepted**: JSON -- **Maximum upload limit**: 50 MB +- **Maximum upload limit**: 50MB - **Requirements**: - An API that will connect to ours - The API key from [your DOAJ account](/account/login) @@ -54,15 +54,29 @@ Choose how you want to upload article metadata to us. - **Speed and efficiency**: medium - **Level**: medium - **Formats accepted**: JSON -- **Maximum upload limit**: [DOAJ XML](/docs/xml/), [Crossref 5.3.1 XML](https://www.crossref.org/documentation/schema-library/metadata-deposit-schema-5-3-1/), [Crossref 4.4.2 XML](https://www.crossref.org/documentation/schema-library/resource-only-deposit-schema-4-4-2/) +- **Maximum upload limit**: 50MB - **Requirements**: - A way to generate structured XML and validate it against the required XSD file. - The XSD files: [DOAJ](/static/doaj/doajArticles.xsd), [Crossref 5.3.1](/static/crossref/crossref5.3.1.xsd), [Crossref 4.4.2](/static/crossref/crossref4.4.2.xsd) -- **Help available?** +- **Help available?** Yes - For [DOAJ XML](/docs/xml/) - For Crossref XML, contact [Crossref support](mailto:support@crossref.org) - **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) -- **Documentation** [Yes](/docs/xml/) +- **Documentation** [DOAJ XML](/docs/xml/), [Crossref 5.3.1 XML](https://www.crossref.org/documentation/schema-library/metadata-deposit-schema-5-3-1/), [Crossref 4.4.2 XML](https://www.crossref.org/documentation/schema-library/resource-only-deposit-schema-4-4-2/) - **FAQS** No - **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. - **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 + +### Enter article metadata manually + +- **Speed and efficiency**: low +- **Level**: easy +- **Formats accepted**: text, entered via [our webform](/publisher/metadata) +- **Maximum upload limit**: N/A +- **Requirements**: + - Plain text only + - No email addresses + - The abstract metadata for the article: title, full text URL, DOI (if applicable), author names, ORCiD (if applicable), affiliations, publication date, ISSN(s), Volume/Issue/Page (if applicable), abstract +- **Help available?** Yes. [Contact our Help Desk](mailto:helpdesk@doaj.org). +- **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) +- **Documentation** No From 7aa0372a5d1680d8a26651c666775196037da81f Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 08:23:41 +0100 Subject: [PATCH 107/184] Update faq.md --- cms/pages/docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 188ceab209..d49a4699a5 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -53,7 +53,7 @@ Choose how you want to upload article metadata to us. - **Speed and efficiency**: medium - **Level**: medium -- **Formats accepted**: JSON +- **Formats accepted**: DOAJ and Crossref XML - **Maximum upload limit**: 50MB - **Requirements**: - A way to generate structured XML and validate it against the required XSD file. From d0e56a7ca9f428c809692b9c55eca4c7a9a60400 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:04:55 +0100 Subject: [PATCH 108/184] Update faq.md --- cms/pages/docs/faq.md | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index d49a4699a5..ab0b5ae574 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -48,6 +48,13 @@ Choose how you want to upload article metadata to us. - **FAQS** [Yes](/api/v3/docs#api-faq) - **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. - **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 +- **Troubleshooting uploads**: please [submit a bug report](https://github.com/DOAJ/doaj/issues/new/choose) (via GitHub) or [contact us](mailto:helpdesk@doaj.org) with the following details: + - The time of the file upload. (If you saw the error more than 12 hours ago, please try the upload again before you contact us.) + - Whether the error happened once or repeatedly + - The exact error message as it appeared during the upload. Include a screenshot. + - The DOAJ account ID that you are logged in with. You will find this under 'Settings' in the Dashboard dropdown menu. + - The file(s) you had problems with + - The ISSN(s) of the journal ### XML @@ -59,13 +66,23 @@ Choose how you want to upload article metadata to us. - A way to generate structured XML and validate it against the required XSD file. - The XSD files: [DOAJ](/static/doaj/doajArticles.xsd), [Crossref 5.3.1](/static/crossref/crossref5.3.1.xsd), [Crossref 4.4.2](/static/crossref/crossref4.4.2.xsd) - **Help available?** Yes - - For [DOAJ XML](/docs/xml/) - - For Crossref XML, contact [Crossref support](mailto:support@crossref.org) + - For creating XML, see our [DOAJ XML](/docs/xml/) documentation + - For help with [error messages when uploading XML](/publisher/help#explanations) + - For help with Crossref XML, contact [Crossref support](mailto:support@crossref.org) - **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) - **Documentation** [DOAJ XML](/docs/xml/), [Crossref 5.3.1 XML](https://www.crossref.org/documentation/schema-library/metadata-deposit-schema-5-3-1/), [Crossref 4.4.2 XML](https://www.crossref.org/documentation/schema-library/resource-only-deposit-schema-4-4-2/) - **FAQS** No - **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. - **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 +- **Troubleshooting uploads**: please [submit a bug report](https://github.com/DOAJ/doaj/issues/new/choose) (via GitHub) or [contact us](mailto:helpdesk@doaj.org) with the following details: + - The time of the file upload. (If you saw the error more than 12 hours ago, please try the upload again before you contact us.) + - Whether the error happened once or repeatedly + - Whether you are uploading DOAJ or Crossref XML + - Whether you are uploading a file or uploading from a link + - The exact error message as it appears in the “Notes” column of the [History of uploads section](/publisher/uploadfile), including the detail under the 'show error details' link. + - A screenshot of the error message with the 'show error details' link expanded. + - The DOAJ account ID that you are logged in with + - The file(s) you had problems with ### Enter article metadata manually @@ -76,7 +93,12 @@ Choose how you want to upload article metadata to us. - **Requirements**: - Plain text only - No email addresses - - The abstract metadata for the article: title, full text URL, DOI (if applicable), author names, ORCiD (if applicable), affiliations, publication date, ISSN(s), Volume/Issue/Page (if applicable), abstract + - The abstract metadata for the article: title, full-text URL, DOI (if applicable), author names, ORCiD (if applicable), affiliations, publication date, ISSN(s), Volume/Issue/Page (if applicable), abstract - **Help available?** Yes. [Contact our Help Desk](mailto:helpdesk@doaj.org). - **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) - **Documentation** No +- **Troubleshooting**: you must be careful to enter the Print ISSN and Online ISSN in the right field. + +## FAQs for metadata uploads + +When you upload article XML to us, we will let you know if there are problems with the file From 4efa992f005c3f3403e70798ad2bdadd6340f62e Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:17:40 +0100 Subject: [PATCH 109/184] Various corrections and edits --- portality/templates/publisher/help.html | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/portality/templates/publisher/help.html b/portality/templates/publisher/help.html index abf09a4cff..32c9c29c65 100644 --- a/portality/templates/publisher/help.html +++ b/portality/templates/publisher/help.html @@ -20,11 +20,11 @@

    Uploading metadata/article content

    There are instructions on how to prepare and upload your XML file on our XML documentation page.

    -

    Are you receiving an error about one of your ISSNs that you haven't seen before? We recently changed the rules for uploading article metadata. We now require that a Print ISSN is in an issn tag and the Online ISSN is in an eissn tag. [See our sample XML file](https://doaj.org/docs/xml/#example-doaj-xml-file) for more information.

    +

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN is in the eissn tag. [See our sample XML file](/docs/xml/#example-doaj-xml-file) for more information.

    Failed XML uploads explained

    -

    This section explains the error messages that you may see when you upload article XML. Use the message in the 'Notes' column of your History of uploads table to correct your XML.

    +

    This section explains the error messages you may see when uploading article XML. Use the message in the 'Notes' column of your History of uploads table to correct your XML.

    If you cannot get the XML to upload, please submit a bug report or contact us with the following details:

    @@ -67,7 +67,7 @@

    Failed XML uploads explained

    - All articles in file failed to import + All articles in the file failed to import There were no successful article imports and one or more failed article imports in the uploaded file(s). @@ -129,7 +129,7 @@

    Failed XML uploads explained

    The URL could not be accessed - The link you gave could not be resolved or the page no longer exists (e.g. it responded with a 404 Not Found). + The link you gave could not be resolved, or the page no longer exists (e.g. it responded with a 404 Not Found). In your answer to the “Provide a URL where we can download the XML” field, check that (1) the URL is the correct URL and (2) there are no access restrictions on the URL (e.g., authentication is required, or the URL is IP-restricted to your organisation). @@ -145,7 +145,7 @@

    Failed XML uploads explained

    Try to upload it again: it may be an Internet connection issue.

    - If you are still unable to get the upload to work after a several attempts, please submit a bug report or contact us with the relevant details. + If you are still unable to get the upload to work after several attempts, please submit a bug report or contact us with the relevant details. @@ -158,11 +158,11 @@

    Failed XML uploads explained

    Check that the file really is valid XML.

    - In some cases, you may have uploaded/given the wrong file, so double-check that it is the correct one. You could get this error, for example, if you have uploaded a Word document or a PDF by mistake.
    + Sometimes, you may have uploaded/given the wrong file, so double-check that it is correct. For example, you could get this error if you have uploaded a Word document or a PDF by mistake.

    - In other cases, files can look like XML, but still be invalid. They should be run through an XML validator to ensure they have the correct structure and that there are no invalid characters.
    + In other cases, files can look like XML but still be invalid. They should be run through an XML validator to ensure they have the correct structure and that there are no invalid characters.

    - Sometimes, special characters – especially those in scientific articles – are submitted in the wrong format. XML does not support many special characters as-is but those that are can be found on Wikipedia. + Sometimes, special characters—especially those in scientific articles—are submitted in the wrong format. XML does not support many special characters as-is, but those that are supported can be found on Wikipedia. @@ -173,11 +173,11 @@

    Failed XML uploads explained

    No XML schema could be found. The schema, in particular DOAJ’s standard schema, is necessary to read an XML document. - Check that the file conforms to the DOAJ schema, using a validator.
    + Check that the file conforms to the DOAJ schema using a validator.

    - Often the problem arises because the XML is missing a required tag. The schema validation will fail if you try to upload XML to DOAJ that is missing a specific tag. You can see exactly which tags are required here.
    + Often, the problem arises because the XML is missing a required tag. The schema validation will fail if you try to upload XML to DOAJ that is missing a specific tag. You can see exactly which tags are required here.

    - If you are trying to upload a file that was automatically generated by the OJS DOAJ plugin, then you should first contact OJS for help.
    + If you are trying to upload a file automatically generated by the OJS DOAJ plugin, then you should ask for help in the PKP OJS Software Support Forum.

    The DOAJ schema is on our site, here: http://doaj.org/static/doaj/doajArticles.xsd

    @@ -199,13 +199,13 @@

    Failed XML uploads explained


    You can only upload XML for the journals that are in DOAJ and that appear in the list under Your journals in the publisher area. If you have more than one account and you would like to merge them, please contact us.

    - The article’s stated ISSNs must match to a journal that is owned by the user account from which you are providing the XML file. If the ISSNs do not match any journals or match a journal that is owned by a different account, the import of that article will fail.
    + The ISSNs stated in the article must match a journal owned by the user account from which you are providing the XML file. If the ISSNs do not match any journals or match a journal that is owned by a different account, the import of that article will fail.

    - Note that it is also possible that if an article has more than one ISSN (i.e. an E-ISSN and a P-ISSN), that those ISSNs could match more than one Journal (one ISSN matching one journal, the other ISSN matching another). If both matched journals are owned by your account, this will not stop the article from being imported, but if one of the matched journals is owned by a different account this will cause the import of the article to fail.
    + Note: it is also possible that if an article has more than one ISSN (i.e. an E-ISSN and a P-ISSN), those ISSNs could match more than one Journal (one ISSN matching one journal, the other ISSN matching another). If your account owns both matched journals, this will not stop the article from being imported, but if one of the matched journals is owned by a different account this will cause the import of the article to fail.

    - A journal may have two ISSNs: an ISSN for the print version and an ISSN for the electronic version. Sometimes the ISSNs of the journal have changed. + A journal may have two ISSNs: an ISSN for the print version and an ISSN for the electronic version. Sometimes, the ISSNs of the journal have changed.
    - The print and online ISSNs you have supplied are identical. If you supply 2 ISSNs they must be different: an ISSN for the print version and an ISSN for the electronic version. + The print and electronic ISSNs you have supplied are identical. If you supply two ISSNs, they must be different: an ISSN for the print version and an ISSN for the electronic version.
    ISSNs provided don't match any journal. We do not have a record of one or both of those ISSNs in DOAJ.

    @@ -227,7 +227,7 @@

    Failed XML uploads explained

    One or more articles in this batch have duplicate identifiers - At least two of the articles in the uploaded file contain the same DOI and/or the same full-text url + At least two of the articles in the uploaded file contain the same DOI and/or the same full-text URL Make sure that there are no duplicated articles in the file and that you are using the correct DOIs and URLs for all the articles. @@ -238,7 +238,7 @@

    Failed XML uploads explained

    One or more articles in this batch matched multiple articles as duplicates; entire batch ingest halted - At least one of your articles matched two or more other articles already in DOAJ, and as a result we don't know which one to update. + At least one of your articles matched two or more other articles already in DOAJ, so we don't know which one to update. Ensure that all your articles have the correct DOIs and full-text links. If it still doesn’t work please submit a bug report or contact us with the details; we may need to clean up your existing articles manually. From 3b049e6073ffd867ba369043e529b1dd78d6f564 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:19:11 +0100 Subject: [PATCH 110/184] Update help.html --- portality/templates/publisher/help.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/publisher/help.html b/portality/templates/publisher/help.html index 32c9c29c65..64e575f9c0 100644 --- a/portality/templates/publisher/help.html +++ b/portality/templates/publisher/help.html @@ -20,7 +20,7 @@

    Uploading metadata/article content

    There are instructions on how to prepare and upload your XML file on our XML documentation page.

    -

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN is in the eissn tag. [See our sample XML file](/docs/xml/#example-doaj-xml-file) for more information.

    +

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN is in the eissn tag. See our sample XML file for more information.

    Failed XML uploads explained

    From bd4f88781d7548a1a3eba7abeb081be8776e5f74 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:21:27 +0100 Subject: [PATCH 111/184] Update help.html --- portality/templates/publisher/help.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/publisher/help.html b/portality/templates/publisher/help.html index 64e575f9c0..ff1cfa3960 100644 --- a/portality/templates/publisher/help.html +++ b/portality/templates/publisher/help.html @@ -20,7 +20,7 @@

    Uploading metadata/article content

    There are instructions on how to prepare and upload your XML file on our XML documentation page.

    -

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN is in the eissn tag. See our sample XML file for more information.

    +

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the tag and the Electronic ISSN in the tag. See our sample XML file for more information.

    Failed XML uploads explained

    From b26ebfda5ebe2b22072f9767e8282dc5e743e8c5 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Wed, 15 Nov 2023 08:22:26 +0000 Subject: [PATCH 112/184] cosmetic changes to migrate script --- portality/migrate/3575_make_notes_searchable/migrate.json | 2 +- portality/scripts/es_reindex.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/portality/migrate/3575_make_notes_searchable/migrate.json b/portality/migrate/3575_make_notes_searchable/migrate.json index 8ca5754bca..f129942092 100644 --- a/portality/migrate/3575_make_notes_searchable/migrate.json +++ b/portality/migrate/3575_make_notes_searchable/migrate.json @@ -1,5 +1,5 @@ { - "new_version": "-20231109", + "new_version": "-20231114", "old_version": "", "types": [ { diff --git a/portality/scripts/es_reindex.py b/portality/scripts/es_reindex.py index 920ce5c938..4893b1afc4 100644 --- a/portality/scripts/es_reindex.py +++ b/portality/scripts/es_reindex.py @@ -4,7 +4,7 @@ run the script: portality/scripts/es_reindex.py ex: -portality/scripts/es_reindex.py -u /portality/migrate/3575_make_notes_searchable/migrate.json +DOAJENV=dev python portality/scripts/es_reindex.py portality/migrate/3575_make_notes_searchable/migrate.json example json file: { @@ -46,7 +46,7 @@ def do_import(config): previous_version = config.get("old_version") # get the types we are going to work with - print("==Carrying out the following import==") + print("\n==Reindexing the following types / indices==") for s in config.get("types", []): if s.get("migrate", False) is True: print(s.get("type")) @@ -63,7 +63,7 @@ def do_import(config): # Iterate through the types then # 1. create new index # 2. re index with old index - # 3. set alias for new index + # 3. set alias for new index (if requested) for s in config.get("types", []): import_type = s["type"] if import_type in mappings: @@ -130,6 +130,6 @@ def do_import(config): args = parser.parse_args() with open(args.config, "r", encoding="utf-8") as f: - config = json.loads(f.read()) + config = json.load(f) do_import(config) \ No newline at end of file From fb690738e67690161db4a19cad0d5b25cc1d185f Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:22:57 +0100 Subject: [PATCH 113/184] Update help.html --- portality/templates/publisher/help.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/publisher/help.html b/portality/templates/publisher/help.html index ff1cfa3960..6e2f81a64c 100644 --- a/portality/templates/publisher/help.html +++ b/portality/templates/publisher/help.html @@ -20,7 +20,7 @@

    Uploading metadata/article content

    There are instructions on how to prepare and upload your XML file on our XML documentation page.

    -

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the tag and the Electronic ISSN in the tag. See our sample XML file for more information.

    +

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN in the eissn tag. See our sample XML file for more information.

    Failed XML uploads explained

    From 962bf7b6209c211646d8d22bed0600bed2af0378 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 09:25:04 +0100 Subject: [PATCH 114/184] Update faq.md --- cms/pages/docs/faq.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index ab0b5ae574..8c090b5ffb 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -51,7 +51,7 @@ Choose how you want to upload article metadata to us. - **Troubleshooting uploads**: please [submit a bug report](https://github.com/DOAJ/doaj/issues/new/choose) (via GitHub) or [contact us](mailto:helpdesk@doaj.org) with the following details: - The time of the file upload. (If you saw the error more than 12 hours ago, please try the upload again before you contact us.) - Whether the error happened once or repeatedly - - The exact error message as it appeared during the upload. Include a screenshot. + - The exact error message that appeared during the upload. Include a screenshot. - The DOAJ account ID that you are logged in with. You will find this under 'Settings' in the Dashboard dropdown menu. - The file(s) you had problems with - The ISSN(s) of the journal @@ -79,7 +79,7 @@ Choose how you want to upload article metadata to us. - Whether the error happened once or repeatedly - Whether you are uploading DOAJ or Crossref XML - Whether you are uploading a file or uploading from a link - - The exact error message as it appears in the “Notes” column of the [History of uploads section](/publisher/uploadfile), including the detail under the 'show error details' link. + - The exact error message shown in the 'Notes' column of the [History of uploads section](/publisher/uploadfile), including the detail under the 'show error details' link. - A screenshot of the error message with the 'show error details' link expanded. - The DOAJ account ID that you are logged in with - The file(s) you had problems with @@ -97,7 +97,7 @@ Choose how you want to upload article metadata to us. - **Help available?** Yes. [Contact our Help Desk](mailto:helpdesk@doaj.org). - **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) - **Documentation** No -- **Troubleshooting**: you must be careful to enter the Print ISSN and Online ISSN in the right field. +- **Troubleshooting**: you must be careful to enter the Print ISSN and Electronic ISSN in the right field. ## FAQs for metadata uploads From e61edfb3adae84689fa034f84b0d4d8acab44cea Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Wed, 15 Nov 2023 09:00:46 +0000 Subject: [PATCH 115/184] Read-only mode for searchable notes release --- production.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/production.cfg b/production.cfg index d9d501f253..2a90ac7e81 100644 --- a/production.cfg +++ b/production.cfg @@ -46,9 +46,9 @@ LOGIN_VIA_ACCOUNT_ID = True # This puts the UI into READ_ONLY mode # ~~->ReadOnlyMode:Feature~~ -READ_ONLY_MODE = False +READ_ONLY_MODE = True # This puts the cron jobs into READ_ONLY mode -SCRIPTS_READ_ONLY_MODE = False +SCRIPTS_READ_ONLY_MODE = True # 2022-09-14 issue #3369 disable Plausible custom events ( set to None) # 2023-01-12 issue #3443 re-enable Plausible custom events @@ -64,4 +64,4 @@ KAFKA_BROKER = "kafka://10.131.35.14:9092" KAFKA_BOOTSTRAP_SERVER = "10.131.35.14:9092" # https://github.com/DOAJ/doajPM/issues/3565 2023-03-07 -PRESERVATION_PAGE_UNDER_MAINTENANCE = False +PRESERVATION_PAGE_UNDER_MAINTENANCE = True From cd4607cff123f65ae5a74989545763568c26668e Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Wed, 15 Nov 2023 09:21:46 +0000 Subject: [PATCH 116/184] Version bump for searchable notes release --- portality/settings.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 8213730ace..d561e977d8 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.4.7" +DOAJ_VERSION = "6.5.0" API_VERSION = "3.0.1" ###################################### diff --git a/setup.py b/setup.py index 49da5a8c38..ce7e2121e4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='6.4.7', + version='6.5.0', packages=find_packages(), install_requires=[ "awscli==1.20.50", From 241cf921295133f3634e5370606b44aa19050771 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Wed, 15 Nov 2023 09:24:09 +0000 Subject: [PATCH 117/184] Add new name for searchable notes index --- portality/migrate/3575_make_notes_searchable/migrate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/migrate/3575_make_notes_searchable/migrate.json b/portality/migrate/3575_make_notes_searchable/migrate.json index f129942092..f1d8c58054 100644 --- a/portality/migrate/3575_make_notes_searchable/migrate.json +++ b/portality/migrate/3575_make_notes_searchable/migrate.json @@ -1,5 +1,5 @@ { - "new_version": "-20231114", + "new_version": "-20231115_searchable_notes", "old_version": "", "types": [ { From 6a012790fe35cc7673f8077cb9acee61964e6ab8 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Wed, 15 Nov 2023 10:00:57 +0000 Subject: [PATCH 118/184] Release read-only mode --- production.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/production.cfg b/production.cfg index 2a90ac7e81..d9d501f253 100644 --- a/production.cfg +++ b/production.cfg @@ -46,9 +46,9 @@ LOGIN_VIA_ACCOUNT_ID = True # This puts the UI into READ_ONLY mode # ~~->ReadOnlyMode:Feature~~ -READ_ONLY_MODE = True +READ_ONLY_MODE = False # This puts the cron jobs into READ_ONLY mode -SCRIPTS_READ_ONLY_MODE = True +SCRIPTS_READ_ONLY_MODE = False # 2022-09-14 issue #3369 disable Plausible custom events ( set to None) # 2023-01-12 issue #3443 re-enable Plausible custom events @@ -64,4 +64,4 @@ KAFKA_BROKER = "kafka://10.131.35.14:9092" KAFKA_BOOTSTRAP_SERVER = "10.131.35.14:9092" # https://github.com/DOAJ/doajPM/issues/3565 2023-03-07 -PRESERVATION_PAGE_UNDER_MAINTENANCE = True +PRESERVATION_PAGE_UNDER_MAINTENANCE = False From 0020c24355f08dd957b6052a9c5a2ac0455006ce Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Wed, 15 Nov 2023 11:17:07 +0100 Subject: [PATCH 119/184] Update faq.md --- cms/pages/docs/faq.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 8c090b5ffb..1de2081cc6 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -99,6 +99,14 @@ Choose how you want to upload article metadata to us. - **Documentation** No - **Troubleshooting**: you must be careful to enter the Print ISSN and Electronic ISSN in the right field. -## FAQs for metadata uploads +## Help with metadata uploads -When you upload article XML to us, we will let you know if there are problems with the file +### My authors have multiple affiliations + +We are currently unable to display more than one affiliation per author. We are investigating how we can change this. More information will be posted on our blog. + +### My article abstracts are in more than one language + +Our XML format only supports one language for Article Title and Abstract. We are working on a solution that will allow multiple languages to be uploaded to us and displayed. + +Metadata containing multiple languages can still be uploaded to us. However, you cannot choose which language is displayed. Please only send us one language to avoid your articles being displayed in a mixture of languages. From a5f912d91d188854220afb03cfecd69d5cdd081c Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Thu, 16 Nov 2023 16:01:39 +0100 Subject: [PATCH 120/184] Added new text --- cms/pages/docs/faq.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 1de2081cc6..372ff48ada 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -110,3 +110,29 @@ We are currently unable to display more than one affiliation per author. We are Our XML format only supports one language for Article Title and Abstract. We are working on a solution that will allow multiple languages to be uploaded to us and displayed. Metadata containing multiple languages can still be uploaded to us. However, you cannot choose which language is displayed. Please only send us one language to avoid your articles being displayed in a mixture of languages. + +### I am seeing a 403 forbidden error when I try to upload article metadata + +You may see the 403 forbidden error for different reasons. These apply to both the API and uploading XML. + +- ISSNs + - You may be sending us an extra ISSN that we don’t have in your journal record. + - You may be sending only one ISSN, but we have two in the journal record. + - We may have the journal's ISSNs in an old version of your journal record. +- Wrong account + - You may be sending us an ISSN that belongs to a journal attached to a different account. +- You are trying to update an article's Full Text URL (FTUs) or DOI + - Two articles with the same FTU or DOI are not allowed. + - Please contact us if you want to update the URLs or DOIs of your articles. We need to delete the old versions first. + +### I am seeing a timeout error or a 'blocked' message from Cloudflare + +If you see a timeout error, please try splitting your upload into smaller files, even if your file is under our 50MB limit. Many may be uploading content to us, and the server is taking longer than usual to collect your file. + +If you see a screen from Cloudflare that says you have been blocked, please [contact us](mailto:helpdesk@doaj.org). Include a screenshot that shows the Ray ID at the very bottom of the page. We need this to troubleshoot the problem. + +## Downloading your metadata + +You can download our metadata about your journal by [downloading our CSV](https://doaj.org/csv). + +You can download your article metadata by [using our API](https://doaj.org/docs/api/) or by using our [public data dump service](https://doaj.org/docs/public-data-dump/)https://doaj.org/docs/public-data-dump/. From a038569dc9ebac1a7334821aaf831f75911f9ba7 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Thu, 16 Nov 2023 16:03:45 +0100 Subject: [PATCH 121/184] Update faq.md --- cms/pages/docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 372ff48ada..6e47d029bb 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -135,4 +135,4 @@ If you see a screen from Cloudflare that says you have been blocked, please [con You can download our metadata about your journal by [downloading our CSV](https://doaj.org/csv). -You can download your article metadata by [using our API](https://doaj.org/docs/api/) or by using our [public data dump service](https://doaj.org/docs/public-data-dump/)https://doaj.org/docs/public-data-dump/. +You can download your article metadata by [using our API](https://doaj.org/docs/api/) or by using our [public data dump service](/docs/public-data-dump/). From b3f0775f90fe33ab6202145f057939782a8cc203 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Thu, 16 Nov 2023 16:07:24 +0100 Subject: [PATCH 122/184] Update faq.md --- cms/pages/docs/faq.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 6e47d029bb..d09877e040 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -65,16 +65,16 @@ Choose how you want to upload article metadata to us. - **Requirements**: - A way to generate structured XML and validate it against the required XSD file. - The XSD files: [DOAJ](/static/doaj/doajArticles.xsd), [Crossref 5.3.1](/static/crossref/crossref5.3.1.xsd), [Crossref 4.4.2](/static/crossref/crossref4.4.2.xsd) -- **Help available?** Yes - - For creating XML, see our [DOAJ XML](/docs/xml/) documentation - - For help with [error messages when uploading XML](/publisher/help#explanations) - - For help with Crossref XML, contact [Crossref support](mailto:support@crossref.org) - **Testing available**: on a case-by-case basis and only for [publisher supporters](/support/publisher-supporters/) - **Documentation** [DOAJ XML](/docs/xml/), [Crossref 5.3.1 XML](https://www.crossref.org/documentation/schema-library/metadata-deposit-schema-5-3-1/), [Crossref 4.4.2 XML](https://www.crossref.org/documentation/schema-library/resource-only-deposit-schema-4-4-2/) - **FAQS** No - **OJS plugin available?** Yes. Refer to [PKP documentation](https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin)https://docs.pkp.sfu.ca/admin-guide/3.3/en/data-import-and-export#doaj-export-plugin. - **OJS support**: for help, refer to [the OJS Technical Forum](https://forum.pkp.sfu.ca/c/questions/5)https://forum.pkp.sfu.ca/c/questions/5 -- **Troubleshooting uploads**: please [submit a bug report](https://github.com/DOAJ/doaj/issues/new/choose) (via GitHub) or [contact us](mailto:helpdesk@doaj.org) with the following details: +- **Help available?** Yes + - For creating XML, see our [DOAJ XML](/docs/xml/) documentation + - For explanations of [specific error messages when uploading XML](/publisher/help#explanations) + - For help with Crossref XML, contact [Crossref support](mailto:support@crossref.org) +- **Troubleshooting uploads**: if our [error message definitions](/publisher/help#explanations) don't help you, please [submit a bug report](https://github.com/DOAJ/doaj/issues/new/choose) (via GitHub) or [contact us](mailto:helpdesk@doaj.org) with the following details: - The time of the file upload. (If you saw the error more than 12 hours ago, please try the upload again before you contact us.) - Whether the error happened once or repeatedly - Whether you are uploading DOAJ or Crossref XML @@ -111,7 +111,7 @@ Our XML format only supports one language for Article Title and Abstract. We are Metadata containing multiple languages can still be uploaded to us. However, you cannot choose which language is displayed. Please only send us one language to avoid your articles being displayed in a mixture of languages. -### I am seeing a 403 forbidden error when I try to upload article metadata +### I am seeing a 403 forbidden error You may see the 403 forbidden error for different reasons. These apply to both the API and uploading XML. @@ -125,7 +125,7 @@ You may see the 403 forbidden error for different reasons. These apply to both t - Two articles with the same FTU or DOI are not allowed. - Please contact us if you want to update the URLs or DOIs of your articles. We need to delete the old versions first. -### I am seeing a timeout error or a 'blocked' message from Cloudflare +### I am seeing a timeout or a 'blocked' error If you see a timeout error, please try splitting your upload into smaller files, even if your file is under our 50MB limit. Many may be uploading content to us, and the server is taking longer than usual to collect your file. From 168524d04ad34e66f62a9fd4ef239b6637d31856 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Thu, 16 Nov 2023 16:15:31 +0100 Subject: [PATCH 123/184] Update help.html --- portality/templates/publisher/help.html | 37 +++---------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/portality/templates/publisher/help.html b/portality/templates/publisher/help.html index 6e2f81a64c..dc87041992 100644 --- a/portality/templates/publisher/help.html +++ b/portality/templates/publisher/help.html @@ -1,43 +1,14 @@ {% extends "publisher/publisher_base.html" %} -{% block page_title %}Help for publishers{% endblock %} +{% block page_title %}Explanations of XML errors{% endblock %} {% block publisher_content %} -

    Help for publishers

    +

    Explanation of XML errors

    -

    Uploading metadata/article content

    +

    The table below explains the different error messages that may appear when uploading article XML. Use the detailed information in the 'Notes' column of your History of uploads table to correct your XML.

    -

    There are three ways to upload article metadata to DOAJ:

    - -
      -
    1. API (you will need an API key to do this),
      2. -
    2. uploading XML, or
      3. -
    3. entering the metadata manually.
      4. -
    - -

    There are instructions and FAQs on how to send JSON using our API on our API documentation page.

    - -

    There are instructions on how to prepare and upload your XML file on our XML documentation page.

    - -

    Are you receiving an error about one of your ISSNs you haven't seen before? We recently changed the rules for uploading article metadata. We now require that the correct ISSN is in the correct corresponding tag, i.e. the Print ISSN must be in the issn tag and the Electronic ISSN in the eissn tag. See our sample XML file for more information.

    - -

    Failed XML uploads explained

    - -

    This section explains the error messages you may see when uploading article XML. Use the message in the 'Notes' column of your History of uploads table to correct your XML.

    - -

    If you cannot get the XML to upload, please submit a bug report or contact us with the following details:

    - -
      -
    • The time of the file upload
      • -
    • Whether the error happened once or repeatedly
      • -
    • Whether you are uploading DOAJ or Crossref XML
      • -
    • Whether you are uploading a file or uploading from a link
      • -
    • The exact error message as it appears in the “Notes” column, including the detail under the 'show error details' link
      • -
    • A screenshot of the error message
      • -
    • The DOAJ account ID that you are logged in with
      • -
    • The file(s) you had problems with
      • -
    +

    If you cannot get the XML to upload, please refer to our detailed Metadata Help page, which contains more information, including how to ask for help.

    From f84b12267d4349bea4a239e0f0cb299699dbe9bc Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 17 Nov 2023 11:50:10 +0000 Subject: [PATCH 124/184] restrict access to the validation tab to only users who have been given it --- .../testbook/publisher_csv/validate_csv.yml | 19 ++++++++++++++++++ doajtest/testdrive/publisher_csv_upload.py | 2 +- portality/constants.py | 2 ++ portality/settings.py | 3 ++- portality/templates/publisher/nav.html | 20 +++++++++---------- portality/view/publisher.py | 7 ++++++- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/doajtest/testbook/publisher_csv/validate_csv.yml b/doajtest/testbook/publisher_csv/validate_csv.yml index 98e01108b9..3f75ea66e2 100644 --- a/doajtest/testbook/publisher_csv/validate_csv.yml +++ b/doajtest/testbook/publisher_csv/validate_csv.yml @@ -77,3 +77,22 @@ tests: - step: Click 'Validate' results: - You are shown a validation message that says validation is successful + - step: If you are finished testing, you can teardown the testdrive using the link provided. If you are going to continue + with the other tests in this suite you can leave the testdrive in place until the very end + +- title: Access to the feature + context: + role: Publisher + testdrive: publisher_csv_upload + setup: + - If you have not done so already, use the publisher_csv_upload testdrive to setup for this test at /testdrive/publisher_csv_upload + steps: + - step: Log in as the publisher account specified by the testdrive result + results: + - The "Validate your CSV" navigation tab is visible + - step: As an administrator, edit the user's account and remove the "journal_csv" role + - step: As the publisher again, reload the publisher home page + results: + - The "Validate your CSV" tab is no longer visible + + diff --git a/doajtest/testdrive/publisher_csv_upload.py b/doajtest/testdrive/publisher_csv_upload.py index 01b680ed8a..e4b59e3aa3 100644 --- a/doajtest/testdrive/publisher_csv_upload.py +++ b/doajtest/testdrive/publisher_csv_upload.py @@ -12,7 +12,7 @@ class PublisherCsvUpload(TestDrive): def setup(self) -> dict: un = self.create_random_str() pw = self.create_random_str() - acc = models.Account.make_account(un + "@example.com", un, "Publisher " + un, [constants.ROLE_PUBLISHER]) + acc = models.Account.make_account(un + "@example.com", un, "Publisher " + un, [constants.ROLE_PUBLISHER, constants.ROLE_PUBLISHER_JOURNAL_CSV]) acc.set_password(pw) acc.save() diff --git a/portality/constants.py b/portality/constants.py index 5fec2fff90..3e5a43790f 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -81,6 +81,8 @@ ROLE_ASSOCIATE_EDITOR = 'associate_editor' ROLE_PUBLIC_DATA_DUMP = "public_data_dump" ROLE_PUBLISHER = "publisher" +ROLE_PUBLISHER_JOURNAL_CSV = "journal_csv" +ROLE_PUBLISHER_PRESERVATION = "preservation" CRON_NEVER = {"month": "2", "day": "31", "day_of_week": "*", "hour": "*", "minute": "*"} diff --git a/portality/settings.py b/portality/settings.py index 92284bd2be..72322c862a 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -296,7 +296,8 @@ "api", "ultra_bulk_delete", "preservation", - constants.ROLE_PUBLIC_DATA_DUMP + constants.ROLE_PUBLIC_DATA_DUMP, + constants.ROLE_PUBLISHER_JOURNAL_CSV ] ROLE_MAP = { diff --git a/portality/templates/publisher/nav.html b/portality/templates/publisher/nav.html index 76c277fa5f..989afa6bdf 100644 --- a/portality/templates/publisher/nav.html +++ b/portality/templates/publisher/nav.html @@ -8,23 +8,23 @@ {% set csv = url_for('publisher.journal_csv') %} {% set tabs = [ - (index, "My drafts", 0), - (journals, "My journals", 1), - (urs, "My update requests", 2), - (xml, "Upload article XML", 3), - (metadata, "Enter article metadata", 4), - (preservation, "Upload preservation file", 5), - (csv, "Validate your Journal CSV", 6), - (help, "Help", 7), + (index, "My drafts", 0, constants.ROLE_PUBLISHER), + (journals, "My journals", 1, constants.ROLE_PUBLISHER), + (urs, "My update requests", 2, constants.ROLE_PUBLISHER), + (xml, "Upload article XML", 3, constants.ROLE_PUBLISHER), + (metadata, "Enter article metadata", 4, constants.ROLE_PUBLISHER), + (preservation, "Upload preservation file", 5, constants.ROLE_PUBLISHER_PRESERVATION), + (csv, "Validate your Journal CSV", 6, constants.ROLE_PUBLISHER_JOURNAL_CSV), + (help, "Help", 7, constants.ROLE_PUBLISHER), ] %}