From 648f084c1e2b919255c2988c432ffef385bc3b4a Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 28 Jan 2022 14:50:26 +0000 Subject: [PATCH 001/466] add first editor todo list category --- portality/bll/services/todo.py | 36 +++++++++++++++++++++++- portality/constants.py | 1 + portality/templates/includes/header.html | 2 +- portality/view/dashboard.py | 2 +- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index f0d5feb10c..2be86d6972 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -13,6 +13,7 @@ def top_todo(self, account, size=25): Returns the top number of todo items for a given user :param account: + :param size: :return: """ # first validate the incoming arguments to ensure that we've got the right thing @@ -29,12 +30,17 @@ def top_todo(self, account, size=25): queries.append(TodoRules.maned_completed(size)) queries.append(TodoRules.maned_assign_pending(size)) + if account.has_role("editor"): + groups = [g for g in models.EditorGroup.groups_by_editor(account.id)] + if len(groups) > 0: + queries.append(TodoRules.editor_stalled(groups, size)) + todos = [] for aid, q, boost in queries: applications = models.Application.object_query(q=q.query()) for ap in applications: todos.append({ - "date": ap.last_manual_update_timestamp, + "date": ap.last_manual_update_timestamp, # FIXME: this is wrong, we need to put a fix in the maned branch "action_id" : [aid], "title" : ap.bibjson().title, "object_id" : ap.id, @@ -138,6 +144,25 @@ def maned_assign_pending(cls, size): ) return constants.TODO_MANED_ASSIGN_PENDING, assign_pending, False + @classmethod + def editor_stalled(cls, groups, size): + stalled = TodoQuery( + musts=[ + TodoQuery.lmu_older_than(6), + TodoQuery.editor_groups(groups) + ], + must_nots=[ + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_READY + ]) + ], + sort="last_manual_update", + size=size + ) + return constants.TODO_EDITOR_STALLED, stalled, False + class TodoQuery(object): """ @@ -204,3 +229,12 @@ def exists(cls, field): "field" : field } } + + @classmethod + def editor_groups(cls, groups): + gids = [g.name for g in groups] + return { + "terms" : { + "admin.editor_group.exact" : gids + } + } diff --git a/portality/constants.py b/portality/constants.py index c415394429..f34d3b065d 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -43,3 +43,4 @@ TODO_MANED_READY = "todo_maned_ready" TODO_MANED_COMPLETED = "todo_maned_completed" TODO_MANED_ASSIGN_PENDING = "todo_maned_assign_pending" +TODO_EDITOR_STALLED = "todo_editor_stalled" \ No newline at end of file diff --git a/portality/templates/includes/header.html b/portality/templates/includes/header.html index 9106c8162b..9815d93a2d 100644 --- a/portality/templates/includes/header.html +++ b/portality/templates/includes/header.html @@ -56,7 +56,7 @@

Secondary actions

#} + {% if current_user.has_role("editor") or current_user.has_role("associate_editor") %}
  • - - Back to dashboard + + Editor dashboard + +
  • + {% endif %} +
  • + + Admin dashboard
  • From 9e905c1db1a3cd5fb7887847ce47a52df80246c1 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 21 Sep 2022 21:11:06 +0100 Subject: [PATCH 016/466] minor tweaks required to support associate dashboard --- portality/static/js/edges/associate.applications.edge.js | 1 + 1 file changed, 1 insertion(+) diff --git a/portality/static/js/edges/associate.applications.edge.js b/portality/static/js/edges/associate.applications.edge.js index 4ed785213e..f4abcaf85e 100644 --- a/portality/static/js/edges/associate.applications.edge.js +++ b/portality/static/js/edges/associate.applications.edge.js @@ -303,6 +303,7 @@ $.extend(true, doaj, { category: "selected-filters", fieldDisplays: { 'admin.application_status.exact': 'Application Status', + 'index.application_type.exact': "Record type", 'index.classification.exact' : 'Classification', 'index.language.exact' : 'Journal language', 'index.country.exact' : 'Country of publisher', From c61ca1f07596221e4a73b7ac7ee5a3fe994c7047 Mon Sep 17 00:00:00 2001 From: Sophy Date: Fri, 7 Oct 2022 13:29:16 +0100 Subject: [PATCH 017/466] Add username and role to ed and associate ed See doaj/doajPM#3218 --- .../templates/layouts/dashboard_base.html | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index fcada4b5cf..bef8b6f158 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -74,18 +74,31 @@

    DOAJ Dashboard

    {% block main_panel %}
    - {% if request.path == "/dashboard/" %}
    -

    {{ current_user.id }} (Managing Editor)

    {# TODO: insert role of user here #} - {% endif %} +

    + + + + {{ current_user.id }} + {% if current_user.has_role("admin") %} + (Managing Editor) + {% elif current_user.has_role("editor") %} + (Editor) + {% elif current_user.has_role("associate_editor") %} + (Associate Editor) + {% endif %} + + +

    {% block page_title %} - Hi{% if current_user.name %}, {{ current_user.name }}{% endif %}! + Hi, {% if current_user.name %}{{ current_user.name }}{% else %} + {{ current_user.id }}{% endif %}! {% endblock %}

    - {% if request.path == "/dashboard/" %} + {% if current_user.has_role("admin") %} + {% endif %}
    - {% endif %}
    {% include "includes/_flash_notification.html" %} {% block content %}{% endblock %} From 25179842892eda90add68e1984840856da76b58e Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 12 Oct 2022 16:38:34 +0100 Subject: [PATCH 018/466] add editor_of wrapper change authorisation check in doaj service create editor dashboard template make sure all necessary attributes are passed to html --- portality/app.py | 14 ++++++++ portality/templates/editor/dashboard.html | 35 +++++++++++++++++++ .../templates/layouts/dashboard_base.html | 1 + portality/view/doajservices.py | 3 +- portality/view/editor.py | 4 +-- 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/portality/app.py b/portality/app.py index 919d4f90ef..268abd0cc9 100644 --- a/portality/app.py +++ b/portality/app.py @@ -302,6 +302,20 @@ def maned_of(): return dict(maned_of=maned_of) +@app.context_processor +def editor_of_wrapper(): + def editor_of(): + # ~~-> EditorGroup:Model ~~ + egs = [] + assignments = {} + if current_user.has_role("editor"): + egs = models.EditorGroup.groups_by_editor(current_user.id) + if len(egs) > 0: + assignments = models.Application.assignment_to_editor_groups(egs) + return egs, assignments + return dict(editor_of_fun=editor_of) + + # ~~-> Account:Model~~ # ~~-> AuthNZ:Feature~~ @app.before_request diff --git a/portality/templates/editor/dashboard.html b/portality/templates/editor/dashboard.html index 01cee1c9c4..1e73aa5fc2 100644 --- a/portality/templates/editor/dashboard.html +++ b/portality/templates/editor/dashboard.html @@ -2,4 +2,39 @@ {% block editor_content %} {% include "dashboard/_todo.html" %} +
    + {# ~~->$GroupStatus:Feature~~ #} +

    Activity

    +
    + + + {# TODO: there’s a bit of a11y work to be done here; we need to indicate which tabs are hidden and which + aren’t using ARIA attributes. #} + {# TODO: the first tab content needs to be shown by default, without a "click to see" message. #} +
    +
    +
    +
    +
    {% endblock %} + +{% block extra_js_bottom %} + + +{% endblock %} \ No newline at end of file diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index bef8b6f158..dc69abc8ab 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -4,6 +4,7 @@ {# we're potentially going need this in a few places in inherited files, so lets just put it here #} {# ~~-> EditorGroup:Model ~~ #} {% set managed_groups, maned_assignments = maned_of() %} +{% set editor_of_groups, editor_of_assignments = editor_of_fun() %} {% block base_content %} diff --git a/portality/view/doajservices.py b/portality/view/doajservices.py index 3e0076ba7b..6efbfcb99f 100644 --- a/portality/view/doajservices.py +++ b/portality/view/doajservices.py @@ -1,5 +1,6 @@ import json, urllib.request, urllib.parse, urllib.error, requests +import models from flask import Blueprint, make_response, request, abort, render_template from flask_login import current_user, login_required @@ -107,7 +108,7 @@ def group_status(group_id): :param group_id: :return: """ - if not current_user.has_role("admin"): + if (not (current_user.has_role("editor") and models.EditorGroup.pull(group_id).editor == current_user.id)) and (not current_user.has_role("admin")): abort(404) svc = DOAJ.todoService() stats = svc.group_stats(group_id) diff --git a/portality/view/editor.py b/portality/view/editor.py index 3a66cdb56c..44dfe8272e 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -29,9 +29,9 @@ def index(): # ~~-> Todo:Service~~ svc = DOAJ.todoService() todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE")) - + editor_of = models.EditorGroup.groups_by_editor(current_user.id) # ~~-> Dashboard:Page~~ - return render_template('editor/dashboard.html', todos=todos) + return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of) # build an editor's page where things can be done @blueprint.route('/group-info') From be0b8453aa0efd26ae57ee50f9e62276b1c74916 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Thu, 13 Oct 2022 11:46:19 +0100 Subject: [PATCH 019/466] make sure the Activity is not shown for ass eds --- portality/templates/editor/dashboard.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/portality/templates/editor/dashboard.html b/portality/templates/editor/dashboard.html index 1e73aa5fc2..c60d5646c4 100644 --- a/portality/templates/editor/dashboard.html +++ b/portality/templates/editor/dashboard.html @@ -4,6 +4,7 @@ {% include "dashboard/_todo.html" %}
    {# ~~->$GroupStatus:Feature~~ #} + {% if editor_of | length != 0 %}

    Activity

    + {% endif %} {# TODO: there’s a bit of a11y work to be done here; we need to indicate which tabs are hidden and which aren’t using ARIA attributes. #} From 060acde961898981e95e14266849314762c59df8 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Thu, 13 Oct 2022 13:09:34 +0100 Subject: [PATCH 020/466] do not show admin dashboard link to non-admins --- .../templates/layouts/dashboard_base.html | 186 +++++++++--------- 1 file changed, 94 insertions(+), 92 deletions(-) diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index dc69abc8ab..c8d63ac254 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -15,14 +15,14 @@ {% endif %}
    -

    DOAJ Dashboard

    +

    DOAJ Dashboard

    - + {% if current_user.has_role("editor") or current_user.has_role("associate_editor") %} +
  • + + Editor dashboard + +
  • + {% endif %} + {% if current_user.has_role("admin") %} +
  • + + Admin dashboard + +
  • + {% endif %} +
  • + + DOAJ home + +
  • + + {% block nav %}{% endblock %} - {% block extra_header %}{% endblock %} + {% block extra_header %}{% endblock %}
    {% block main_panel %} -
    -
    -
    -
    -

    - - - - {{ current_user.id }} - {% if current_user.has_role("admin") %} - (Managing Editor) - {% elif current_user.has_role("editor") %} - (Editor) - {% elif current_user.has_role("associate_editor") %} - (Associate Editor) - {% endif %} - - -

    -

    - {% block page_title %} - Hi, {% if current_user.name %}{{ current_user.name }}{% else %} - {{ current_user.id }}{% endif %}! - {% endblock %} -

    -
    - {% if current_user.has_role("admin") %} - + {% endif %} +
    +
    + {% include "includes/_flash_notification.html" %} + {% block content %}{% endblock %} -

    - - - - Log out - - - - Settings - - -

    -
    - {% include "includes/_back-to-top.html" %} +

    + + + + Log out + + + + Settings + + +

    +
    + {% include "includes/_back-to-top.html" %} {% endblock %} {% include '_js_includes.html' %} From 57c543d81ae81ad9495f9dd6d873f3f8ef6be45a Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Thu, 13 Oct 2022 18:21:09 +0100 Subject: [PATCH 021/466] add functional tests in testbook --- .../dashboards/editorial_group_status.yml | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 doajtest/testbook/dashboards/editorial_group_status.yml diff --git a/doajtest/testbook/dashboards/editorial_group_status.yml b/doajtest/testbook/dashboards/editorial_group_status.yml new file mode 100644 index 0000000000..895e2b27bb --- /dev/null +++ b/doajtest/testbook/dashboards/editorial_group_status.yml @@ -0,0 +1,89 @@ +suite: Editorial Group Status +testset: Editors + +tests: + - title: Record Counts + context: + role: editor + setup: + - You must set up at least 2 editorial groups with multiple members and a good selection of applications assigned to a number of associate editors. You should set yourself as the editor of those groups. + steps: + - step: Go to your Dashboard page + path: /dashboard + - step: Scroll to the bottom of the dashbaord page, after your TODO list items. + results: + - The editorial groups you are editor of are listed under the heading Activity + - The first group is already selected and the data shown + - step: Click on one of the other group names + results: + - The editorial group's statistics are presented on the screen + - You can see the total number of applications assigned to the group + - You can see the number of applications assigned to each associated editor + - You can see how many applications have not been assigned + - You can see the breakdown of the statuses of all the applications + - step: Click on the application count next to the group's name (you may want to right click to keep the dashboard tab open during this test) + results: + - You are taken to the application search which shows the open applications assigned to this group. + - The number of search results is the same as shown on the dashboard + - step: Go back to the dashboard page + - step: Click on an associated editor's name + results: + - A mail window opens in your mail client + - step: Click on the application count next to an editor's name + results: + - You are taken to the application search which shows the open applications assigned to the user + - The number of search results is the same as shown on the dashboard + - step: Go back to the dashboard page + - step: Click on the application count next to the "unassigned" label + results: + - You are taken to the application search which shows the open applications with no assigned editor + - The number of search results is the same as shown on the dashboard + - step: Go back to the dashboard page + - step: Click on one of the statuses under "Applications By Status" + results: + - You are taken to the application search which shows the open applications in that status + - The number of search results is the same as shown on the dashboard + + - title: Updating Associated Editors + context: + role: editor + setup: + - You must set up at least 2 editorial groups with multiple members and a good selection of applications assigned to a number of associate editors. You should set yourself as the editor of those groups. + steps: + - step: Go to your ManEd Dashboard page + path: /dashboard + - step: Select an editorial group to see its status information + - step: Take a note of the number of applications assigned to one of your associated editors + - step: Click on the count of applications next to the "unassigned" tag + - step: Click "Review Application" on the application in the search interface + - step: Assign the application to the editor selected above + - step: Go back to the dashboard page (you may need to refresh it if you kept the page open) + results: + - The editor selected now has one more application assigned to them + - The number of unassigned applications has reduced by one + - step: Click on the count of applications next to the selected editor + results: + - The application search is shown + - The application you assigned to the editor is listed in the search + + - title: Changing status + context: + role: editor + setup: + - You must set up at least 2 editorial groups with multiple members and a good selection of applications assigned to a number of associate editors. You should set yourself as the editor of those groups. + steps: + - step: Go to your ManEd Dashboard page + path: /dashboard + - step: Select an editorial group to see its status information + - step: Take a note of the number of applications assigned to a specific status (e.g. in progress) + - step: Click on the link to applications in that status + - step: Click "Review Application" on the application in the search interface + - step: Change the status to something different + - step: Go back to the dashboard page (you may need to refresh it if you kept the page open) + results: + - The status you selected now has one more application in that state + - The previous status has one fewer application in that state + - step: Click on the new status link + results: + - The application search is shown + - The application you put into this status is visible From 8fc3fe760279b8dc4993db4f42c3e4e9b9a8e505 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 17 Oct 2022 13:50:04 +0100 Subject: [PATCH 022/466] add contextualized url to badger add information for no groups --- portality/static/js/dashboard.js | 11 ++++++----- portality/templates/dashboard/index.html | 8 +++++++- portality/templates/editor/dashboard.html | 8 +++++++- portality/view/editor.py | 3 ++- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/portality/static/js/dashboard.js b/portality/static/js/dashboard.js index 0309e2dc5b..0bca817273 100644 --- a/portality/static/js/dashboard.js +++ b/portality/static/js/dashboard.js @@ -13,7 +13,8 @@ doaj.dashboard = { ] }; -doaj.dashboard.init = function() { +doaj.dashboard.init = function(context) { + doaj.dashboard.context = context; $(".js-group-tab").on("click", doaj.dashboard.groupTabClick); // trigger a click on the first one, so there is something for the user to look at @@ -91,7 +92,7 @@ doaj.dashboard.renderGroupInfo = function(data) { } editorListFrag += `
  • ${ed}${isEd} - ${appCount} applications + ${appCount} applications
  • `; } @@ -112,7 +113,7 @@ doaj.dashboard.renderGroupInfo = function(data) { // ]}) editorListFrag += `
  • Unassigned - ${data.unassigned.applications} applications + ${data.unassigned.applications} applications
  • `; let appStatusProgressBar = ""; @@ -131,7 +132,7 @@ doaj.dashboard.renderGroupInfo = function(data) { "sort": [{"admin.date_applied": {"order": "asc"}}] }) appStatusProgressBar += `
  • - + ${data.by_status[status].applications}
  • `; } @@ -154,7 +155,7 @@ doaj.dashboard.renderGroupInfo = function(data) { let frag = `

    ${data.editor_group.name}’s open applications - ${data.total.applications} applications + ${data.total.applications} applications

    diff --git a/portality/templates/dashboard/index.html b/portality/templates/dashboard/index.html index ff01f1aa8c..1f60067277 100644 --- a/portality/templates/dashboard/index.html +++ b/portality/templates/dashboard/index.html @@ -14,11 +14,17 @@

    Activity

      {# managed_groups is inherited from the dashboard_base template #} {# ~~^-> EditorGroup:Model ~~ #} + {% if managed_groups|length == 0 %} +

      + You do not manage any groups now. +

      + {% else %} {% for eg in managed_groups %}
    • {{ eg.name }} ({{ maned_assignments[eg.name] }} applications)
    • {% endfor %} + {% endif %}
    @@ -36,7 +42,7 @@

    Activity

    {% endblock %} diff --git a/portality/templates/editor/dashboard.html b/portality/templates/editor/dashboard.html index c60d5646c4..5d6bf43566 100644 --- a/portality/templates/editor/dashboard.html +++ b/portality/templates/editor/dashboard.html @@ -13,11 +13,17 @@

    Activity

    {% endif %} @@ -36,7 +42,7 @@

    Activity

    {% endblock %} \ No newline at end of file diff --git a/portality/view/editor.py b/portality/view/editor.py index 44dfe8272e..e40a4f4599 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -30,8 +30,9 @@ def index(): svc = DOAJ.todoService() todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE")) editor_of = models.EditorGroup.groups_by_editor(current_user.id) + context = {"role": current_user.role} # ~~-> Dashboard:Page~~ - return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of) + return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of, context=context) # build an editor's page where things can be done @blueprint.route('/group-info') From 239097d6c28c05b67394193c208ac5c596a28379 Mon Sep 17 00:00:00 2001 From: Sophy Date: Mon, 31 Oct 2022 16:08:21 +0000 Subject: [PATCH 023/466] Add group info in header for associate editors and editors --- .../templates/layouts/dashboard_base.html | 86 ++++++++++++++----- portality/view/editor.py | 4 +- 2 files changed, 68 insertions(+), 22 deletions(-) diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index bef8b6f158..763363cb05 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -98,31 +98,75 @@

    {% endblock %}

    - {% if current_user.has_role("admin") %} - {% endif %}
    {% include "includes/_flash_notification.html" %} diff --git a/portality/view/editor.py b/portality/view/editor.py index 3a66cdb56c..5516065845 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -29,9 +29,11 @@ def index(): # ~~-> Todo:Service~~ svc = DOAJ.todoService() todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE")) + editor_of = models.EditorGroup.groups_by_editor(current_user.id) + associate_of = models.EditorGroup.groups_by_associate(current_user.id) # ~~-> Dashboard:Page~~ - return render_template('editor/dashboard.html', todos=todos) + return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of, associate_of=associate_of, managing_editor=app.config.get("MANAGING_EDITOR_EMAIL")) # build an editor's page where things can be done @blueprint.route('/group-info') From 33347c46dbd03e7860c89da89154d2f9146f0b87 Mon Sep 17 00:00:00 2001 From: Sophy Date: Mon, 31 Oct 2022 16:09:44 +0000 Subject: [PATCH 024/466] Visually hide label --- portality/templates/layouts/dashboard_base.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index 763363cb05..fe2ef59a3c 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -121,7 +121,7 @@

    {% endif %} {% elif current_user.has_role("editor") %} -
    +
    Editor groups:
    @@ -142,7 +142,7 @@

    {% elif current_user.has_role("associate_editor") %} -
    +
    Editor groups:
    From 7a860718eb29bb712f1bb32c821197380c964b3b Mon Sep 17 00:00:00 2001 From: Sophy Date: Mon, 31 Oct 2022 16:45:16 +0000 Subject: [PATCH 025/466] =?UTF-8?q?Associate=20Eds=20don=E2=80=99t=20need?= =?UTF-8?q?=20links=20to=20ed=20groups=E2=80=99=20apps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- portality/templates/layouts/dashboard_base.html | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index fe2ef59a3c..651a3561c8 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -149,12 +149,7 @@

      {% for group in associate_of %}
    • - {% set app_source = search_query_source(term=[ - {"admin.editor_group.exact" : group.name}, - {"index.application_type.exact" : "new application"} - ], sort=[{"admin.date_applied": {"order": "asc"}}] - ) %} - {{ group.name }} + {{ group.name }} {% set ed = group.get_editor_account() %} {{ ed.id }} (Ed.) {% set maned = group.get_maned_account() %} From 013229fadc7d61b39bafe294376009459c17f0f0 Mon Sep 17 00:00:00 2001 From: Sophy Date: Mon, 31 Oct 2022 16:49:23 +0000 Subject: [PATCH 026/466] Drop group info page (all moved to dashboard landing page Update icons in sidenav --- portality/templates/editor/nav.html | 16 +++++++--------- portality/view/editor.py | 8 -------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/portality/templates/editor/nav.html b/portality/templates/editor/nav.html index 44a051847e..30a62bf69a 100644 --- a/portality/templates/editor/nav.html +++ b/portality/templates/editor/nav.html @@ -1,17 +1,15 @@ {% set index = url_for("editor.index") %} -{% set groupinfo = url_for("editor.groupinfo") %} -{% set group_journals = url_for('editor.group_journals') %} {% set group_apps = url_for('editor.group_suggestions') %} -{% set ass_journals = url_for('editor.associate_journals') %} +{% set group_journals = url_for('editor.group_journals') %} {% set ass_apps = url_for('editor.associate_suggestions') %} +{% set ass_journals = url_for('editor.associate_journals') %} {% set tabs = [ (index, "Dashboard", None, "list"), - (groupinfo, "Group Info", None, "users"), - (group_journals, "Your group’s journals", "list_group_journals", "layers"), - (group_apps, "Your group’s applications", "list_group_suggestions", "database"), - (ass_journals, "Journals assigned to you", None, "book-open"), - (ass_apps, "Applications assigned to you", None, "file-text") + (group_apps, "Your group’s applications", "list_group_suggestions", "users"), + (group_journals, "Your group’s journals", "list_group_journals", "book"), + (ass_apps, "Applications assigned to you", None, "file-text"), + (ass_journals, "Journals assigned to you", None, "book-open") ] %} @@ -29,4 +27,4 @@ {% endif %} {% endfor %} - \ No newline at end of file + diff --git a/portality/view/editor.py b/portality/view/editor.py index 5516065845..a17c9a78dd 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -36,14 +36,6 @@ def index(): return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of, associate_of=associate_of, managing_editor=app.config.get("MANAGING_EDITOR_EMAIL")) # build an editor's page where things can be done -@blueprint.route('/group-info') -@login_required -@ssl_required -def groupinfo(): - editor_of = models.EditorGroup.groups_by_editor(current_user.id) - associate_of = models.EditorGroup.groups_by_associate(current_user.id) - return render_template('editor/index.html', editor_of=editor_of, associate_of=associate_of, managing_editor=app.config.get("MANAGING_EDITOR_EMAIL")) - @blueprint.route('/group_journals') @login_required @ssl_required From 4683dcc72f43df442b00583a7428d9b32448d3bf Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 8 Nov 2022 14:16:44 +0000 Subject: [PATCH 027/466] remove editor_of_fun --- portality/app.py | 2 +- portality/templates/layouts/dashboard_base.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/app.py b/portality/app.py index 268abd0cc9..9b10675233 100644 --- a/portality/app.py +++ b/portality/app.py @@ -313,7 +313,7 @@ def editor_of(): if len(egs) > 0: assignments = models.Application.assignment_to_editor_groups(egs) return egs, assignments - return dict(editor_of_fun=editor_of) + return dict(editor_of=editor_of) # ~~-> Account:Model~~ diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index c8d63ac254..56f1b00767 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -4,7 +4,7 @@ {# we're potentially going need this in a few places in inherited files, so lets just put it here #} {# ~~-> EditorGroup:Model ~~ #} {% set managed_groups, maned_assignments = maned_of() %} -{% set editor_of_groups, editor_of_assignments = editor_of_fun() %} +{% set editor_of_groups, editor_of_assignments = editor_of() %} {% block base_content %} From ad338cb8988ed1f9fc643d69844b8167576728e5 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 8 Nov 2022 17:27:42 +0000 Subject: [PATCH 028/466] fix the iteration in the editors dashboard --- portality/templates/editor/dashboard.html | 72 +++++++++++------------ portality/view/editor.py | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/portality/templates/editor/dashboard.html b/portality/templates/editor/dashboard.html index 5d6bf43566..905cf97f6f 100644 --- a/portality/templates/editor/dashboard.html +++ b/portality/templates/editor/dashboard.html @@ -1,48 +1,48 @@ {% extends "editor/editor_base.html" %} {% block editor_content %} - {% include "dashboard/_todo.html" %} + {% include "dashboard/_todo.html" %}
      - {# ~~->$GroupStatus:Feature~~ #} - {% if editor_of | length != 0 %} -

      Activity

      -
      -
    - - {% endif %} - {# TODO: there’s a bit of a11y work to be done here; we need to indicate which tabs are hidden and which + {# TODO: there’s a bit of a11y work to be done here; we need to indicate which tabs are hidden and which aren’t using ARIA attributes. #} - {# TODO: the first tab content needs to be shown by default, without a "click to see" message. #} -
    -
    -
    - - + {# TODO: the first tab content needs to be shown by default, without a "click to see" message. #} +
    +
    +
    + + {% endblock %} {% block extra_js_bottom %} - - + + {% endblock %} \ No newline at end of file diff --git a/portality/view/editor.py b/portality/view/editor.py index e40a4f4599..414a43b67c 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -32,7 +32,7 @@ def index(): editor_of = models.EditorGroup.groups_by_editor(current_user.id) context = {"role": current_user.role} # ~~-> Dashboard:Page~~ - return render_template('editor/dashboard.html', todos=todos, editor_of=editor_of, context=context) + return render_template('editor/dashboard.html', todos=todos) # build an editor's page where things can be done @blueprint.route('/group-info') From 66cf875ad7f3db2ef5bf820fbd752b5b60dfc409 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 14 Nov 2022 21:45:28 +0000 Subject: [PATCH 029/466] add test drive for editor and associate editor dashboard displays, and the associated fixes and improvements to the rules and other features to support the tests --- doajtest/testdrive/__init__.py | 0 doajtest/testdrive/factory.py | 25 +++++ doajtest/testdrive/todo_associate.py | 93 ++++++++++++++++++ doajtest/testdrive/todo_editor.py | 97 +++++++++++++++++++ doajtest/testdrive/todo_editor_associate.py | 40 ++++++++ doajtest/unit/test_bll_todo_top_todo.py | 17 ++-- portality/app.py | 19 +++- portality/bll/services/todo.py | 61 ++++++++---- portality/constants.py | 1 + portality/templates/dashboard/_todo.html | 10 +- .../templates/layouts/dashboard_base.html | 79 ++++++++------- portality/view/editor.py | 6 +- portality/view/testdrive.py | 21 ++++ 13 files changed, 397 insertions(+), 72 deletions(-) create mode 100644 doajtest/testdrive/__init__.py create mode 100644 doajtest/testdrive/factory.py create mode 100644 doajtest/testdrive/todo_associate.py create mode 100644 doajtest/testdrive/todo_editor.py create mode 100644 doajtest/testdrive/todo_editor_associate.py create mode 100644 portality/view/testdrive.py diff --git a/doajtest/testdrive/__init__.py b/doajtest/testdrive/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doajtest/testdrive/factory.py b/doajtest/testdrive/factory.py new file mode 100644 index 0000000000..cd94e90ffb --- /dev/null +++ b/doajtest/testdrive/factory.py @@ -0,0 +1,25 @@ +from portality.lib import plugin +import random +import string + + +class TestDrive(): + def create_random_str(self, n_char=10): + s = string.ascii_letters + string.digits + return ''.join(random.choices(s, k=n_char)) + + def setup(self) -> dict: + return {"status": "not implemented"} + + def teardown(self, setup_params) -> dict: + return {"status": "not implemented"} + + +class TestFactory(): + @classmethod + def get(cls, test_id): + modname = test_id + classname = test_id.replace("_", " ").title().replace(" ", "") + classpath = "doajtest.testdrive." + modname + "." + classname + klazz = plugin.load_class(classpath) + return klazz() \ No newline at end of file diff --git a/doajtest/testdrive/todo_associate.py b/doajtest/testdrive/todo_associate.py new file mode 100644 index 0000000000..c85a88669d --- /dev/null +++ b/doajtest/testdrive/todo_associate.py @@ -0,0 +1,93 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from doajtest.fixtures.v2.applications import ApplicationFixtureFactory +from portality.lib import dates +from portality import models +from datetime import datetime + + +class TodoAssociate(TestDrive): + + def setup(self) -> dict: + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "TodoAssociate " + un, [constants.ROLE_ASSOCIATE_EDITOR]) + acc.set_password(pw) + acc.save() + + gn = "TodoAssociate Group " + un + eg = models.EditorGroup(**{ + "name": gn + }) + eg.add_associate(acc.id) + eg.save() + + apps = build_applications(un) + + return { + "account": { + "username": acc.id, + "password": pw + }, + "editor_group": { + "id": eg.id, + "name": eg.name + }, + "applications": apps + } + + +def build_applications(un): + w = 7 * 24 * 60 * 60 + + apps = {} + + app = build_application(un + " Stalled Application", 3 * w, 3 * w, + constants.APPLICATION_STATUS_IN_PROGRESS, editor=un) + app.save() + apps["stalled"] = [{ + "id": app.id, + "title": un + " Stalled Application" + }] + + app = build_application(un + " Old Application", 6 * w, 6 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + editor=un) + app.save() + apps["old"] = [{ + "id": app.id, + "title": un + " Old Application" + }] + + app = build_application(un + " Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, + editor=un) + app.save() + apps["pending"] = [{ + "id": app.id, + "title": un + " Pending Application" + }] + + app = build_application(un + " All Other Applications", 2 * w, 2 * w, + constants.APPLICATION_STATUS_IN_PROGRESS, editor=un) + app.save() + apps["all"] = [{ + "id": app.id, + "title": un + " All Other Applications" + }] + + return apps + + +def build_application(title, lmu_diff, cd_diff, status, editor=None): + source = ApplicationFixtureFactory.make_application_source() + ap = models.Application(**source) + ap.bibjson().title = title + ap.set_id(ap.makeid()) + ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) + ap.set_created(dates.before(datetime.utcnow(), cd_diff)) + ap.set_application_status(status) + + if editor is not None: + ap.set_editor(editor) + + ap.save() + return ap diff --git a/doajtest/testdrive/todo_editor.py b/doajtest/testdrive/todo_editor.py new file mode 100644 index 0000000000..aa81164018 --- /dev/null +++ b/doajtest/testdrive/todo_editor.py @@ -0,0 +1,97 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from doajtest.fixtures.v2.applications import ApplicationFixtureFactory +from portality.lib import dates +from portality import models +from datetime import datetime + + +class TodoEditor(TestDrive): + + def setup(self) -> dict: + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "TodoEditor " + un, ["editor"]) + acc.set_password(pw) + acc.save() + + gn = "TodoEditor Group " + un + eg = models.EditorGroup(**{ + "name": gn + }) + eg.set_editor(acc.id) + eg.save() + + apps = build_applications(un, eg) + + return { + "account": { + "username": acc.id, + "password": pw + }, + "editor_group": { + "id": eg.id, + "name": eg.name + }, + "applications": apps + } + + +def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None): + source = ApplicationFixtureFactory.make_application_source() + ap = models.Application(**source) + ap.bibjson().title = title + ap.set_id(ap.makeid()) + ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) + ap.set_created(dates.before(datetime.utcnow(), cd_diff)) + ap.set_application_status(status) + + if editor is not None: + ap.set_editor(editor) + + if editor_group is not None: + ap.set_editor_group(editor_group) + + ap.save() + return ap + + +def build_applications(un, eg): + w = 7 * 24 * 60 * 60 + + apps = {} + + app = build_application(un + " Stalled Application", 6 * w, 12 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + editor_group=eg.name) + app.save() + apps["stalled"] = [{ + "id": app.id, + "title": un + " Stalled Application" + }] + + app = build_application(un + " Old Application", 8 * w, 8 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + editor_group=eg.name) + app.save() + apps["old"] = [{ + "id": app.id, + "title": un + " Old Application" + }] + + app = build_application(un + " Completed Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_COMPLETED, + editor_group=eg.name) + app.save() + apps["completed"] = [{ + "id": app.id, + "title": un + " Completed Application" + }] + + app = build_application(un + " Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, + editor_group=eg.name) + app.remove_editor() + app.save() + apps["pending"] = [{ + "id": app.id, + "title": un + " Pending Application" + }] + + return apps \ No newline at end of file diff --git a/doajtest/testdrive/todo_editor_associate.py b/doajtest/testdrive/todo_editor_associate.py new file mode 100644 index 0000000000..35161f1d5e --- /dev/null +++ b/doajtest/testdrive/todo_editor_associate.py @@ -0,0 +1,40 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from doajtest.testdrive.todo_associate import build_applications as build_associate_applications +from doajtest.testdrive.todo_editor import build_applications as build_editor_applications +from portality import models + + +class TodoEditorAssociate(TestDrive): + + def setup(self) -> dict: + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "TodoEditorAssociate " + un, ["editor", constants.ROLE_ASSOCIATE_EDITOR]) + acc.set_password(pw) + acc.save() + + gn = "TodoEditorAssociate Group " + un + eg = models.EditorGroup(**{ + "name": gn + }) + eg.set_editor(acc.id) + eg.save() + + aapps = build_associate_applications(un) + eapps = build_editor_applications(un, eg) + + return { + "account": { + "username": acc.id, + "password": pw + }, + "editor_group": { + "id": eg.id, + "name": eg.name + }, + "applications": { + "associate": aapps, + "editor": eapps + } + } \ No newline at end of file diff --git a/doajtest/unit/test_bll_todo_top_todo.py b/doajtest/unit/test_bll_todo_top_todo.py index d1e2d66009..3ef3b5b1f5 100644 --- a/doajtest/unit/test_bll_todo_top_todo.py +++ b/doajtest/unit/test_bll_todo_top_todo.py @@ -13,8 +13,7 @@ from portality.bll import DOAJ from portality.bll import exceptions from portality.constants import TODO_ASSOCIATE_PROGRESS_STALLED, TODO_ASSOCIATE_START_PENDING, \ - TODO_ASSOCIATE_FOLLOW_UP_OLD -from portality.lib import dates + TODO_ASSOCIATE_FOLLOW_UP_OLD, TODO_ASSOCIATE_ALL_APPLICATIONS from portality.lib.paths import rel2abs from portality.lib import dates from datetime import datetime @@ -139,14 +138,12 @@ def test_todo__assed_normal(self): # breakpoint() excepted_app_action_dict = { - "maned_stalled__maned_old": {TODO_ASSOCIATE_PROGRESS_STALLED, TODO_ASSOCIATE_START_PENDING, - TODO_ASSOCIATE_FOLLOW_UP_OLD}, - "unstalled": {TODO_ASSOCIATE_START_PENDING}, - "rejected": {TODO_ASSOCIATE_START_PENDING, TODO_ASSOCIATE_PROGRESS_STALLED}, - "maned_old": {TODO_ASSOCIATE_FOLLOW_UP_OLD, TODO_ASSOCIATE_START_PENDING}, - "not_old": {TODO_ASSOCIATE_START_PENDING}, - "accepted": {TODO_ASSOCIATE_START_PENDING, TODO_ASSOCIATE_PROGRESS_STALLED}, - "pending_12": {TODO_ASSOCIATE_PROGRESS_STALLED, TODO_ASSOCIATE_FOLLOW_UP_OLD}, + "maned_stalled__maned_old": {TODO_ASSOCIATE_PROGRESS_STALLED, + TODO_ASSOCIATE_FOLLOW_UP_OLD, TODO_ASSOCIATE_ALL_APPLICATIONS}, + "unstalled": {TODO_ASSOCIATE_ALL_APPLICATIONS}, + "maned_old": {TODO_ASSOCIATE_FOLLOW_UP_OLD, TODO_ASSOCIATE_ALL_APPLICATIONS}, + "not_old": {TODO_ASSOCIATE_ALL_APPLICATIONS}, + "pending_12": {TODO_ASSOCIATE_START_PENDING, TODO_ASSOCIATE_PROGRESS_STALLED, TODO_ASSOCIATE_FOLLOW_UP_OLD, TODO_ASSOCIATE_ALL_APPLICATIONS}, } self.assertEqual(app_action_dict, excepted_app_action_dict) diff --git a/portality/app.py b/portality/app.py index 9b10675233..a03df01447 100644 --- a/portality/app.py +++ b/portality/app.py @@ -46,6 +46,9 @@ from portality.lib.normalise import normalise_doi from portality.view.dashboard import blueprint as dashboard +if app.config.get("DEBUG", False): + from portality.view.testdrive import blueprint as testdrive + app.register_blueprint(account, url_prefix='/account') #~~->Account:Blueprint~~ app.register_blueprint(admin, url_prefix='/admin') #~~-> Admin:Blueprint~~ app.register_blueprint(publisher, url_prefix='/publisher') #~~-> Publisher:Blueprint~~ @@ -68,12 +71,14 @@ app.register_blueprint(apply, url_prefix='/apply') # ~~-> Apply:Blueprint~~ app.register_blueprint(jct, url_prefix="/jct") # ~~-> JCT:Blueprint~~ app.register_blueprint(dashboard, url_prefix="/dashboard") #~~-> Dashboard:Blueprint~~ - app.register_blueprint(oaipmh) # ~~-> OAIPMH:Blueprint~~ app.register_blueprint(openurl) # ~~-> OpenURL:Blueprint~~ app.register_blueprint(atom) # ~~-> Atom:Blueprint~~ app.register_blueprint(doaj) # ~~-> DOAJ:Blueprint~~ +if app.config.get("DEBUG", False): + app.register_blueprint(testdrive, url_prefix="/testdrive") # ~~-> Testdrive:Feature ~~ + # initialise the index - don't put into if __name__ == '__main__' block, # because that does not run if gunicorn is loading the app, as opposed # to the app being run directly by python portality/app.py @@ -315,6 +320,18 @@ def editor_of(): return egs, assignments return dict(editor_of=editor_of) +@app.context_processor +def associate_of_wrapper(): + def associate_of(): + # ~~-> EditorGroup:Model ~~ + egs = [] + assignments = {} + if current_user.has_role("associate_editor"): + egs = models.EditorGroup.groups_by_associate(current_user.id) + if len(egs) > 0: + assignments = models.Application.assignment_to_editor_groups(egs) + return egs, assignments + return dict(associate_of=associate_of) # ~~-> Account:Model~~ # ~~-> AuthNZ:Feature~~ diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index bf25f4ddcb..8c540f8d65 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -77,8 +77,8 @@ def top_todo(self, account, size=25): queries = [] if account.has_role("admin"): maned_of = models.EditorGroup.groups_by_maned(account.id) - queries.append(TodoRules.maned_stalled(size, maned_of)) queries.append(TodoRules.maned_follow_up_old(size, maned_of)) + queries.append(TodoRules.maned_stalled(size, maned_of)) queries.append(TodoRules.maned_ready(size, maned_of)) queries.append(TodoRules.maned_completed(size, maned_of)) queries.append(TodoRules.maned_assign_pending(size, maned_of)) @@ -86,16 +86,17 @@ def top_todo(self, account, size=25): if account.has_role("editor"): groups = [g for g in models.EditorGroup.groups_by_editor(account.id)] if len(groups) > 0: - queries.append(TodoRules.editor_stalled(groups, size)) queries.append(TodoRules.editor_follow_up_old(groups, size)) + queries.append(TodoRules.editor_stalled(groups, size)) queries.append(TodoRules.editor_completed(groups, size)) queries.append(TodoRules.editor_assign_pending(groups, size)) if account.has_role(constants.ROLE_ASSOCIATE_EDITOR): queries.extend([ - TodoRules.associate_stalled(account.id, size), TodoRules.associate_follow_up_old(account.id, size), + TodoRules.associate_stalled(account.id, size), TodoRules.associate_start_pending(account.id, size), + TodoRules.associate_all_applications(account.id, size) ]) todos = [] @@ -117,10 +118,12 @@ def top_todo(self, account, size=25): return todos def _rationalise_todos(self, todos, size): - boosted = list(filter(lambda x: x["boost"], todos)) - unboosted = list(filter(lambda x: not x["boost"], todos)) + boost_groups = sorted(list(set([x["boost"] for x in todos])), reverse=True) - stds = sorted(boosted, key=lambda x: x['date']) + sorted(unboosted, key=lambda x: x['date']) + stds = [] + for bg in boost_groups: + group = list(filter(lambda x: x["boost"] == bg, todos)) + stds += sorted(group, key=lambda x: x['date']) id_map = {} removals = [] @@ -154,7 +157,7 @@ def maned_stalled(cls, size, maned_of): sort="last_manual_update", size=size ) - return constants.TODO_MANED_STALLED, stalled, "last_manual_update", False + return constants.TODO_MANED_STALLED, stalled, "last_manual_update", 0 @classmethod def maned_follow_up_old(cls, size, maned_of): @@ -169,7 +172,7 @@ def maned_follow_up_old(cls, size, maned_of): sort="created_date", size=size ) - return constants.TODO_MANED_FOLLOW_UP_OLD, follow_up_old, "created_date", False + return constants.TODO_MANED_FOLLOW_UP_OLD, follow_up_old, "created_date", 0 @classmethod def maned_ready(cls, size, maned_of): @@ -181,7 +184,7 @@ def maned_ready(cls, size, maned_of): sort="last_manual_update", size=size ) - return constants.TODO_MANED_READY, ready, "last_manual_update", True + return constants.TODO_MANED_READY, ready, "last_manual_update", 1 @classmethod def maned_completed(cls, size, maned_of): @@ -194,7 +197,7 @@ def maned_completed(cls, size, maned_of): sort="last_manual_update", size=size ) - return constants.TODO_MANED_COMPLETED, completed, "last_manual_update", False + return constants.TODO_MANED_COMPLETED, completed, "last_manual_update", 0 @classmethod def maned_assign_pending(cls, size, maned_of): @@ -211,7 +214,7 @@ def maned_assign_pending(cls, size, maned_of): sort="created_date", size=size ) - return constants.TODO_MANED_ASSIGN_PENDING, assign_pending, "last_manual_update", False + return constants.TODO_MANED_ASSIGN_PENDING, assign_pending, "last_manual_update", 0 @classmethod def editor_stalled(cls, groups, size): @@ -230,7 +233,7 @@ def editor_stalled(cls, groups, size): sort="last_manual_update", size=size ) - return constants.TODO_EDITOR_STALLED, stalled, "last_manual_update", False + return constants.TODO_EDITOR_STALLED, stalled, "last_manual_update", 0 @classmethod def editor_follow_up_old(cls, groups, size): @@ -249,7 +252,7 @@ def editor_follow_up_old(cls, groups, size): sort="created_date", size=size ) - return constants.TODO_EDITOR_FOLLOW_UP_OLD, follow_up_old, "created_date", False + return constants.TODO_EDITOR_FOLLOW_UP_OLD, follow_up_old, "created_date", 0 @classmethod def editor_completed(cls, groups, size): @@ -261,7 +264,7 @@ def editor_completed(cls, groups, size): sort="last_manual_update", size=size ) - return constants.TODO_EDITOR_COMPLETED, completed, "last_manual_update", True + return constants.TODO_EDITOR_COMPLETED, completed, "last_manual_update", 1 @classmethod def editor_assign_pending(cls, groups, size): @@ -276,7 +279,7 @@ def editor_assign_pending(cls, groups, size): sort="created_date", size=size ) - return constants.TODO_EDITOR_ASSIGN_PENDING, assign_pending, "created_date", False + return constants.TODO_EDITOR_ASSIGN_PENDING, assign_pending, "created_date", 1 @classmethod def associate_stalled(cls, acc_id, size): @@ -297,7 +300,7 @@ def associate_stalled(cls, acc_id, size): sort=sort_field, size=size ) - return constants.TODO_ASSOCIATE_PROGRESS_STALLED, stalled, sort_field, False + return constants.TODO_ASSOCIATE_PROGRESS_STALLED, stalled, sort_field, 0 @classmethod def associate_follow_up_old(cls, acc_id, size): @@ -318,7 +321,7 @@ def associate_follow_up_old(cls, acc_id, size): sort=sort_field, size=size ) - return constants.TODO_ASSOCIATE_FOLLOW_UP_OLD, follow_up_old, sort_field, False + return constants.TODO_ASSOCIATE_FOLLOW_UP_OLD, follow_up_old, sort_field, 0 @classmethod def associate_start_pending(cls, acc_id, size): @@ -331,7 +334,27 @@ def associate_start_pending(cls, acc_id, size): sort=sort_field, size=size ) - return constants.TODO_ASSOCIATE_START_PENDING, assign_pending, sort_field, False + return constants.TODO_ASSOCIATE_START_PENDING, assign_pending, sort_field, 0 + + @classmethod + def associate_all_applications(cls, acc_id, size): + sort_field = "created_date" + all = TodoQuery( + musts=[ + TodoQuery.editor(acc_id) + ], + must_nots=[ + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_READY, + constants.APPLICATION_STATUS_COMPLETED + ]) + ], + sort=sort_field, + size=size + ) + return constants.TODO_ASSOCIATE_ALL_APPLICATIONS, all, sort_field, -1 class TodoQuery(object): @@ -421,7 +444,7 @@ def editor_groups(cls, groups): def editor(cls, acc_id): return { "terms": { - "admin.editor": [acc_id], + "admin.editor.exact": [acc_id], } } diff --git a/portality/constants.py b/portality/constants.py index 8b417623b2..d2cb1166e7 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -51,6 +51,7 @@ TODO_ASSOCIATE_PROGRESS_STALLED = "todo_associate_progress_stalled" TODO_ASSOCIATE_FOLLOW_UP_OLD = "todo_associate_follow_up_old" TODO_ASSOCIATE_START_PENDING = "todo_associate_start_pending" +TODO_ASSOCIATE_ALL_APPLICATIONS = "todo_associate_all_applications" # Roles ROLE_ASSOCIATE_EDITOR = 'associate_editor' diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html index 838ec0b6f0..2a05d26742 100644 --- a/portality/templates/dashboard/_todo.html +++ b/portality/templates/dashboard/_todo.html @@ -34,8 +34,8 @@ }, constants.TODO_EDITOR_FOLLOW_UP_OLD: { "text" : "Old +8 wks old", - "colour" : "var(--yellow)", - "feather": "inbox", + "colour" : "var(--sanguine)", + "feather": "clock" }, constants.TODO_EDITOR_COMPLETED: { "text" : "Completed Recently completed", @@ -64,6 +64,12 @@ "colour" : "var(--yellow)", "feather": "inbox", "link" : url_for('editor.associate_suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"pending"}]) + }, + constants.TODO_ASSOCIATE_ALL_APPLICATIONS: { + "text" : "Application Open application", + "colour" : "var(--yellow)", + "feather": "inbox", + "link" : url_for('editor.associate_suggestions') } } %} diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index 48369138ca..5156514c45 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -5,6 +5,7 @@ {# ~~-> EditorGroup:Model ~~ #} {% set managed_groups, maned_assignments = maned_of() %} {% set editor_of_groups, editor_of_assignments = editor_of() %} +{% set associate_of_groups, associate_of_assignments = associate_of() %} {% block base_content %} @@ -124,44 +125,48 @@

    {% endif %} {% elif current_user.has_role("editor") %} -
    - Editor groups: -
    -
    -
      - {% for group in editor_of %} -
    • - {% set app_source = search_query_source(term=[ - {"admin.editor_group.exact" : group.name}, - {"index.application_type.exact" : "new application"} - ], sort=[{"admin.date_applied": {"order": "asc"}}] - ) %} - {{ group.name }} - {% set maned = group.get_maned_account() %} - {{ maned.id }} (Man. Ed.) - {{ " / " if not loop.last else "" }} -
    • - {% endfor %} -
    -
    + {% if editor_of_groups and editor_of_groups|length > 0 %} +
    + Editor groups: +
    +
    +
      + {% for eg in editor_of_groups|sort(attribute="name") %} +
    • + {% set app_source = search_query_source(term=[ + {"admin.editor_group.exact" : eg.name}, + {"index.application_type.exact" : "new application"} + ], sort=[{"admin.date_applied": {"order": "asc"}}] + ) %} + {{ eg.name }} + {% set maned = eg.get_maned_account() %} + {{ maned.id }} (Man. Ed.) + {{ " / " if not loop.last else "" }} +
    • + {% endfor %} +
    +
    + {% endif %} {% elif current_user.has_role("associate_editor") %} -
    - Editor groups: -
    -
    -
      - {% for group in associate_of %} -
    • - {{ group.name }} - {% set ed = group.get_editor_account() %} - {{ ed.id }} (Ed.) - {% set maned = group.get_maned_account() %} - {{ maned.id }} (Man. Ed.) - {{ " / " if not loop.last else "" }} -
    • - {% endfor %} -
    -
    + {% if associate_of_groups and associate_of_groups|length > 0 %} +
    + Editor groups: +
    +
    +
      + {% for group in associate_of_groups|sort(attribute="name") %} +
    • + {{ group.name }} + {% set ed = group.get_editor_account() %} + {{ ed.id }} (Ed.) + {% set maned = group.get_maned_account() %} + {{ maned.id }} (Man. Ed.) + {{ " / " if not loop.last else "" }} +
    • + {% endfor %} +
    +
    + {% endif %} {% endif %} diff --git a/portality/view/editor.py b/portality/view/editor.py index 8a7cfac172..b059030294 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -37,9 +37,9 @@ def index(): @login_required @ssl_required def groupinfo(): - editor_of = models.EditorGroup.groups_by_editor(current_user.id) - associate_of = models.EditorGroup.groups_by_associate(current_user.id) - return render_template('editor/index.html', editor_of=editor_of, associate_of=associate_of, managing_editor=app.config.get("MANAGING_EDITOR_EMAIL")) + # editor_of = models.EditorGroup.groups_by_editor(current_user.id) + # associate_of = models.EditorGroup.groups_by_associate(current_user.id) + return render_template('editor/index.html', managing_editor=app.config.get("MANAGING_EDITOR_EMAIL")) @blueprint.route('/group_journals') @login_required diff --git a/portality/view/testdrive.py b/portality/view/testdrive.py new file mode 100644 index 0000000000..764e5fe841 --- /dev/null +++ b/portality/view/testdrive.py @@ -0,0 +1,21 @@ +from flask import Blueprint, make_response, abort +from doajtest.testdrive.factory import TestFactory +from portality import util +from portality.core import app +import json + +# ~~Testdrive:Blueprint->$Testdrive:Feature~~ +blueprint = Blueprint('testdrive', __name__) + +@blueprint.route('/') +@util.jsonp +def testdrive(test_id): + # if not app.config.get("DEBUG", False): + # abort(404) + test = TestFactory.get(test_id) + if not test: + abort(404) + params = test.setup() + resp = make_response(json.dumps(params)) + resp.mimetype = "application/json" + return resp \ No newline at end of file From d49b80dabd490e4357ff6b75351d6be73015e4d1 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 23 Nov 2022 14:33:07 +0000 Subject: [PATCH 030/466] add testdrive for dashboard and associated functional tests and any fixes required to the dashboards --- doajtest/testbook/dashboards/assed_todo.yml | 52 ++++++ doajtest/testbook/dashboards/editor_todo.yml | 57 +++++++ .../dashboards/editorial_group_status.yml | 4 +- doajtest/testbook/dashboards/maned_todo.yml | 61 +++++++ doajtest/testdrive/todo_associate.py | 11 ++ doajtest/testdrive/todo_editor.py | 16 +- doajtest/testdrive/todo_editor_associate.py | 13 +- .../testdrive/todo_maned_editor_associate.py | 161 ++++++++++++++++++ portality/bll/services/todo.py | 7 +- portality/static/js/dashboard.js | 26 +-- portality/templates/editor/dashboard.html | 2 +- portality/templates/editor/index.html | 52 ------ portality/view/editor.py | 8 - portality/view/testdrive.py | 17 +- 14 files changed, 407 insertions(+), 80 deletions(-) create mode 100644 doajtest/testbook/dashboards/assed_todo.yml create mode 100644 doajtest/testbook/dashboards/editor_todo.yml create mode 100644 doajtest/testbook/dashboards/maned_todo.yml create mode 100644 doajtest/testdrive/todo_maned_editor_associate.py delete mode 100644 portality/templates/editor/index.html diff --git a/doajtest/testbook/dashboards/assed_todo.yml b/doajtest/testbook/dashboards/assed_todo.yml new file mode 100644 index 0000000000..921f36dccb --- /dev/null +++ b/doajtest/testbook/dashboards/assed_todo.yml @@ -0,0 +1,52 @@ +suite: Dashboard +testset: Associate Editor Todo List + +tests: + - title: Associate Editor Todo List + context: + role: associate editor + testdrive: todo_associate + setup: + - Use the todo_associate testdrive to setup for this test, of follow the next steps + - You should set up a user account which has only the associate editor role + - "The user account should be assigned at least 4 applications which meet the following criteria: one that was created over 6 weeks ago, + one that has not been modified for 3 weeks, one which has recently been assigned to the user and is in the pending state, and one + recent application" + steps: + - step: log in as an associate editor + - step: Go to the editor's dashboard page + path: /editor + results: + - You can see 4 applications in your priority list + - The highest priority application is for an old application + - The second priority is for a stalled application + - The third priority is for a recently assigned/pending application + - The lowest priority is for an open application + - step: click on the highest priority application + results: + - The application opens in a new browser tab/window + - step: Edit the application in some minor way and save + - step: close the tab, return to the dashboard and reload the page + results: + - Your priority list is unchanged + - step: Click on the highest priority application again + - step: Change the application status to "Completed" and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can see 3 applications in your priority list + - The application you have just edited as disappeared from your priority list + - step: click on the new highest priority application (stalled) + - step: Edit the application in some minor way and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can still see 3 applications in your priority list + - The stalled application you modified in the previous steps is no longer your highest priority, it is now + listed only as an "open application" + - Your new highest priority is a pending application + - step: click on the new highest priority application (pending) + - step: Change the application status to "In Progress" and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can still see 3 applications in your priority list + - All your applications are now "open applications" + - They are ordered by created date, with the oldest first \ No newline at end of file diff --git a/doajtest/testbook/dashboards/editor_todo.yml b/doajtest/testbook/dashboards/editor_todo.yml new file mode 100644 index 0000000000..f52b43da49 --- /dev/null +++ b/doajtest/testbook/dashboards/editor_todo.yml @@ -0,0 +1,57 @@ +suite: Dashboard +testset: Editor Todo List + +tests: + - title: Editor Todo List + context: + role: editor + testdrive: todo_editor_associate + setup: + - Use the todo_editor_associate testdrive to setup for this test, of follow the next steps + - You should set up a user account which has both the editor and associate editor roles, and is the editor of at least one editorial group + - "The user account should be assigned at least 4 applications which meet the following criteria: one that was created over 6 weeks ago, + one that has not been modified for 3 weeks, one which has recently been assigned to the user and is in the pending state, and one + recent application" + - "The user account should be assigned another 4 applications which meet the following criteria: one that is in the completed state and assigned + to your editorial group, one that is assigned to your editorial group in the pending state but with no associate editor assigned, one that + is in your editorial group which was created less than 8 weeks ago but which hasn't been updated for 6 weeks, and one that is + in your editorial group which was created more than 8 weeks ago" + steps: + - step: log in as an editor + - step: Go to the editor's dashboard page + path: /editor + results: + - You can see 8 applications in your priority list + - The highest priority is for a recently completed application (in your editorial group) + - The second priority is for a recently assigned/pending application (in your editorial group) + - The third priority is for an old (+8 weeks) application (in your editorial group) + - The fourth priority is for a stalled (+6 weeks) application (in your editorial group) + - The fifth priority is for an old application (+6 weeks) (assigned to you as an associate editor) + - The sixth priority is for a stalled application (assigned to you as an associate editor) + - The seventh priority is for a recently assigned/pending application (assigned to you as an associate editor) + - The lowest priority is for an open application (assigned to you as an associate editor) + - step: click on the highest priority application + - step: Change the application status to "Ready" and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can see 7 applications in your priority list + - The application you have just edited as disappeared from your priority list + - step: click on the new highest priority application (pending) + - step: assign an associate editor (ideally yourself) to the application and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can still see 7 applications in your priority list + - The pending application you modified in the previous steps is no longer your highest priority, it is now + listed as recently assigned/pending, further down the list + - Your new highest priority is an old application + - step: click on the new highest priority application (old) + - step: Modify the item in some minor way and save + - step: close the tab, return to the dashboard and reload the page + results: + - Your list of priorities has not changed + - step: click on your second highest priority (stalled) + - step: Modify the item in some minor way and save + - step: close the tab, return to the dashboard and reload the page + results: + - You have 6 applications left in your todo list + - The stalled application you just edited is no longer visible \ No newline at end of file diff --git a/doajtest/testbook/dashboards/editorial_group_status.yml b/doajtest/testbook/dashboards/editorial_group_status.yml index 895e2b27bb..8c595f01bd 100644 --- a/doajtest/testbook/dashboards/editorial_group_status.yml +++ b/doajtest/testbook/dashboards/editorial_group_status.yml @@ -1,5 +1,5 @@ -suite: Editorial Group Status -testset: Editors +suite: Dashboard +testset: Editorial Group Status for Editors tests: - title: Record Counts diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml new file mode 100644 index 0000000000..8bc3564a6d --- /dev/null +++ b/doajtest/testbook/dashboards/maned_todo.yml @@ -0,0 +1,61 @@ +suite: Dashboard +testset: ManEd Todo List + +tests: + - title: ManEd Todo List + context: + role: admin + testdrive: todo_maned_editor_associate + setup: + - Use the todo_maned_editor_associate testdrive to setup for this test, OR follow the next steps + - You should set up a user account which has the admin, editor and associate editor roles, and is the maned of at least one editorial group, and + editor of at least one other editorial group + - "The user account should be assigned at least 4 applications which meet the following criteria: one that was created over 6 weeks ago, + one that has not been modified for 3 weeks, one which has recently been assigned to the user and is in the pending state, and one + recent application" + - "The user account should be assigned another 4 applications which meet the following criteria: one that is in the completed state and assigned + to your editorial group, one that is assigned to your editorial group in the pending state but with no associate editor assigned, one that + is in your editorial group which was created less than 8 weeks ago but which hasn't been updated for 6 weeks, and one that is + in your editorial group which was created more than 8 weeks ago" + - "The user account should be assigned another 5 applications which meet the following criteria: an application in your maned group which is + in the ready state, an application in your maned group which is in the completed state, an application in your maned group which has + not had an associate editor assigned, an application created over 10 weeks ago in your maned group, an application in your maned + group which has not been updated for 8 weeks." + steps: + - step: log in as a managing editor + - step: Go to the maned dashboard page + path: /dashboard + results: + - You can see 13 applications in your priority list + - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), + editor items (actions related to teams you are the editor for) and associate items (actions related to applications which + are assigned specifically to you for review). + - At least one of your priority items is for an application which is older than 10 weeks (it should indicate that it is for your maned group) + - At least one of your priority items is for an application which has been inactive (stalled) for more than 8 weeks (it should indicate that it is for your maned group) + - At least one of your priority items is for an application in the state ready (it should indicate that it is for your maned group) + - At least one of your priority items is for an application in the completed state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) + - At least one of your priority items is for an application in the pending state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) + - step: click on the managing editor's ready application + - step: Change the application status to "Accepted" and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can see 12 applications in your priority list + - The application you have just edited has disappeared from your priority list + - step: click on the [in progress] stalled managing editor's application + - step: make any minor adjustment to the metadata and save + - step: close the tab, return to the dashboard and reload the page + results: + - You can see 11 applications in your priority list + - The application you just edited has disappeared from your priority list + - step: click on the "completed" maned application + - step: Change the application to "ready" status + - step: close the tab, return to the dashboard and reload the page + results: + - You can still see 11 applications in your priority list + - The completed application you just moved to ready is now in your priority list as a ready application + - step: click on the pending managing editor's application + - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select) + - step: close the tab, return to the dashboard and reload the page + results: + - You have 10 applications left in your todo list + - The pending application you just edited is no longer visible \ No newline at end of file diff --git a/doajtest/testdrive/todo_associate.py b/doajtest/testdrive/todo_associate.py index c85a88669d..be877c3a2f 100644 --- a/doajtest/testdrive/todo_associate.py +++ b/doajtest/testdrive/todo_associate.py @@ -36,6 +36,14 @@ def setup(self) -> dict: "applications": apps } + def teardown(self, params) -> dict: + models.Account.remove_by_id(params["account"]["username"]) + models.EditorGroup.remove_by_id(params["editor_group"]["id"]) + for nature, details in params["applications"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + return {"status": "success"} + def build_applications(un): w = 7 * 24 * 60 * 60 @@ -81,6 +89,9 @@ def build_application(title, lmu_diff, cd_diff, status, editor=None): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.bibjson().title = title + ap.remove_current_journal() + ap.remove_related_journal() + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION ap.set_id(ap.makeid()) ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) ap.set_created(dates.before(datetime.utcnow(), cd_diff)) diff --git a/doajtest/testdrive/todo_editor.py b/doajtest/testdrive/todo_editor.py index aa81164018..c73939597d 100644 --- a/doajtest/testdrive/todo_editor.py +++ b/doajtest/testdrive/todo_editor.py @@ -36,12 +36,23 @@ def setup(self) -> dict: "applications": apps } + def teardown(self, params) -> dict: + models.Account.remove_by_id(params["account"]["username"]) + models.EditorGroup.remove_by_id(params["editor_group"]["id"]) + for nature, details in params["applications"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + return {"status": "success"} + def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.bibjson().title = title ap.set_id(ap.makeid()) + ap.remove_current_journal() + ap.remove_related_journal() + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) ap.set_created(dates.before(datetime.utcnow(), cd_diff)) ap.set_application_status(status) @@ -61,7 +72,7 @@ def build_applications(un, eg): apps = {} - app = build_application(un + " Stalled Application", 6 * w, 12 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + app = build_application(un + " Stalled Application", 6 * w, 7 * w, constants.APPLICATION_STATUS_IN_PROGRESS, editor_group=eg.name) app.save() apps["stalled"] = [{ @@ -94,4 +105,5 @@ def build_applications(un, eg): "title": un + " Pending Application" }] - return apps \ No newline at end of file + return apps + diff --git a/doajtest/testdrive/todo_editor_associate.py b/doajtest/testdrive/todo_editor_associate.py index 35161f1d5e..095d9b2539 100644 --- a/doajtest/testdrive/todo_editor_associate.py +++ b/doajtest/testdrive/todo_editor_associate.py @@ -37,4 +37,15 @@ def setup(self) -> dict: "associate": aapps, "editor": eapps } - } \ No newline at end of file + } + + def teardown(self, params) -> dict: + models.Account.remove_by_id(params["account"]["username"]) + models.EditorGroup.remove_by_id(params["editor_group"]["id"]) + for nature, details in params["applications"]["associate"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + for nature, details in params["applications"]["editor"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + return {"status": "success"} \ No newline at end of file diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py new file mode 100644 index 0000000000..a30f45cea0 --- /dev/null +++ b/doajtest/testdrive/todo_maned_editor_associate.py @@ -0,0 +1,161 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from doajtest.fixtures.v2.applications import ApplicationFixtureFactory +from doajtest.testdrive.todo_associate import build_applications as build_associate_applications +from doajtest.testdrive.todo_editor import build_applications as build_editor_applications +from portality import models +from portality.lib import dates +from datetime import datetime + + +class TodoManedEditorAssociate(TestDrive): + + def setup(self) -> dict: + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "TodoManedEditorAssociate " + un, ["admin", "editor", constants.ROLE_ASSOCIATE_EDITOR]) + acc.set_password(pw) + acc.save() + + oun = self.create_random_str() + owner = models.Account.make_account(oun + "@example.com", oun, "Owner " + un, ["publisher"]) + owner.save() + + eun = self.create_random_str() + editor = models.Account.make_account(eun + "@example.com", eun, "Associate Editor " + un, ["associate_editor"]) + editor.save() + + gn1 = "Maned Group " + un + eg1 = models.EditorGroup(**{ + "name": gn1 + }) + eg1.set_maned(acc.id) + eg1.add_associate(editor.id) + eg1.save() + + gn2 = "Editor Group " + un + eg2 = models.EditorGroup(**{ + "name": gn2 + }) + eg2.set_editor(acc.id) + eg2.save() + + aapps = build_associate_applications(un) + eapps = build_editor_applications(un, eg2) + mapps = build_maned_applications(un, eg1, owner.id) + + return { + "account": { + "username": acc.id, + "password": pw + }, + "users": [ + owner.id, + editor.id + ], + "editor_group": { + "id": eg2.id, + "name": eg2.name + }, + "maned_group": { + "id": eg1.id, + "name": eg1.name + }, + "applications": { + "associate": aapps, + "editor": eapps, + "maned": mapps + } + } + + def teardown(self, params) -> dict: + models.Account.remove_by_id(params["account"]["username"]) + for user in params["users"]: + models.Account.remove_by_id(user) + models.EditorGroup.remove_by_id(params["editor_group"]["id"]) + models.EditorGroup.remove_by_id(params["maned_group"]["id"]) + for nature, details in params["applications"]["associate"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + for nature, details in params["applications"]["editor"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + for nature, details in params["applications"]["maned"].items(): + for detail in details: + models.Application.remove_by_id(detail["id"]) + return {"status": "success"} + + +def build_maned_applications(un, eg, owner): + w = 7 * 24 * 60 * 60 + + apps = {} + + app = build_application(un + " Maned Stalled Application", 8 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + editor_group=eg.name, owner=owner) + app.save() + apps["stalled"] = [{ + "id": app.id, + "title": un + " Maned Stalled Application" + }] + + app = build_application(un + " Maned Old Application", 10 * w, 10 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + editor_group=eg.name, owner=owner) + app.save() + apps["old"] = [{ + "id": app.id, + "title": un + " Maned Old Application" + }] + + app = build_application(un + " Maned Ready Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_READY, + editor_group=eg.name, owner=owner) + app.save() + apps["ready"] = [{ + "id": app.id, + "title": un + " Maned Completed Application" + }] + + app = build_application(un + " Maned Completed Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_COMPLETED, + editor_group=eg.name, owner=owner) + app.save() + apps["completed"] = [{ + "id": app.id, + "title": un + " Maned Completed Application" + }] + + app = build_application(un + " Maned Pending Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_PENDING, + editor_group=eg.name, owner=owner) + app.remove_editor() + app.save() + apps["pending"] = [{ + "id": app.id, + "title": un + " Maned Pending Application" + }] + + return apps + + +def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None, owner=None): + source = ApplicationFixtureFactory.make_application_source() + ap = models.Application(**source) + ap.bibjson().title = title + ap.set_id(ap.makeid()) + ap.remove_current_journal() + ap.remove_related_journal() + del ap.bibjson().discontinued_date + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION + ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) + ap.set_created(dates.before(datetime.utcnow(), cd_diff)) + ap.set_application_status(status) + + if editor is not None: + ap.set_editor(editor) + + if editor_group is not None: + ap.set_editor_group(editor_group) + + if owner is not None: + ap.set_owner(owner) + + ap.save() + return ap diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 8c540f8d65..f9683d7c47 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -18,9 +18,10 @@ def group_stats(self, group_id): editors = [eg.editor] + eg.associates for editor in editors: acc = models.Account.pull(editor) - stats["editors"][editor] = { - "email" : acc.email - } + if acc is not None: + stats["editors"][editor] = { + "email" : acc.email + } q = GroupStatsQuery(eg.name) resp = models.Application.query(q=q.query()) diff --git a/portality/static/js/dashboard.js b/portality/static/js/dashboard.js index 0bca817273..ae6b6a2752 100644 --- a/portality/static/js/dashboard.js +++ b/portality/static/js/dashboard.js @@ -54,9 +54,13 @@ doaj.dashboard.renderGroupInfo = function(data) { // ~~-> EditorGroup:Model~~ // first remove the editor from the associates list if they are there - let edInAssEd = data.editor_group.associates.indexOf(data.editor_group.editor) - if (edInAssEd > -1) { - data.editor_group.associates.splice(edInAssEd, 1); + if (data.editor_group.associates && data.editor_group.associates.length > 0) { + let edInAssEd = data.editor_group.associates.indexOf(data.editor_group.editor) + if (edInAssEd > -1) { + data.editor_group.associates.splice(edInAssEd, 1); + } + } else { + data.editor_group.associates = []; // just to avoid having to keep checking it below } let allEditors = [data.editor_group.editor].concat(data.editor_group.associates); @@ -86,14 +90,16 @@ doaj.dashboard.renderGroupInfo = function(data) { urCount = data.by_editor[ed].update_requests || 0; } - let isEd = ""; - if (i === 0) { // first one in the list is always the editor - isEd = " (Ed.)" + if (data.editors[ed]) { + let isEd = ""; + if (i === 0) { // first one in the list is always the editor + isEd = " (Ed.)" + } + editorListFrag += `
  • + ${ed}${isEd} + ${appCount} applications +
  • `; } - editorListFrag += `
  • - ${ed}${isEd} - ${appCount} applications -
  • `; } // ~~-> ApplicationSearch:Page~~ diff --git a/portality/templates/editor/dashboard.html b/portality/templates/editor/dashboard.html index 905cf97f6f..f22196c2b2 100644 --- a/portality/templates/editor/dashboard.html +++ b/portality/templates/editor/dashboard.html @@ -13,7 +13,7 @@

    Activity

    - {% include 'dashboard/nav.html' %} + {% block nav %} + {% include 'dashboard/nav.html' %} + {% endblock %} {% block extra_header %}{% endblock %} From 427ad2bb8b229422857740ee52f82bb640a3c23c Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 27 Mar 2023 12:00:03 +0100 Subject: [PATCH 082/466] new full new notification type - alert --- portality/constants.py | 2 +- .../static/js/edges/notifications.edge.js | 5 ++ portality/tasks/find_discontinued_soon.py | 52 +++++++++---------- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/portality/constants.py b/portality/constants.py index 2600c65173..cfd7346711 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -55,7 +55,7 @@ EVENT_JOURNAL_EDITOR_GROUP_ASSIGNED = "journal:editor_group:assigned" EVENT_JOURNAL_DISCONTINUING_SOON = "journal:discontinuing_soon" -NOTIFICATION_CLASSIFICATION_STATUS = "status" +NOTIFICATION_CLASSIFICATION_STATUS = "alert" NOTIFICATION_CLASSIFICATION_STATUS_CHANGE = "status_change" NOTIFICATION_CLASSIFICATION_ASSIGN = "assign" NOTIFICATION_CLASSIFICATION_CREATE = "create" diff --git a/portality/static/js/edges/notifications.edge.js b/portality/static/js/edges/notifications.edge.js index 65788540ad..4b8c00e083 100644 --- a/portality/static/js/edges/notifications.edge.js +++ b/portality/static/js/edges/notifications.edge.js @@ -10,6 +10,10 @@ $.extend(true, doaj, { seen_url: "/dashboard/notifications/{notification_id}/seen", icons: { + alert: ` + + + `, finished: ` @@ -34,6 +38,7 @@ $.extend(true, doaj, { }, classifications: { + alert: "Requires attention", finished: "Task has completed", status_change: "Application status change", assign: "Assigned to user" diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 5111a4af51..16786480d2 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -14,10 +14,6 @@ from portality.ui.messages import Messages from portality import constants - -def _date(): - return (datetime.datetime.today() + datetime.timedelta(days=app.config.get('DISCONTINUED_DATE_DELTA', 1))).strftime( - '%Y-%m-%d') class DiscontinuedSoonQuery: def __init__(self, time_delta=None): self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); @@ -43,6 +39,10 @@ def query(self): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" + def __init__(self, time_delta=None): + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); + self._date = dates.days_after_now(days=self._delta) + def find_journals_discontinuing_soon(self, job): jdata = [] @@ -63,7 +63,7 @@ def run(self): "system", { "journal": j, - "discontinue_date": _date() + "discontinue_date": self._date() })) else: job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) @@ -118,24 +118,24 @@ def find_discontinued_soon(job_id): # Test code - please do not clean up until after the tests -# def find_journals_discontinuing_soon(): -# jdata = [] -# -# q = DiscontinuedSoonQuery() -# for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): -# # ~~->Journal:Model~~ -# jdata.append(journal.id) -# -# return jdata -# -# if __name__ == "__main__": -# journals = find_journals_discontinuing_soon() -# if len(journals): -# for j in journals: -# DOAJ.eventsService().trigger(models.Event( -# constants.EVENT_JOURNAL_DISCONTINUING_SOON, -# "system", -# { -# "journal": j, -# "discontinue_date": _date() -# })) \ No newline at end of file +def find_journals_discontinuing_soon(): + jdata = [] + + q = DiscontinuedSoonQuery() + for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): + # ~~->Journal:Model~~ + jdata.append(journal.id) + + return jdata + +if __name__ == "__main__": + journals = find_journals_discontinuing_soon() + if len(journals): + for j in journals: + DOAJ.eventsService().trigger(models.Event( + constants.EVENT_JOURNAL_DISCONTINUING_SOON, + "system", + { + "journal": j, + "discontinue_date": dates.days_after_now(days=0) + })) \ No newline at end of file From 890f2263acba70f9512986c17ee761b160249622 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 14 Apr 2023 10:05:18 +0100 Subject: [PATCH 083/466] fix is_before usage --- portality/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/app.py b/portality/app.py index 218fd4e793..1083899d85 100644 --- a/portality/app.py +++ b/portality/app.py @@ -281,7 +281,7 @@ def form_diff_table_subject_expand(val): @app.template_filter("is_in_the_past") def is_in_the_past(dttm): - return dates.is_before(dates.today()) + return dates.is_before(dttm, dates.today()) ####################################################### From 6139624309485ee1f391e943b3f5c0a66eb756dc Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 17 Apr 2023 13:28:48 +0100 Subject: [PATCH 084/466] clean up --- portality/tasks/find_discontinued_soon.py | 26 +---------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 16786480d2..d9c281abb2 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -114,28 +114,4 @@ def scheduled_find_discontinued_soon(): def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) - BackgroundApi.execute(task) - - -# Test code - please do not clean up until after the tests -def find_journals_discontinuing_soon(): - jdata = [] - - q = DiscontinuedSoonQuery() - for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): - # ~~->Journal:Model~~ - jdata.append(journal.id) - - return jdata - -if __name__ == "__main__": - journals = find_journals_discontinuing_soon() - if len(journals): - for j in journals: - DOAJ.eventsService().trigger(models.Event( - constants.EVENT_JOURNAL_DISCONTINUING_SOON, - "system", - { - "journal": j, - "discontinue_date": dates.days_after_now(days=0) - })) \ No newline at end of file + BackgroundApi.execute(task) \ No newline at end of file From 73416774d544d29013f7796fb8b8b1e81687f8fd Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 18 Apr 2023 12:48:16 +0100 Subject: [PATCH 085/466] Update the cron schedule for discontinued task --- portality/settings.py | 2 +- test.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 645ae5da18..30c9ddc85f 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -434,7 +434,7 @@ "anon_export": {"month": "*", "day": "10", "day_of_week": "*", "hour": "6", "minute": "30"}, "old_data_cleanup": {"month": "*", "day": "12", "day_of_week": "*", "hour": "6", "minute": "30"}, "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, - "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "7", "minute": "0"}, + "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "0", "minute": "3"} } HUEY_TASKS = { diff --git a/test.cfg b/test.cfg index 37eea9ca92..907b59bb7b 100644 --- a/test.cfg +++ b/test.cfg @@ -55,7 +55,8 @@ HUEY_SCHEDULE = { "harvest": {"month": "*", "day": "*", "day_of_week": "*", "hour": "5", "minute": "30"}, "anon_export": CRON_NEVER, "old_data_cleanup": {"month": "*", "day": "*", "day_of_week": "3", "hour": "12", "minute": "0"}, - "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"} + "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, + "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "0", "minute": "3"} } # ======================= From ea294dc4e1419cad8c692c12c60645cf5be48de4 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 2 May 2023 17:52:57 +0100 Subject: [PATCH 086/466] Add missing import for new background job --- portality/tasks/consumer_long_running.py | 2 +- portality/tasks/consumer_main_queue.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/portality/tasks/consumer_long_running.py b/portality/tasks/consumer_long_running.py index bf0aa61b61..ff8763e148 100644 --- a/portality/tasks/consumer_long_running.py +++ b/portality/tasks/consumer_long_running.py @@ -16,4 +16,4 @@ from portality.tasks.harvester import scheduled_harvest # noqa from portality.tasks.anon_export import scheduled_anon_export, anon_export # noqa from portality.tasks.old_data_cleanup import scheduled_old_data_cleanup, execute_old_data_cleanup # noqa -from portality.tasks.monitor_bgjobs import scheduled_monitor_bgjobs, execute_monitor_bgjobs +from portality.tasks.monitor_bgjobs import scheduled_monitor_bgjobs, execute_monitor_bgjobs # noqa diff --git a/portality/tasks/consumer_main_queue.py b/portality/tasks/consumer_main_queue.py index 2eda75d8da..ea3774d560 100644 --- a/portality/tasks/consumer_main_queue.py +++ b/portality/tasks/consumer_main_queue.py @@ -26,3 +26,4 @@ from portality.tasks.async_workflow_notifications import async_workflow_notifications # noqa from portality.tasks.check_latest_es_backup import scheduled_check_latest_es_backup, check_latest_es_backup # noqa from portality.tasks.request_es_backup import scheduled_request_es_backup, request_es_backup # noqa +from portality.tasks.find_discontinued_soon import scheduled_find_discontinued_soon, find_discontinued_soon # noqa From c648cca673eaebfbd411a00ef16819ec5cfcf902 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Wed, 3 May 2023 14:33:17 +0100 Subject: [PATCH 087/466] update the dashboard card text and fix bug with todo rules for editors --- portality/bll/services/todo.py | 4 ++-- portality/templates/dashboard/_todo.html | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 13a203461c..2ea93bcc95 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -227,7 +227,7 @@ def editor_stalled(cls, groups, size): TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_COMPLETED + constants.APPLICATION_STATUS_READY ]) ], sort="last_manual_update", @@ -246,7 +246,7 @@ def editor_follow_up_old(cls, groups, size): TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_COMPLETED + constants.APPLICATION_STATUS_READY ]) ], sort="created_date", diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html index b2dcaa81d0..7a3cbea5a8 100644 --- a/portality/templates/dashboard/_todo.html +++ b/portality/templates/dashboard/_todo.html @@ -1,66 +1,66 @@ {% set TODOS = { constants.TODO_MANED_STALLED: { - "text" : "Stalled Contact the editor", + "text" : "Stalled Chase Editor", "show_status": true, "colour" : "var(--salmon)", "feather": "coffee" }, constants.TODO_MANED_FOLLOW_UP_OLD: { - "text": "Old Chase progress", + "text": "Old Chase Editor", "show_status": true, "colour" : "var(--sanguine)", "feather": "clock" }, constants.TODO_MANED_READY: { - "text" : "Ready Make a decision", + "text" : "Ready Make decision", "colour" : "var(--dark-green)", "feather": "check-circle", "link" : url_for('admin.suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"ready"}]) }, constants.TODO_MANED_COMPLETED: { - "text" : "Completed Make a decision", + "text" : "Completed Chase Editor", "colour" : "var(--mid-green)", "feather": "user-check", "link" : url_for('admin.suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"completed"}]) }, constants.TODO_MANED_ASSIGN_PENDING: { - "text" : "Pending Chase the Editor to assign", + "text" : "Pending Chase Editor", "colour" : "var(--yellow)", "feather": "inbox", "link" : url_for('admin.suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"pending"}]) }, constants.TODO_EDITOR_STALLED: { - "text" : "Stalled Contact the Associate Editor", + "text" : "Stalled Chase Associate Editor", "show_status": true, "colour" : "var(--yellow)", "feather": "inbox", }, constants.TODO_EDITOR_FOLLOW_UP_OLD: { - "text" : "Old Chase progress", + "text" : "Old Chase Associate Editor", "show_status": true, "colour" : "var(--sanguine)", "feather": "clock" }, constants.TODO_EDITOR_COMPLETED: { - "text" : "Completed Make a decision", + "text" : "Completed Review and set to Ready", "colour" : "var(--yellow)", "feather": "inbox", "link" : url_for('editor.group_suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"completed"}]) }, constants.TODO_EDITOR_ASSIGN_PENDING: { - "text" : "Pending Assign an Associate Editor", + "text" : "Pending Assign to Associate Editor", "colour" : "var(--yellow)", "feather": "inbox", "link" : url_for('editor.group_suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"pending"}]) }, constants.TODO_ASSOCIATE_PROGRESS_STALLED: { - "text" : "Stalled Review progress", + "text" : "Stalled Complete as soon as possible", "show_status": true, "colour" : "var(--yellow)", "feather": "inbox", }, constants.TODO_ASSOCIATE_FOLLOW_UP_OLD: { - "text": "Old Chase progress", + "text": "Old Complete as soon as possible", "show_status": true, "colour" : "var(--sanguine)", "feather": "clock" From 80ddf82fa228c0baab3d848c4f5a5c81845aa09f Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 4 May 2023 14:30:11 +0100 Subject: [PATCH 088/466] add super init to the bg job --- portality/tasks/find_discontinued_soon.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index d9c281abb2..1edac3b390 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -39,23 +39,23 @@ def query(self): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" - def __init__(self, time_delta=None): + def __init__(self, job, time_delta=None): + super(FindDiscontinuedSoonBackgroundTask,self).__init__(job) self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); self._date = dates.days_after_now(days=self._delta) - def find_journals_discontinuing_soon(self, job): + def find_journals_discontinuing_soon(self): jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery().query(), keepalive='5m', wrap=True): # ~~->Journal:Model~~ jdata.append(journal.id) - job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) + self.background_job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) return jdata def run(self): - job = self.background_job - journals = self.find_journals_discontinuing_soon(job=job) + journals = self.find_journals_discontinuing_soon() if len(journals): for j in journals: DOAJ.eventsService().trigger(models.Event( @@ -63,10 +63,10 @@ def run(self): "system", { "journal": j, - "discontinue_date": self._date() + "discontinue_date": self._date })) else: - job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) + self.background_job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) def cleanup(self): """ From 1fdb9661551dc7f448a05fcd0827eb3dafd3ba12 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 18 May 2023 12:15:39 +0100 Subject: [PATCH 089/466] Propagate refactor of dates.after to dates.seconds_after --- doajtest/unit/api_tests/test_apiv3_discovery.py | 4 ++-- .../sync_journals_applications.py | 2 +- portality/tasks/article_duplicate_report.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doajtest/unit/api_tests/test_apiv3_discovery.py b/doajtest/unit/api_tests/test_apiv3_discovery.py index b847c48b5b..ffc94feacc 100644 --- a/doajtest/unit/api_tests/test_apiv3_discovery.py +++ b/doajtest/unit/api_tests/test_apiv3_discovery.py @@ -241,7 +241,7 @@ def test_03_applications(self): for i in range(5): a = models.Suggestion() a.set_owner("owner") - a.set_created(dates.format(dates.after(now, i))) + a.set_created(dates.format(dates.seconds_after(now, i))) bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) @@ -256,7 +256,7 @@ def test_03_applications(self): for i in range(5): a = models.Suggestion() a.set_owner("stranger") - a.set_created(dates.format(dates.after(now, i + 5))) + a.set_created(dates.format(dates.seconds_after(now, i + 5))) bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) diff --git a/portality/migrate/20180106_1463_ongoing_updates/sync_journals_applications.py b/portality/migrate/20180106_1463_ongoing_updates/sync_journals_applications.py index adb85410aa..4315618d51 100644 --- a/portality/migrate/20180106_1463_ongoing_updates/sync_journals_applications.py +++ b/portality/migrate/20180106_1463_ongoing_updates/sync_journals_applications.py @@ -52,7 +52,7 @@ app_created = application.created_timestamp for journal in related_journals: almu = application.last_manual_update_timestamp - almu_adjusted = dates.after(almu, 3600) + almu_adjusted = dates.seconds_after(almu, 3600) # do a load of reporting prep jc_ac_diff = int((journal.created_timestamp - app_created).total_seconds()) diff --git a/portality/tasks/article_duplicate_report.py b/portality/tasks/article_duplicate_report.py index b92616d837..04419674e6 100644 --- a/portality/tasks/article_duplicate_report.py +++ b/portality/tasks/article_duplicate_report.py @@ -72,7 +72,7 @@ def run(self): n = dates.now() diff = (n - start).total_seconds() expected_total = ((diff / a_count) * total) - estimated_finish = dates.format(dates.after(start, expected_total)) + estimated_finish = dates.format(dates.seconds_after(start, expected_total)) a_count += 1 article = models.Article(_source={'id': a[0], 'created_date': a[1], 'bibjson': {'identifier': json.loads(a[2]), 'link': json.loads(a[3]), 'title': a[4]}, 'admin': {'in_doaj': json.loads(a[5])}}) From f70586e7f4c3e0356ae95532a7ee01cc368ed3b3 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 18 May 2023 16:24:31 +0100 Subject: [PATCH 090/466] A few minor fixes to journals discontinuing task --- portality/tasks/find_discontinued_soon.py | 32 ++++++++++++----------- portality/ui/messages.py | 4 +-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 1edac3b390..8ca7a2524a 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -1,32 +1,30 @@ -import datetime -import csv -import json - from portality.core import app from portality.bll import DOAJ from portality.lib import dates -from portality import models,app_email +from portality import models from portality.tasks.redis_huey import main_queue -from portality.background import BackgroundTask, BackgroundSummary, BackgroundApi +from portality.background import BackgroundTask, BackgroundApi from portality.tasks.helpers import background_helper from portality.ui.messages import Messages from portality import constants + class DiscontinuedSoonQuery: def __init__(self, time_delta=None): - self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0) self._date = dates.days_after_now(days=self._delta) + def query(self): return { "query": { "bool": { "filter": { - "bool" : { + "bool": { "must": [ - {"term" : {"bibjson.discontinued_date": dates.format(self._date, format="%Y-%m-%d")}}, - {"term" : {"admin.in_doaj":True}} + {"term": {"bibjson.discontinued_date": dates.format(self._date, format="%Y-%m-%d")}}, + {"term": {"admin.in_doaj": True}} ] } } @@ -34,14 +32,14 @@ def query(self): } } -# ~~FindDiscontinuedSoonBackgroundTask:Task~~ +# ~~FindDiscontinuedSoonBackgroundTask:Task~~ class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" def __init__(self, job, time_delta=None): - super(FindDiscontinuedSoonBackgroundTask,self).__init__(job) - self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); + super(FindDiscontinuedSoonBackgroundTask, self).__init__(job) + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0) self._date = dates.days_after_now(days=self._delta) def find_journals_discontinuing_soon(self): @@ -60,11 +58,12 @@ def run(self): for j in journals: DOAJ.eventsService().trigger(models.Event( constants.EVENT_JOURNAL_DISCONTINUING_SOON, - "system", + self.background_job.user, { "journal": j, "discontinue_date": self._date })) + self.background_job.add_audit_message(Messages.DISCONTINUED_JOURNALS_FOUND_NOTIFICATION_SENT_LOG) else: self.background_job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) @@ -81,6 +80,7 @@ def prepare(cls, username, **kwargs): Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, or fail with a suitable exception + :param username: User account for this task to complete as :param kwargs: arbitrary keyword arguments pertaining to this task type :return: a BackgroundJob instance representing this task """ @@ -101,8 +101,10 @@ def submit(cls, background_job): background_job.save() find_discontinued_soon.schedule(args=(background_job.id,), delay=10) + huey_helper = FindDiscontinuedSoonBackgroundTask.create_huey_helper(main_queue) + @huey_helper.register_schedule def scheduled_find_discontinued_soon(): user = app.config.get("SYSTEM_USERNAME") @@ -114,4 +116,4 @@ def scheduled_find_discontinued_soon(): def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) - BackgroundApi.execute(task) \ No newline at end of file + BackgroundApi.execute(task) diff --git a/portality/ui/messages.py b/portality/ui/messages.py index aedd06ffdd..ac7f9163bc 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -111,8 +111,8 @@ class Messages(object): NOTIFY__DEFAULT_SHORT_NOTIFICATION = "You have a new notification" DISCONTINUED_JOURNAL_FOUND_LOG = "Journal discontinuing soon found: {id}" - DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG = "Email with journals discontinuing soon sent" - DISCONTINUED_JOURNALS_FOUND_EMAIL_ERROR_LOG = "Error sending email with journals discountinuing soon." + DISCONTINUED_JOURNALS_FOUND_NOTIFICATION_SENT_LOG = "Notification with journals discontinuing soon sent." + DISCONTINUED_JOURNALS_FOUND_NOTIFICATION_ERROR_LOG = "Error sending notification with journals discontinuing soon." NO_DISCONTINUED_JOURNALS_FOUND_LOG = "No journals discontinuing soon found" From 43633cc189c00fa4ea33ba550507372209e6c641 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Fri, 19 May 2023 16:25:45 +0100 Subject: [PATCH 091/466] Remove delta arg for find discontinued soon --- portality/tasks/find_discontinued_soon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 8ca7a2524a..379618dc0a 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -12,8 +12,8 @@ class DiscontinuedSoonQuery: - def __init__(self, time_delta=None): - self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0) + def __init__(self): + self._delta = app.config.get('DISCONTINUED_DATE_DELTA', 0) self._date = dates.days_after_now(days=self._delta) def query(self): @@ -37,9 +37,9 @@ def query(self): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" - def __init__(self, job, time_delta=None): + def __init__(self, job): super(FindDiscontinuedSoonBackgroundTask, self).__init__(job) - self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0) + self._delta = app.config.get('DISCONTINUED_DATE_DELTA', 0) self._date = dates.days_after_now(days=self._delta) def find_journals_discontinuing_soon(self): From e4df8f6f9838f4679d1c659aed289a450d15360f Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 5 Jun 2023 12:08:42 +0100 Subject: [PATCH 092/466] add blank_field_finder.py --- portality/lib/csv_utils.py | 18 +++++ portality/scripts/blank_field_finder.py | 87 +++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 portality/lib/csv_utils.py create mode 100644 portality/scripts/blank_field_finder.py diff --git a/portality/lib/csv_utils.py b/portality/lib/csv_utils.py new file mode 100644 index 0000000000..e37bc5de3a --- /dev/null +++ b/portality/lib/csv_utils.py @@ -0,0 +1,18 @@ +import csv +from typing import Iterable + + +def read_all(csv_path) -> Iterable[list]: + with open(csv_path, 'r') as f: + for row in csv.reader(f): + yield row + + +def read_all_as_dict(csv_path) -> Iterable[dict]: + headers = None + for row in read_all(csv_path): + if headers is None: + headers = row + continue + row = dict(zip(headers, row)) + yield row diff --git a/portality/scripts/blank_field_finder.py b/portality/scripts/blank_field_finder.py new file mode 100644 index 0000000000..c8ce3275ba --- /dev/null +++ b/portality/scripts/blank_field_finder.py @@ -0,0 +1,87 @@ +import argparse +from pathlib import Path +from typing import Any, Iterable + +from portality.bll.services.journal import JournalService +from portality.lib import csv_utils +from portality.models import Application, Journal + + +def to_k_v(item: Any, prefix: list = None): + if prefix is None: + prefix = [] + + if isinstance(item, dict): + for k, v in item.items(): + yield from to_k_v(v, prefix=prefix + [k]) + + elif isinstance(item, list): + for k, v in enumerate(item): + yield from to_k_v(v, prefix=prefix + [k]) + else: + yield '.'.join(map(str, prefix)), str(item) + + +def tee(txt: str, out_file): + print(txt) + out_file.write(txt + '\n') + + +def write_bad_data_domain_object(domain_object_class: Any, out_path): + with open(out_path, 'w') as f: + items = iter(domain_object_class.iterall()) + while True: + try: + j = next(items, None) + except: + continue + + if j is None: + break + + for k, v in filter_bad_only(to_k_v(j.data)): + tee(f'{j.id} {k} [{v}]', f) + + +def main2(): + with open('/tmp/journals.csv', 'w') as f: + JournalService._make_journals_csv(f) + + +def is_bad_str(v: str): + return isinstance(v, str) and v != v.strip() + + +def filter_bad_only(row: Iterable): + return (i for i in row if is_bad_str(i[1])) + + +def write_bad_data_journals_csv(csv_path, out_path): + with open(out_path, 'w') as out_file: + for row in csv_utils.read_all_as_dict(csv_path): + for k, v in filter_bad_only(row.items()): + tee(f'{k} [{v}]', out_file) + + +def write_results(journal_csv_path, out_dir): + # out_dir = Path('/tmp') + # journal_csv_path = '/home/kk/tmp/journals.csv' + out_dir = Path(out_dir) + write_bad_data_domain_object(Application, out_dir / 'bad_app.txt') + write_bad_data_domain_object(Journal, out_dir / 'bad_journals.txt') + if journal_csv_path: + write_bad_data_journals_csv(journal_csv_path, out_dir / 'bad_journals_csv.txt') + + +def main(): + parser = argparse.ArgumentParser(description='Output file with bad data') + parser.add_argument('-i', '--input', help='Path of input CSV file', type=str, default=None) + parser.add_argument('-o', '--output', help='Output directory', type=str, default='.') + args = parser.parse_args( + # ['-i', '/home/kk/tmp/journals.csv', '-o', '/tmp'] + ) + write_results(args.input, args.output) + + +if __name__ == '__main__': + main() From 978fac881ca853b8bd13ca0ab8fea694c4b29165 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 5 Jun 2023 12:17:24 +0100 Subject: [PATCH 093/466] format style --- portality/upgrade.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/portality/upgrade.py b/portality/upgrade.py index fb600977cc..0f3d2bf8c7 100644 --- a/portality/upgrade.py +++ b/portality/upgrade.py @@ -14,12 +14,12 @@ from portality.dao import ScrollTimeoutException MODELS = { - "journal": models.Journal, #~~->Journal:Model~~ - "article": models.Article, #~~->Article:Model~~ - "suggestion": models.Suggestion, #~~->Application:Model~~ + "journal": models.Journal, # ~~->Journal:Model~~ + "article": models.Article, # ~~->Article:Model~~ + "suggestion": models.Suggestion, # ~~->Application:Model~~ "application": models.Application, - "account": models.Account, #~~->Account:Model~~ - "background_job": models.BackgroundJob #~~->BackgroundJob:Model~~ + "account": models.Account, # ~~->Account:Model~~ + "background_job": models.BackgroundJob # ~~->BackgroundJob:Model~~ } @@ -54,7 +54,8 @@ def do_upgrade(definition, verbose, save_batches=None): # Iterate through all of the records in the model class try: - for result in model_class.iterate(q=tdef.get("query", default_query), keepalive=tdef.get("keepalive", "1m"), page_size=tdef.get("scroll_size", 1000), wrap=False): + for result in model_class.iterate(q=tdef.get("query", default_query), keepalive=tdef.get("keepalive", "1m"), + page_size=tdef.get("scroll_size", 1000), wrap=False): original = deepcopy(result) if tdef.get("init_with_model", True): @@ -83,7 +84,8 @@ def do_upgrade(definition, verbose, save_batches=None): result.prep() except AttributeError: if verbose: - print(tdef.get("type"), result.id, "has no prep method - no, pre-save preparation being done") + print(tdef.get("type"), result.id, + "has no prep method - no, pre-save preparation being done") pass data = result.data @@ -134,7 +136,8 @@ def do_upgrade(definition, verbose, save_batches=None): f.write(json.dumps(batch, indent=2)) print(dates.now(), "wrote batch to file {x}".format(x=fn)) - print(dates.now(), "scroll timed out / writing ", len(batch), "to", tdef.get("type"), ";", total, "of", max) + print(dates.now(), "scroll timed out / writing ", len(batch), "to", + tdef.get("type"), ";", total, "of", max) model_class.bulk(batch, action=action, req_timeout=120) batch = [] @@ -180,6 +183,7 @@ def recurse(context, c, o): if __name__ == "__main__": # ~~->Migrate:Script~~ import argparse + parser = argparse.ArgumentParser() parser.add_argument("-u", "--upgrade", help="path to upgrade definition") parser.add_argument("-v", "--verbose", action="store_true", help="verbose output to stdout during processing") From 0846ef57f4601ba1655096daa007aafb5d08e2fd Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 5 Jun 2023 12:36:38 +0100 Subject: [PATCH 094/466] add upgrade definition typed dict docs --- portality/upgrade.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/portality/upgrade.py b/portality/upgrade.py index 0f3d2bf8c7..202cc2c899 100644 --- a/portality/upgrade.py +++ b/portality/upgrade.py @@ -6,6 +6,8 @@ from datetime import datetime, timedelta from copy import deepcopy from collections import OrderedDict +from typing import TypedDict, List, Dict + from portality import models from portality.dao import ScrollTimeoutException from portality.lib import plugin, dates @@ -29,7 +31,37 @@ def upgrade_article(self, article): pass -def do_upgrade(definition, verbose, save_batches=None): +class UpgradeType(TypedDict): + type: str # name / key of the MODELS class + action: str + query: dict # ES query to use to find the records to upgrade + keepalive: str # ES keepalive time for the scroll, e.g. 10m + scroll_size: int # ES scroll size + + """ + python path of functions to run on the record + interface of the function should be: + my_function(instance: DomainObject | dict) -> DomainObject | dict + """ + functions: List[str] + init_with_model: bool # instance would be a DomainObject if True, otherwise a dict + + """ + tasks to run on the record + that will only work if init_with_model is True + + format of each task: + { function name of model : kwargs } + """ + tasks: List[Dict[str, dict]] + + +class Definition(TypedDict): + batch: int + types: List[UpgradeType] + + +def do_upgrade(definition: Definition, verbose, save_batches=None): # get the source and target es definitions # ~~->Elasticsearch:Technology~~ From a610bcb16224e9ee4795e288083dabda95cda991 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 5 Jun 2023 13:32:45 +0100 Subject: [PATCH 095/466] add 903_remove_blanks migration --- portality/migrate/903_remove_blanks/README.md | 13 ++++++++++++ .../migrate/903_remove_blanks/__init__.py | 0 .../migrate/903_remove_blanks/functions.py | 21 +++++++++++++++++++ .../migrate/903_remove_blanks/migrate.json | 21 +++++++++++++++++++ portality/upgrade.py | 13 ++++++++---- 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 portality/migrate/903_remove_blanks/README.md create mode 100644 portality/migrate/903_remove_blanks/__init__.py create mode 100644 portality/migrate/903_remove_blanks/functions.py create mode 100644 portality/migrate/903_remove_blanks/migrate.json diff --git a/portality/migrate/903_remove_blanks/README.md b/portality/migrate/903_remove_blanks/README.md new file mode 100644 index 0000000000..833c2c102f --- /dev/null +++ b/portality/migrate/903_remove_blanks/README.md @@ -0,0 +1,13 @@ +# Remove Blank + +remove blank from start or end of string in Journal and Application + +### Run +``` +python portality/upgrade.py -u portality/migrate/903_remove_blanks/migrate.json +``` + +### verify +``` +python -m portality.scripts.blank_field_finder +``` \ No newline at end of file diff --git a/portality/migrate/903_remove_blanks/__init__.py b/portality/migrate/903_remove_blanks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/migrate/903_remove_blanks/functions.py b/portality/migrate/903_remove_blanks/functions.py new file mode 100644 index 0000000000..f36c435314 --- /dev/null +++ b/portality/migrate/903_remove_blanks/functions.py @@ -0,0 +1,21 @@ +def remove_blanks(obj) -> dict: + if not isinstance(obj, dict): + return obj + + for k, v in obj.items(): + if isinstance(v, dict): + obj[k] = remove_blanks(v) + + elif isinstance(v, list): + if not v: + continue + if isinstance(v[0], dict): + obj[k] = [remove_blanks(item) for item in v] + elif isinstance(v[0], str): + obj[k] = [item.strip() for item in v] + + elif isinstance(v, str) and v != v.strip(): + print(f'remove blanks: {k} = [{v}]') + obj[k] = v.strip() + + return obj diff --git a/portality/migrate/903_remove_blanks/migrate.json b/portality/migrate/903_remove_blanks/migrate.json new file mode 100644 index 0000000000..39f9d10077 --- /dev/null +++ b/portality/migrate/903_remove_blanks/migrate.json @@ -0,0 +1,21 @@ +{ + "batch" : 10000, + "types": [ + { + "type" : "application", + "init_with_model" : false, + "keepalive" : "10m", + "functions" : [ + "portality.migrate.903_remove_blanks.functions.remove_blanks" + ] + }, + { + "type" : "journal", + "init_with_model" : false, + "keepalive" : "10m", + "functions" : [ + "portality.migrate.903_remove_blanks.functions.remove_blanks" + ] + } + ] +} \ No newline at end of file diff --git a/portality/upgrade.py b/portality/upgrade.py index 202cc2c899..1f33ba6140 100644 --- a/portality/upgrade.py +++ b/portality/upgrade.py @@ -33,10 +33,10 @@ def upgrade_article(self, article): class UpgradeType(TypedDict): type: str # name / key of the MODELS class - action: str + action: str # default is update query: dict # ES query to use to find the records to upgrade - keepalive: str # ES keepalive time for the scroll, e.g. 10m - scroll_size: int # ES scroll size + keepalive: str # ES keepalive time for the scroll, default 1m + scroll_size: int # ES scroll size, default 1000 """ python path of functions to run on the record @@ -44,7 +44,12 @@ class UpgradeType(TypedDict): my_function(instance: DomainObject | dict) -> DomainObject | dict """ functions: List[str] - init_with_model: bool # instance would be a DomainObject if True, otherwise a dict + + """ + instance would be a DomainObject if True, otherwise a dict + default is True + """ + init_with_model: bool # """ tasks to run on the record From 369285eb2832f9f6ba752ed528d511f69f25a46a Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 5 Jun 2023 16:14:08 +0100 Subject: [PATCH 096/466] fix test `test_04_reject_resubmit`, routes missing --- doajtest/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doajtest/helpers.py b/doajtest/helpers.py index f65c11f241..1f716bcf3b 100644 --- a/doajtest/helpers.py +++ b/doajtest/helpers.py @@ -120,6 +120,7 @@ class DoajTestCase(TestCase): @classmethod def setUpClass(cls) -> None: + import portality.app # noqa, needed to registing routes cls.originals = patch_config(app, { "STORE_IMPL": "portality.store.StoreLocal", "STORE_LOCAL_DIR": paths.rel2abs(__file__, "..", "tmp", "store", "main", cls.__name__.lower()), From c691831569c47a579103bb97e5b52e07ae6b0364 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 7 Jun 2023 12:41:24 +0100 Subject: [PATCH 097/466] change person notified to journal editor --- .../consumers/journal_discontinuing_soon_notify.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index 7f41dd8ba7..11da31bb96 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -29,18 +29,10 @@ def consume(cls, event): if journal is None: return - app_id = journal.current_application - if app_id is None: - app_id = journal.latest_related_application_id() - if app_id is None: - return - - application = models.Application.pull(app_id) - - if not application.editor_group: + if not journal.editor_group: return - eg = models.EditorGroup.pull_by_key("name", application.editor_group) + eg = models.EditorGroup.pull_by_key("name", journal.editor_group) managing_editor = eg.maned if not managing_editor: return From 60b72ed80ea7bf62393fe409efca834e3e82f850 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 8 Jun 2023 07:05:49 +0100 Subject: [PATCH 098/466] use csv.DictReader --- portality/lib/csv_utils.py | 17 ++++------------- portality/scripts/blank_field_finder.py | 2 +- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/portality/lib/csv_utils.py b/portality/lib/csv_utils.py index e37bc5de3a..c5a46f37fd 100644 --- a/portality/lib/csv_utils.py +++ b/portality/lib/csv_utils.py @@ -1,18 +1,9 @@ import csv -from typing import Iterable +from typing import Iterable, Union -def read_all(csv_path) -> Iterable[list]: +def read_all(csv_path, as_dict=False) -> Iterable[Union[list, dict]]: + reader = csv.DictReader if as_dict else csv.reader with open(csv_path, 'r') as f: - for row in csv.reader(f): + for row in reader(f): yield row - - -def read_all_as_dict(csv_path) -> Iterable[dict]: - headers = None - for row in read_all(csv_path): - if headers is None: - headers = row - continue - row = dict(zip(headers, row)) - yield row diff --git a/portality/scripts/blank_field_finder.py b/portality/scripts/blank_field_finder.py index c8ce3275ba..028332b1a6 100644 --- a/portality/scripts/blank_field_finder.py +++ b/portality/scripts/blank_field_finder.py @@ -58,7 +58,7 @@ def filter_bad_only(row: Iterable): def write_bad_data_journals_csv(csv_path, out_path): with open(out_path, 'w') as out_file: - for row in csv_utils.read_all_as_dict(csv_path): + for row in csv_utils.read_all(csv_path, as_dict=True): for k, v in filter_bad_only(row.items()): tee(f'{k} [{v}]', out_file) From 1d3e8bbdb84c96a22b4ee3faf368305ddf5c79a8 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 10:40:08 +0100 Subject: [PATCH 099/466] add check --- portality/bll/services/article.py | 4 ++++ portality/ui/messages.py | 1 + 2 files changed, 5 insertions(+) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 7b55894d24..1e0e2ac76d 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,6 +170,10 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) + journals = models.Journal.find_by_issn([pissn,eissn], True) + if journals is None or len(journals) > 1: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): diff --git a/portality/ui/messages.py b/portality/ui/messages.py index 094751c97c..cf8092e212 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -61,6 +61,7 @@ class Messages(object): EXCEPTION_NO_CONTRIBUTORS_EXPLANATION = "DOAJ requires at least one author for each article." EXCEPTION_TOO_MANY_ISSNS = "Too many ISSNs. Only 2 ISSNs are allowed: one Print ISSN and one Online ISSN." + EXCEPTION_MISMATCHED_ISSNS = "Issns provided don't match any journal." EXCEPTION_ISSNS_OF_THE_SAME_TYPE = "Both ISSNs have the same type: {type}" EXCEPTION_IDENTICAL_PISSN_AND_EISSN = "The Print and Online ISSNs supplied are identical. If you supply 2 ISSNs they must be different." EXCEPTION_NO_ISSNS = "Neither Print ISSN nor Online ISSN has been supplied. DOAJ requires at least one ISSN." From 83b6fded639bba06090b38c8569f81f5da819e00 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 14:01:38 +0100 Subject: [PATCH 100/466] add check and unit tests --- ...issn_validation_against_journal.matrix.csv | 17 +++ ...sn_validation_against_journal.settings.csv | 19 +++ ...n_validation_against_journal.settings.json | 119 ++++++++++++++++++ ...test_article_acceptable_and_permissions.py | 56 ++++++++- portality/bll/services/article.py | 10 +- 5 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv b/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv new file mode 100644 index 0000000000..0d2f704aba --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv @@ -0,0 +1,17 @@ +test_id,eissn,pissn,validated +1,eissn_in_doaj,pissn_in_doaj,yes +2,eissn_in_doaj,eissn_not_in_doaj, +3,eissn_in_doaj,pissn_not_in_doaj, +4,eissn_in_doaj,!eissn_in_doaj, +5,pissn_in_doaj,eissn_in_doaj, +6,pissn_in_doaj,eissn_not_in_doaj, +7,pissn_in_doaj,pissn_not_in_doaj, +8,pissn_in_doaj,!pissn_in_doaj, +9,eissn_not_in_doaj,eissn_in_doaj, +10,eissn_not_in_doaj,pissn_in_doaj, +11,eissn_not_in_doaj,pissn_not_in_doaj, +12,eissn_not_in_doaj,!eissn_not_in_doaj, +13,pissn_not_in_doaj,eissn_in_doaj, +14,pissn_not_in_doaj,pissn_in_doaj, +15,pissn_not_in_doaj,eissn_not_in_doaj, +16,pissn_not_in_doaj,!pissn_not_in_doaj, diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv new file mode 100644 index 0000000000..a8eab3f4ce --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv @@ -0,0 +1,19 @@ +field,test_id,eissn,pissn,validated +type,index,generated,generated,conditional +deafult,,,,no +,,,, +values,,eissn_in_doaj,eissn_in_doaj,yes +values,,pissn_in_doaj,pissn_in_doaj,no +values,,eissn_not_in_doaj,eissn_not_in_doaj, +values,,pissn_not_in_doaj,pissn_not_in_doaj, +,,,, +,,,, +conditional validated,,eissn_in_doaj,pissn_in_doaj,yes +constraint eissn,,eissn_in_doaj,!eissn_in_doaj, +constraint eissn,,eissn_not_in_doaj,!eissn_not_in_doaj, +constraint eissn,,pissn_not_in_doaj,!pissn_not_in_doaj, +constraint eissn,,pissn_in_doaj,!pissn_in_doaj, +constraint pissn,,eissn_in_doaj,!eissn_in_doaj, +constraint pissn,,eissn_not_in_doaj,!eissn_not_in_doaj, +constraint pissn,,pissn_not_in_doaj,!pissn_not_in_doaj, +constraint pissn,,pissn_in_doaj,!pissn_in_doaj, \ No newline at end of file diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json new file mode 100644 index 0000000000..11d1012a96 --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json @@ -0,0 +1,119 @@ +{ + "parameters": [ + { + "name": "test_id", + "type": "index" + }, + { + "name": "eissn", + "type": "generated", + "values": { + "eissn_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "eissn_in_doaj" + ] + } + } + }, + "pissn_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "pissn_in_doaj" + ] + } + } + }, + "eissn_not_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "eissn_not_in_doaj" + ] + } + } + }, + "pissn_not_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "pissn_not_in_doaj" + ] + } + } + } + } + }, + { + "name": "pissn", + "type": "generated", + "values": { + "eissn_in_doaj": {}, + "pissn_in_doaj": {}, + "eissn_not_in_doaj": {}, + "pissn_not_in_doaj": {}, + "!eissn_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "eissn_in_doaj" + ] + } + } + }, + "!eissn_not_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "eissn_not_in_doaj" + ] + } + } + }, + "!pissn_not_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "pissn_not_in_doaj" + ] + } + } + }, + "!pissn_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "pissn_in_doaj" + ] + } + } + } + } + }, + { + "name": "validated", + "type": "conditional", + "values": { + "yes": { + "conditions": [ + { + "eissn": { + "or": [ + "eissn_in_doaj" + ] + }, + "pissn": { + "or": [ + "pissn_in_doaj" + ] + } + } + ] + }, + "no": {} + } + } + ] +} \ No newline at end of file diff --git a/doajtest/unit/test_article_acceptable_and_permissions.py b/doajtest/unit/test_article_acceptable_and_permissions.py index eb4c04d4fb..881323ba92 100644 --- a/doajtest/unit/test_article_acceptable_and_permissions.py +++ b/doajtest/unit/test_article_acceptable_and_permissions.py @@ -14,6 +14,11 @@ def is_acceptable_load_cases(): "test_id", {"test_id": []}) +def issn_validation_against_journal_load_sets(): + return load_parameter_sets(rel2abs(__file__, "..", "matrices", "article_create_article"), "issn_validation_against_journal", + "test_id", + {"test_id": []}) + class TestBLLPrepareUpdatePublisher(DoajTestCase): @@ -110,4 +115,53 @@ def test_has_permissions(self): assert failed_result["unowned"].sort() == [pissn, eissn].sort() # assert failed_result == {'success': 0, 'fail': 1, 'update': 0, 'new': 0, 'shared': [], # 'unowned': [pissn, eissn], - # 'unmatched': []}, "received: {}".format(failed_result) \ No newline at end of file + # 'unmatched': []}, "received: {}".format(failed_result) + + + @parameterized.expand(issn_validation_against_journal_load_sets) + def test_issn_validation_against_journal_load_sets(self, value, kwargs): + kwpissn = kwargs.get("pissn") + kweissn = kwargs.get("eissn") + validated = kwargs.get("validated") + + js = JournalFixtureFactory.make_many_journal_sources(2) + journal_in_doaj = Journal(**js[0]) + journal_in_doaj.set_in_doaj(True) + journal_in_doaj.bibjson().pissn = "1111-1111" + journal_in_doaj.bibjson().eissn = "2222-2222" + journal_in_doaj.save(blocking=True) + + journal_not_in_doaj = Journal(**js[1]) + journal_not_in_doaj.set_in_doaj(False) + journal_not_in_doaj.bibjson().pissn = "3333-3333" + journal_not_in_doaj.bibjson().eissn = "4444-4444" + journal_not_in_doaj.save(blocking=True) + + if (kwpissn == "pissn_in_doaj"): + pissn = journal_in_doaj.bibjson().pissn + elif (kwpissn == "eissn_in_doaj"): + pissn = journal_in_doaj.bibjson().eissn + elif (kwpissn == "pissn_not_in_doaj"): + pissn = journal_not_in_doaj.bibjson().pissn + else: + pissn = journal_not_in_doaj.bibjson().eissn + + if (kweissn == "pissn_in_doaj"): + eissn = journal_in_doaj.bibjson().pissn + elif (kweissn == "eissn_in_doaj"): + eissn = journal_in_doaj.bibjson().eissn + elif (kweissn == "pissn_not_in_doaj"): + eissn = journal_not_in_doaj.bibjson().pissn + else: + eissn = journal_not_in_doaj.bibjson().eissn + + + art_source = ArticleFixtureFactory.make_article_source(pissn=pissn, eissn=eissn) + article = Article(**art_source) + + if validated: + self.assertIsNone(self.svc.is_acceptable(article)) + + else: + with self.assertRaises(exceptions.ArticleNotAcceptable): + self.svc.is_acceptable(article) \ No newline at end of file diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 1e0e2ac76d..5430b5f3ef 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,8 +170,14 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journals = models.Journal.find_by_issn([pissn,eissn], True) - if journals is None or len(journals) > 1: + journalp = models.Journal.find_by_issn([pissn], True) + journale = models.Journal.find_by_issn([eissn], True) + + # check if only one and the same journal matches pissn and eissn and if they are in the correct fields + if len(journalp) != 1 or \ + len(journale) != 1 or \ + journale[0].id != journalp[0].id or \ + journale[0].bibjson().pissn != pissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, From c5392bd7aeb2f6c9198c037062d52619bdea4900 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 14:05:12 +0100 Subject: [PATCH 101/466] one more unit test --- ...test_article_acceptable_and_permissions.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doajtest/unit/test_article_acceptable_and_permissions.py b/doajtest/unit/test_article_acceptable_and_permissions.py index 881323ba92..5e0328635f 100644 --- a/doajtest/unit/test_article_acceptable_and_permissions.py +++ b/doajtest/unit/test_article_acceptable_and_permissions.py @@ -164,4 +164,24 @@ def test_issn_validation_against_journal_load_sets(self, value, kwargs): else: with self.assertRaises(exceptions.ArticleNotAcceptable): - self.svc.is_acceptable(article) \ No newline at end of file + self.svc.is_acceptable(article) + + def test_check_validation_for_2_journals(self): + + js = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True) + journal_in_doaj = Journal(**js[0]) + journal_in_doaj.bibjson().pissn = "1111-1111" + journal_in_doaj.bibjson().eissn = "2222-2222" + journal_in_doaj.save(blocking=True) + + journal_not_in_doaj = Journal(**js[1]) + journal_not_in_doaj.bibjson().pissn = "3333-3333" + journal_not_in_doaj.bibjson().eissn = "4444-4444" + journal_not_in_doaj.save(blocking=True) + + + art_source = ArticleFixtureFactory.make_article_source(pissn="1111-1111", eissn="4444-4444") + article = Article(**art_source) + + with self.assertRaises(exceptions.ArticleNotAcceptable): + self.svc.is_acceptable(article) \ No newline at end of file From bca84c303821ac5323afbc2c2140acada6d04f1c Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 12:52:30 +0100 Subject: [PATCH 102/466] add new query and unit tests --- doajtest/unit/test_models.py | 27 ++++++++++++++++++++++++ portality/bll/services/article.py | 15 ++++++-------- portality/models/v2/journal.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index 09fb0d0205..1459fab2ab 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1641,3 +1641,30 @@ def test_get_name_safe(self): # account does not exist assert models.Account.get_name_safe('not existing account id') == '' + def test_11_find_by_issn(self): + js = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True) + j1 = models.Journal(**js[0]) + j1.bibjson().pissn = "1111-1111" + j1.bibjson().eissn = "2222-2222" + j1.save(blocking=True) + + j2 = models.Journal(**js[1]) + j2.bibjson().pissn = "3333-3333" + j2.bibjson().eissn = "4444-4444" + j2.save(blocking=True) + + journals = models.Journal.find_by_issn(["1111-1111", "2222-2222"], True) + assert len(journals) == 1 + assert journals[0].id == j1.id + + journals = models.Journal.find_by_issn(["1111-1111", "3333-3333"], True) + assert len(journals) == 2 + assert journals[0].id == j1.id + assert journals[1].id == j2.id + + journals = models.Journal.find_by_issn_exact(["1111-1111", "2222-2222"], True) + assert len(journals) == 1 + assert journals[0].id == j1.id + + journals = models.Journal.find_by_issn_exact(["1111-1111", "3333-3333"], True) + assert len(journals) == 0 \ No newline at end of file diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 5430b5f3ef..d9fb6701e9 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,20 +170,17 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journalp = models.Journal.find_by_issn([pissn], True) - journale = models.Journal.find_by_issn([eissn], True) - - # check if only one and the same journal matches pissn and eissn and if they are in the correct fields - if len(journalp) != 1 or \ - len(journale) != 1 or \ - journale[0].id != journalp[0].id or \ - journale[0].bibjson().pissn != pissn: + journal = models.Journal.find_by_issn_exact([pissn,eissn], True) + + # check if only one journal matches pissn and eissn and if they are in the correct fields + # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too + if len(journal) != 1 or journal[0].bibjson().pissn != pissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): - """ + """# no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too Create an individual article in the database This method will check and merge any duplicates, and report back on successes and failures in a manner consistent with diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 41c50a7ce9..5a4997a0de 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -70,6 +70,22 @@ def find_by_issn(cls, issns, in_doaj=None, max=10): records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] return records + @classmethod + def find_by_issn_exact(cls, issns, in_doaj=None, max=2): + """ + Finds journal that matches given issns exactly - if no data problems should always be only 1 + """ + if not isinstance(issns, list): + issns = [issns] + if len(issns) > 2: + return [] + q = JournalQuery() + q.find_by_issn_exact(issns, in_doaj=in_doaj, max=max) + result = cls.query(q=q.query) + # create an array of objects, using cls rather than Journal, which means subclasses can use it too + records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] + return records + @classmethod def issns_by_owner(cls, owner, in_doaj=None): q = IssnQuery(owner, in_doaj=in_doaj) @@ -922,6 +938,16 @@ class JournalQuery(object): } } + must_query = { + "track_total_hits": True, + "query": { + "bool": { + "must": [ + ] + } + } + } + all_doaj = { "track_total_hits": True, "query": { @@ -947,6 +973,14 @@ def find_by_issn(self, issns, in_doaj=None, max=10): self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}}) self.query["size"] = max + def find_by_issn_exact(self, issns, in_doaj=None, max=10): + self.query = deepcopy(self.must_query) + for issn in issns: + self.query["query"]["bool"]["must"].append({"term": {"index.issn.exact": issn}}) + if in_doaj is not None: + self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}}) + self.query["size"] = max + def all_in_doaj(self): q = deepcopy(self.all_doaj) if self.minified: From 0122262cd79df3e1c803812114a1a8c81541253a Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 13:32:25 +0100 Subject: [PATCH 103/466] remove duplicated code --- portality/bll/services/article.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index d9fb6701e9..feebf2b481 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -159,9 +159,6 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if len(pissn) > 1 or len(eissn) > 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_TOO_MANY_ISSNS) - pissn = article_bibjson.get_one_identifier("pissn") - eissn = article_bibjson.get_one_identifier("eissn") - # no pissn or eissn if not pissn and not eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) From e0db3f581df83e5cc958f96865d073f05e5b11c3 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:21:06 +0100 Subject: [PATCH 104/466] add script to find all articles with invalid issns --- ...230609_find_articles_with_invalid_issns.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 portality/scripts/230609_find_articles_with_invalid_issns.py diff --git a/portality/scripts/230609_find_articles_with_invalid_issns.py b/portality/scripts/230609_find_articles_with_invalid_issns.py new file mode 100644 index 0000000000..acebc0ae7a --- /dev/null +++ b/portality/scripts/230609_find_articles_with_invalid_issns.py @@ -0,0 +1,68 @@ +from portality import models +from portality.bll.services import article as articlesvc +from portality.bll import exceptions +from portality.core import es_connection +from portality.util import ipt_prefix +import esprit +import csv + +IN_DOAJ = { + "query": { + "bool": { + "must": [ + {"term" : {"admin.in_doaj":True}} + ] + } + } +} + + +if __name__ == "__main__": + + # import argparse + # + # parser = argparse.ArgumentParser() + # parser.add_argument("-o", "--out", help="output file path") + # args = parser.parse_args() + # + # if not args.out: + # print("Please specify an output file path with the -o option") + # parser.print_help() + # exit() + + out = "out.csv" + + with open(out, "w", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["ID", "PISSN", "EISSN", "Journals found with article's PISSN", "In doaj?", "Journals found with article's EISSN", "In doaj?", "Error"]) + + for a in models.Article.iterate(q=IN_DOAJ, page_size=100, keepalive='5m'): + print(a["id"]) + article = models.Article(_source=a) + bibjson = article.bibjson() + + try: + articlesvc.ArticleService._validate_issns(bibjson) + except exceptions.ArticleNotAcceptable as e: + id = article.id + pissn = bibjson.get_identifiers("pissn") + eissn = bibjson.get_identifiers("eissn") + j_p = [j["id"] for j in models.Journal.find_by_issn(pissn)] + j_p_in_doaj = [] + if (j_p): + for j in j_p: + jobj = models.Journal.pull(j) + if (jobj): + j_p_in_doaj.append(jobj.is_in_doaj()) + else: + j_p_in_doaj.append("n/a") + j_e = [j["id"] for j in models.Journal.find_by_issn(eissn)] + j_e_in_doaj = [] + if (j_e): + for j in j_e: + jobj = models.Journal.pull(j) + if (jobj): + j_e_in_doaj.append(jobj.is_in_doaj()) + else: + j_e_in_doaj.append("n/a") + writer.writerow([id, pissn, eissn, j_p, j_p_in_doaj, j_e, j_e_in_doaj, str(e)]) \ No newline at end of file From 3911cff2657ede8ddd966898ca287cb990569308 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:25:01 +0100 Subject: [PATCH 105/466] revert mistakenly removed code --- portality/bll/services/article.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index feebf2b481..17f872582d 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -159,6 +159,9 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if len(pissn) > 1 or len(eissn) > 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_TOO_MANY_ISSNS) + pissn = article_bibjson.get_one_identifier("pissn") + eissn = article_bibjson.get_one_identifier("eissn") + # no pissn or eissn if not pissn and not eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) @@ -167,12 +170,24 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journal = models.Journal.find_by_issn_exact([pissn,eissn], True) + issns = [] + if pissn is not None: + issns.append(pissn) + if eissn is not None: + issns.append(eissn) + + journal = models.Journal.find_by_issn_exact(issns, True) # check if only one journal matches pissn and eissn and if they are in the correct fields # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too - if len(journal) != 1 or journal[0].bibjson().pissn != pissn: + if len(journal) != 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if pissn is not None: + if journal[0].bibjson().pissn != pissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if eissn is not None: + if journal[0].bibjson().eissn != eissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): From 27878361116f77a3ae1e2656b599001aa522006b Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:29:15 +0100 Subject: [PATCH 106/466] remove unnecessary print --- portality/scripts/230609_find_articles_with_invalid_issns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/portality/scripts/230609_find_articles_with_invalid_issns.py b/portality/scripts/230609_find_articles_with_invalid_issns.py index acebc0ae7a..c45b8c5d63 100644 --- a/portality/scripts/230609_find_articles_with_invalid_issns.py +++ b/portality/scripts/230609_find_articles_with_invalid_issns.py @@ -37,10 +37,8 @@ writer.writerow(["ID", "PISSN", "EISSN", "Journals found with article's PISSN", "In doaj?", "Journals found with article's EISSN", "In doaj?", "Error"]) for a in models.Article.iterate(q=IN_DOAJ, page_size=100, keepalive='5m'): - print(a["id"]) article = models.Article(_source=a) bibjson = article.bibjson() - try: articlesvc.ArticleService._validate_issns(bibjson) except exceptions.ArticleNotAcceptable as e: From 2fbcf73c9122ece6148d095efb9767e7d7158bf8 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 12 Jun 2023 13:25:32 +0100 Subject: [PATCH 107/466] make expandable facets better optimised for screen readers --- cms/sass/components/_accordion.scss | 3 +++ cms/sass/components/_buttons.scss | 7 +++++++ cms/sass/main.scss | 1 + portality/static/js/doaj.fieldrender.edges.js | 18 +++++++++--------- 4 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 cms/sass/components/_accordion.scss diff --git a/cms/sass/components/_accordion.scss b/cms/sass/components/_accordion.scss new file mode 100644 index 0000000000..e066ee02f6 --- /dev/null +++ b/cms/sass/components/_accordion.scss @@ -0,0 +1,3 @@ +.accordion:focus-within { + border: $grapefruit solid; +} \ No newline at end of file diff --git a/cms/sass/components/_buttons.scss b/cms/sass/components/_buttons.scss index 1e71d3aceb..061c75c454 100644 --- a/cms/sass/components/_buttons.scss +++ b/cms/sass/components/_buttons.scss @@ -117,3 +117,10 @@ button[type="submit"].button--secondary { color: currentColor; } } + +button.aria-button { + all: inherit; + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; +} diff --git a/cms/sass/main.scss b/cms/sass/main.scss index cdd22133b8..9b30911f48 100644 --- a/cms/sass/main.scss +++ b/cms/sass/main.scss @@ -28,6 +28,7 @@ "layout/sidenav", "components/alert", + "components/accordion", "components/back-to-top", "components/buttons", "components/card", diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 01f5589f2f..d2afa72b07 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -645,13 +645,13 @@ $.extend(true, doaj, { toggle = ''; } var placeholder = 'Search ' + this.component.nodeCount + ' subjects'; - var frag = '

    ' + this.title + toggle + '

    \ - '; // substitute in the component parts frag = frag.replace(/{{FILTERS}}/g, treeFrag); @@ -1833,10 +1833,10 @@ $.extend(true, doaj, { if (this.togglable) { toggle = ''; } - var frag = '

    ' + this.component.display + toggle + '

    \ - '; // substitute in the component parts frag = frag.replace(/{{FILTERS}}/g, filterFrag + results); @@ -2084,10 +2084,10 @@ $.extend(true, doaj, { if (this.togglable) { toggle = ''; } - var frag = '

    ' + this.component.display + toggle + '

    \ - '; // substitute in the component parts frag = frag.replace(/{{FILTERS}}/g, filterFrag + results); From 24e4c14c3a2ffb6c7afacfe349c1bb0509a741b4 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 12 Jun 2023 14:10:58 +0100 Subject: [PATCH 108/466] add functional test --- doajtest/testbook/public_site/public_search.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doajtest/testbook/public_site/public_search.yml b/doajtest/testbook/public_site/public_search.yml index 1bce101f8c..bc72ad24f9 100644 --- a/doajtest/testbook/public_site/public_search.yml +++ b/doajtest/testbook/public_site/public_search.yml @@ -166,3 +166,16 @@ tests: results: - You are taken to the full text of this article on the Web. It opens in a new tab +- title: 'Test Public Search Results Display: Accessibility' + context: + role: anonymous + steps: + - step: Go to the DOAJ search page at /search/articles + - step: Turn on a screen reader + results: + - Extendable facets are focusable and focus is marked with an orange solid border + - The screenreader gives the header role ("button") + - The screenreader gives the state of the facet ("extended" or "folded") + - step: click spacebar to fold/unfold the facet + resuts: + - screenreader gives correct state of the facet ("extended" or "folded") From 5b449b194efccc329e95d1c837d9820d015ff6c7 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 14 Jun 2023 12:01:49 +0100 Subject: [PATCH 109/466] skip link added to the public pages --- portality/templates/account/forgot.html | 4 ++-- portality/templates/account/login.html | 4 ++-- portality/templates/account/login_to_apply.html | 4 ++-- portality/templates/account/register.html | 4 ++-- portality/templates/account/reset.html | 4 ++-- portality/templates/api/current/api_docs.html | 4 ++-- portality/templates/application_form/public_application.html | 4 ++-- portality/templates/application_form/readonly_journal.html | 4 ++-- portality/templates/doaj/article.html | 4 ++-- portality/templates/doaj/articles_search.html | 4 ++-- portality/templates/doaj/contact.html | 4 ++-- portality/templates/doaj/index.html | 4 ++-- portality/templates/doaj/journals_search.html | 4 ++-- portality/templates/doaj/toc.html | 4 ++-- portality/templates/editor/editor_base.html | 4 ++-- portality/templates/layouts/public_base.html | 5 ++++- portality/templates/layouts/single_col_page.html | 4 ++-- portality/templates/openurl/404.html | 4 ++-- portality/templates/openurl/help.html | 4 ++-- portality/templates/publisher/publisher_base.html | 4 ++-- 20 files changed, 42 insertions(+), 39 deletions(-) diff --git a/portality/templates/account/forgot.html b/portality/templates/account/forgot.html index 241525adfd..d8f5e9c837 100644 --- a/portality/templates/account/forgot.html +++ b/portality/templates/account/forgot.html @@ -3,7 +3,7 @@ {% block page_title %}Reset your password{% endblock %} {% block content %} -
    +
    @@ -23,5 +23,5 @@

    Reset your password

    -
    + {% endblock %} diff --git a/portality/templates/account/login.html b/portality/templates/account/login.html index 247149641a..726831e9e3 100644 --- a/portality/templates/account/login.html +++ b/portality/templates/account/login.html @@ -3,7 +3,7 @@ {% block page_title %}Login to your account{% endblock %} {% block content %} -
    +
    @@ -17,5 +17,5 @@

    Login

    -
    + {% endblock %} diff --git a/portality/templates/account/login_to_apply.html b/portality/templates/account/login_to_apply.html index c701e9e024..33ac5666fd 100644 --- a/portality/templates/account/login_to_apply.html +++ b/portality/templates/account/login_to_apply.html @@ -3,7 +3,7 @@ {% block page_title %}Login to apply{% endblock %} {% block content %} -
    +
    @@ -46,5 +46,5 @@

    Related help

    -
    + {% endblock %} diff --git a/portality/templates/account/register.html b/portality/templates/account/register.html index 4f91d8250a..a497bd396a 100644 --- a/portality/templates/account/register.html +++ b/portality/templates/account/register.html @@ -12,7 +12,7 @@ {% endblock %} {% block content %} -
    +
    @@ -30,7 +30,7 @@

    Register

    -
    + {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/account/reset.html b/portality/templates/account/reset.html index 2b459de104..fdacc27620 100644 --- a/portality/templates/account/reset.html +++ b/portality/templates/account/reset.html @@ -4,7 +4,7 @@ {% block content %} -
    +
    @@ -20,6 +20,6 @@

    Hi {{ account.name or account.email }}

    -
    + {% endblock %} diff --git a/portality/templates/api/current/api_docs.html b/portality/templates/api/current/api_docs.html index 9c5a2bc8e5..42d3d14588 100644 --- a/portality/templates/api/current/api_docs.html +++ b/portality/templates/api/current/api_docs.html @@ -14,7 +14,7 @@ {% endblock %} {% block content %} -
    +
    {# todo: this nav was bumping into swagger @@ -58,7 +58,7 @@

    API

    -
    + {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/application_form/public_application.html b/portality/templates/application_form/public_application.html index af0e05d63e..31439b94d3 100644 --- a/portality/templates/application_form/public_application.html +++ b/portality/templates/application_form/public_application.html @@ -23,7 +23,7 @@ {% block content scoped %} -
    +
    {% include "application_form/_backend_validation.html" %}
    @@ -64,7 +64,7 @@
    -
    + {% endblock %} diff --git a/portality/templates/application_form/readonly_journal.html b/portality/templates/application_form/readonly_journal.html index 2fed49db61..fe429eea08 100644 --- a/portality/templates/application_form/readonly_journal.html +++ b/portality/templates/application_form/readonly_journal.html @@ -20,7 +20,7 @@ {% block content scoped %} -
    +
    @@ -47,7 +47,7 @@
    -
    + {% endblock %} diff --git a/portality/templates/doaj/article.html b/portality/templates/doaj/article.html index 67720a8585..91f3f80200 100644 --- a/portality/templates/doaj/article.html +++ b/portality/templates/doaj/article.html @@ -67,7 +67,7 @@ {% set doi = bibjson.get_one_identifier("doi") %} {% set normalised_doi = article.get_normalised_doi() %} -
    +

    @@ -224,5 +224,5 @@

    Published in {{jtitle}}

    -
    + {% endblock %} diff --git a/portality/templates/doaj/articles_search.html b/portality/templates/doaj/articles_search.html index cea8977897..4a518fb3ae 100644 --- a/portality/templates/doaj/articles_search.html +++ b/portality/templates/doaj/articles_search.html @@ -10,10 +10,10 @@ {%- block meta_twitter_description -%}Find open access articles in DOAJ.{%- endblock -%} {% block content %} -
    +
    {% include "includes/search-help-modal.html" %} -
    + {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/doaj/contact.html b/portality/templates/doaj/contact.html index 2a4739dec8..2ffb86985f 100644 --- a/portality/templates/doaj/contact.html +++ b/portality/templates/doaj/contact.html @@ -1,7 +1,7 @@ {% extends "layouts/public_base.html" %} {% block content %} -
    +

    Submit your feedback and questions here. Feedback submitted about a particular journal is treated as confidential.

    @@ -52,7 +52,7 @@
    -
    + {% endblock %} diff --git a/portality/templates/doaj/index.html b/portality/templates/doaj/index.html index c1599f238a..43b11960c9 100644 --- a/portality/templates/doaj/index.html +++ b/portality/templates/doaj/index.html @@ -74,7 +74,7 @@

    DOAJ in numbers

    {% endblock %} {% block content %} -
    +
    @@ -242,6 +242,6 @@

    Recently-added journals

    -
    + {% endblock %} diff --git a/portality/templates/doaj/journals_search.html b/portality/templates/doaj/journals_search.html index e38bedd18e..8eef0d2c63 100644 --- a/portality/templates/doaj/journals_search.html +++ b/portality/templates/doaj/journals_search.html @@ -10,10 +10,10 @@ {%- block meta_twitter_description -%}Find open access journals in DOAJ.{%- endblock -%} {% block content %} -
    +
    {% include "includes/search-help-modal.html" %} -
    + {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/doaj/toc.html b/portality/templates/doaj/toc.html index acf0077a7e..9d5302bc64 100644 --- a/portality/templates/doaj/toc.html +++ b/portality/templates/doaj/toc.html @@ -42,7 +42,7 @@ } %} -
    +
    {% if journal.last_manually_updated_since(days=30) %} @@ -450,7 +450,7 @@

    Journal metadata

    -
    + {% include "includes/_hotjar.html" %} {% endblock %} diff --git a/portality/templates/editor/editor_base.html b/portality/templates/editor/editor_base.html index 3d488e6e72..19a36ca7c7 100644 --- a/portality/templates/editor/editor_base.html +++ b/portality/templates/editor/editor_base.html @@ -7,7 +7,7 @@ {% block page_title %}Editor dashboard{% endblock %} {% block content %} -
    +

    Editor dashboard

    {% include 'editor/nav.html' %} @@ -15,7 +15,7 @@

    Editor dashboard

    {% block editor_content %} {% endblock %} -
    + {% include "includes/_hotjar.html" %} {% endblock %} diff --git a/portality/templates/layouts/public_base.html b/portality/templates/layouts/public_base.html index 2e4cad9ddc..ef0a3ae4f0 100644 --- a/portality/templates/layouts/public_base.html +++ b/portality/templates/layouts/public_base.html @@ -2,6 +2,7 @@ {% block base_content %} +Skip to main content