From 3613ab278ec7b265a71a4faaa84bddeea325e047 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 18 Jul 2024 22:41:05 +0100 Subject: [PATCH 01/17] initial implementation (pending unit tests) --- .../testdrive/todo_maned_editor_associate.py | 13 +++++- portality/bll/services/todo.py | 44 ++++++++++++++++--- portality/constants.py | 1 + portality/templates/dashboard/_todo.html | 5 +++ portality/templates/dashboard/index.html | 6 +++ portality/view/dashboard.py | 10 ++++- 6 files changed, 69 insertions(+), 10 deletions(-) diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py index 1fa8ff936e..0ca722c55e 100644 --- a/doajtest/testdrive/todo_maned_editor_associate.py +++ b/doajtest/testdrive/todo_maned_editor_associate.py @@ -142,6 +142,14 @@ def build_maned_applications(un, eg, owner, eponymous_group): "title": un + " Maned Pending Application" }] + app = build_application(un + " Maned On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, + editor_group=eg.name, owner=owner) + app.save() + apps["on_hold"] = [{ + "id": app.id, + "title": un + " Maned On Hold Application" + }] + app = build_application(un + " Maned Low Priority Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, editor_group=eponymous_group.name, owner=owner) @@ -154,11 +162,11 @@ def build_maned_applications(un, eg, owner, eponymous_group): lmur = build_application(un + " Last Month Maned Update Request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, editor_group=eponymous_group.name, owner=owner, update_request=True) - lmur.save() + # lmur.save() tmur = build_application(un + " This Month Maned Update Request", 0, 0, constants.APPLICATION_STATUS_UPDATE_REQUEST, editor_group=eponymous_group.name, owner=owner, update_request=True) - tmur.save() + # tmur.save() apps["update_request"] = [ { @@ -183,6 +191,7 @@ def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_grou if update_request: ap.application_type = constants.APPLICATION_TYPE_UPDATE_REQUEST + ap.set_current_journal(ap.makeid()) else: ap.remove_current_journal() ap.remove_related_journal() diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index fc57f66da7..258094b0c1 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -5,6 +5,7 @@ from portality.lib import dates from datetime import datetime + class TodoService(object): """ ~~Todo:Service->DOAJ:Service~~ @@ -63,8 +64,7 @@ def group_stats(self, group_id): return stats - - def top_todo(self, account, size=25, new_applications=True, update_requests=True): + def top_todo(self, account, size=25, new_applications=True, update_requests=True, on_hold=True): """ Returns the top number of todo items for a given user @@ -89,6 +89,8 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True if update_requests: queries.append(TodoRules.maned_last_month_update_requests(size, maned_of)) queries.append(TodoRules.maned_new_update_requests(size, maned_of)) + if on_hold: + queries.append(TodoRules.maned_on_hold(size, maned_of)) if new_applications: # editor and associate editor roles only deal with new applications if account.has_role("editor"): @@ -174,7 +176,11 @@ def maned_stalled(cls, size, maned_of): TodoQuery.is_new_application() ], must_nots=[ - TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_ON_HOLD + ]) ], sort=sort_date, size=size @@ -191,7 +197,11 @@ def maned_follow_up_old(cls, size, maned_of): TodoQuery.is_new_application() ], must_nots=[ - TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_ON_HOLD + ]) ], sort=sort_date, size=size @@ -262,7 +272,11 @@ def maned_last_month_update_requests(cls, size, maned_of): TodoQuery.is_update_request() ], must_nots=[ - TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_ON_HOLD + ]) # TodoQuery.exists("admin.editor") ], sort=sort_date, @@ -282,7 +296,11 @@ def maned_new_update_requests(cls, size, maned_of): TodoQuery.is_update_request() ], must_nots=[ - TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + TodoQuery.status([ + constants.APPLICATION_STATUS_ACCEPTED, + constants.APPLICATION_STATUS_REJECTED, + constants.APPLICATION_STATUS_ON_HOLD + ]) # TodoQuery.exists("admin.editor") ], sort=sort_date, @@ -290,6 +308,20 @@ def maned_new_update_requests(cls, size, maned_of): ) return constants.TODO_MANED_NEW_UPDATE_REQUEST, assign_pending, sort_date, -2 + @classmethod + def maned_on_hold(cls, size, maned_of): + sort_date = "created_date" + on_holds = TodoQuery( + musts=[ + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application(), + TodoQuery.status([constants.APPLICATION_STATUS_ON_HOLD]) + ], + sort=sort_date, + size=size + ) + return constants.TODO_MANED_ON_HOLD, on_holds, sort_date, 0 + @classmethod def editor_stalled(cls, groups, size): sort_date = "created_date" diff --git a/portality/constants.py b/portality/constants.py index 2fec308772..991b1ec129 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -50,6 +50,7 @@ TODO_MANED_ASSIGN_PENDING = "todo_maned_assign_pending" TODO_MANED_LAST_MONTH_UPDATE_REQUEST = "todo_maned_last_month_update_request" TODO_MANED_NEW_UPDATE_REQUEST = "todo_maned_new_update_request" +TODO_MANED_ON_HOLD = "todo_maned_on_hold" TODO_EDITOR_STALLED = "todo_editor_stalled" TODO_EDITOR_FOLLOW_UP_OLD = "todo_editor_follow_up_old" TODO_EDITOR_COMPLETED = "todo_editor_completed" diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html index 069ba89633..7303003eae 100644 --- a/portality/templates/dashboard/_todo.html +++ b/portality/templates/dashboard/_todo.html @@ -41,6 +41,11 @@ "feather": "edit", "show_status": true }, + constants.TODO_MANED_ON_HOLD: { + "text" : "On Hold Application Review status", + "colour" : "var(--sanguine)", + "feather": "x-circle" + }, constants.TODO_EDITOR_STALLED: { "text" : "Stalled Chase Associate Editor", "show_status": true, diff --git a/portality/templates/dashboard/index.html b/portality/templates/dashboard/index.html index 99459988bd..c2ba80a212 100644 --- a/portality/templates/dashboard/index.html +++ b/portality/templates/dashboard/index.html @@ -21,6 +21,12 @@ {% else %} Update Requests {% endif %} + + {% if request.values.get("filter") == "oh" %} + On Hold + {% else %} + On Hold + {% endif %} {% include "dashboard/_todo.html" %}
diff --git a/portality/view/dashboard.py b/portality/view/dashboard.py index 2f63949a1b..5637cc7263 100644 --- a/portality/view/dashboard.py +++ b/portality/view/dashboard.py @@ -19,10 +19,15 @@ @ssl_required def top_todo(): filter = request.values.get("filter") - new_applications, update_requests = True, True + new_applications, update_requests, on_hold = True, True, True if filter == "na": + on_hold = False update_requests = False elif filter == "ur": + on_hold = False + new_applications = False + elif filter == "oh": + update_requests = False new_applications = False # ~~-> Todo:Service~~ @@ -30,7 +35,8 @@ def top_todo(): todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE"), new_applications=new_applications, - update_requests=update_requests) + update_requests=update_requests, + on_hold=on_hold) # ~~-> Dashboard:Page~~ return render_template('dashboard/index.html', todos=todos) From e7a270979fcf7e4731067999428a2d850ed63655 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 19 Jul 2024 08:42:00 +0100 Subject: [PATCH 02/17] add unit testing for on hold dashboard --- .../bll_todo_maned/top_todo_maned.matrix.csv | 12 +-- .../top_todo_maned.settings.csv | 76 ++++++++++--------- .../top_todo_maned.settings.json | 47 ++++++++++++ doajtest/unit/test_bll_todo_top_todo_maned.py | 6 +- 4 files changed, 98 insertions(+), 43 deletions(-) diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv index 9965c80495..0e0a97573f 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv @@ -1,6 +1,6 @@ -test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order -1,none,ArgumentException,0,0,0,0,0,0,,,,,, -2,no_role,,0,0,0,0,0,0,,,,,, -3,admin,,1,1,1,1,1,1,1,2,3,4,5,6 -4,editor,,0,0,0,0,0,0,,,,,, -5,assed,,0,0,0,0,0,0,,,,,, +test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_on_hold,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order,todo_maned_on_hold_order +1,none,ArgumentException,0,0,0,0,0,0,0,,,,,,, +2,no_role,,0,0,0,0,0,0,0,,,,,,, +3,admin,,1,1,1,1,1,1,1,1,2,3,4,5,6,7 +4,editor,,0,0,0,0,0,0,0,,,,,,, +5,assed,,0,0,0,0,0,0,0,,,,,,, diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv index a8148032f9..3d0f77656f 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv @@ -1,36 +1,40 @@ -field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order -type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional -default,,,,,,,,,,,,,,, -,,,,,,,,,,,,,,, -values,,none,ArgumentException,,,,,,,,,,,, -values,,no_role,,,,,,,,,,,,, -values,,admin,,,,,,,,,,,,, -values,,editor,,,,,,,,,,,,, -values,,assed,,,,,,,,,,,,, -,,,,,,,,,,,,,,, -conditional raises,,none,ArgumentException,,,,,,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_stalled,,admin,,1,,,,,,,,,,, -conditional todo_maned_stalled,,!admin,,0,,,,,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,,,, -conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_ready,,admin,,,,1,,,,,,,,, -conditional todo_maned_ready,,!admin,,,,0,,,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_completed,,admin,,,,,1,,,,,,,, -conditional todo_maned_completed,,!admin,,,,,0,,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_assign_pending,,admin,,,,,,1,,,,,,, -conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_new_update_request,,admin,,,,,,,1,,,,,, -conditional todo_maned_new_update_request,,!admin,,,,,,,0,,,,,, -,,,,,,,,,,,,,,, -conditional todo_maned_new_update_request_order,,admin,,,,,,,,1,,,,, -conditional todo_maned_ready_order,,admin,,,,,,,,,2,,,, -conditional todo_maned_follow_up_old_order,,admin,,,,,,,,,,3,,, -conditional todo_maned_stalled_order,,admin,,,,,,,,,,,4,, -conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,,,5, -conditional todo_maned_completed_order,,admin,,,,,,,,,,,,,6 \ No newline at end of file +field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_on_hold,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order,todo_maned_on_hold_order +type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional +default,,,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +values,,none,ArgumentException,,,,,,,,,,,,,, +values,,no_role,,,,,,,,,,,,,,, +values,,admin,,,,,,,,,,,,,,, +values,,editor,,,,,,,,,,,,,,, +values,,assed,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional raises,,none,ArgumentException,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_stalled,,admin,,1,,,,,,,,,,,,, +conditional todo_maned_stalled,,!admin,,0,,,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,,,,,, +conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_ready,,admin,,,,1,,,,,,,,,,, +conditional todo_maned_ready,,!admin,,,,0,,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_completed,,admin,,,,,1,,,,,,,,,, +conditional todo_maned_completed,,!admin,,,,,0,,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_assign_pending,,admin,,,,,,1,,,,,,,,, +conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_new_update_request,,admin,,,,,,,1,,,,,,,, +conditional todo_maned_new_update_request,,!admin,,,,,,,0,,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_on_hold,,admin,,,,,,,,1,,,,,,, +conditional todo_maned_on_hold,,!admin,,,,,,,,0,,,,,,, +,,,,,,,,,,,,,,,,, +conditional todo_maned_new_update_request_order,,admin,,,,,,,,,1,,,,,, +conditional todo_maned_ready_order,,admin,,,,,,,,,,2,,,,, +conditional todo_maned_follow_up_old_order,,admin,,,,,,,,,,,3,,,, +conditional todo_maned_stalled_order,,admin,,,,,,,,,,,,4,,, +conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,,,,5,, +conditional todo_maned_completed_order,,admin,,,,,,,,,,,,,,6, +conditional todo_maned_on_hold_order,,admin,,,,,,,,,,,,,,,7 \ No newline at end of file diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json index 6625f298f2..a7ef53dc1b 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json @@ -207,6 +207,35 @@ } } }, + { + "name": "todo_maned_on_hold", + "type": "conditional", + "default": "", + "values": { + "1": { + "conditions": [ + { + "account": { + "or": [ + "admin" + ] + } + } + ] + }, + "0": { + "conditions": [ + { + "account": { + "nor": [ + "admin" + ] + } + } + ] + } + } + }, { "name": "todo_maned_new_update_request_order", "type": "conditional", @@ -314,6 +343,24 @@ ] } } + }, + { + "name": "todo_maned_on_hold_order", + "type": "conditional", + "default": "", + "values": { + "7": { + "conditions": [ + { + "account": { + "or": [ + "admin" + ] + } + } + ] + } + } } ] } \ No newline at end of file diff --git a/doajtest/unit/test_bll_todo_top_todo_maned.py b/doajtest/unit/test_bll_todo_top_todo_maned.py index 5322a8c9ae..1c5ae4457d 100644 --- a/doajtest/unit/test_bll_todo_top_todo_maned.py +++ b/doajtest/unit/test_bll_todo_top_todo_maned.py @@ -41,7 +41,8 @@ def test_top_todo(self, name, kwargs): "todo_maned_ready", "todo_maned_completed", "todo_maned_assign_pending", - "todo_maned_new_update_request" + "todo_maned_new_update_request", + "todo_maned_on_hold" ] category_args = { @@ -100,6 +101,9 @@ def assign_pending(ap): self.build_application("maned_update_request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, apps, update_request=True) + # an application that was modifed recently into the ready status (todo_maned_completed) + self.build_application("maned_on_hold", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, apps) + # Applications that should never be reported ############################################ From 5bf71ad623d0c3fa15a6a1ff4113fe3abf21e8a8 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 19 Jul 2024 10:23:38 +0100 Subject: [PATCH 03/17] update maned todo functional test script --- doajtest/testbook/dashboards/maned_todo.yml | 23 +++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml index 96c7b171c4..017965bc75 100644 --- a/doajtest/testbook/dashboards/maned_todo.yml +++ b/doajtest/testbook/dashboards/maned_todo.yml @@ -27,7 +27,7 @@ tests: - step: Go to the maned dashboard page path: /dashboard results: - - You can see 16 applications in your priority list + - You can see 17 applications in your priority list - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), editor items (actions related to teams you are the editor for) and associate items (actions related to applications which are assigned specifically to you for review). @@ -37,30 +37,31 @@ tests: - At least one of your priority items is for an application in the state ready (it should indicate that it is for your maned group) - At least one of your priority items is for an application in the completed state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) - At least one of your priority items is for an application in the pending state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) + - At least one of your priority items is for an application in the "on hold" state - Your lowest priority item (last in the list) is for an update request which was submitted this month - step: click on the managing editor's ready application - step: Change the application status to "Accepted" and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 15 applications in your priority list + - You can see 16 applications in your priority list - The application you have just edited has disappeared from your priority list - step: click on the [in progress] stalled managing editor's application - step: make any minor adjustment to the metadata and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 14 applications in your priority list + - You can see 15 applications in your priority list - The application you just edited has disappeared from your priority list - step: click on the "completed" maned application - step: Change the application to "ready" status - step: close the tab, return to the dashboard and reload the page results: - - You can still see 14 applications in your priority list + - You can still see 13 applications in your priority list - The completed application you just moved to ready is now in your priority list as a ready application - step: click on the pending managing editor's application - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select) - step: close the tab, return to the dashboard and reload the page results: - - You have 13 applications left in your todo list + - You have 12 applications left in your todo list - The pending application you just edited is no longer visible - title: Filtering the todo list @@ -74,22 +75,26 @@ tests: - step: Go to the maned dashboard page path: /dashboard results: - - You can see 16 applications in your priority list + - You can see 17 applications in your priority list - Your highest priority item (first in the list) is for an update request which was submitted last month - Your lowest priority item (last in the list) is for an update request which was submitted this month - - On the top right of the todo list are a set of filter buttons "Show all", "New Applications" and "Update Requests" + - On the top right of the todo list are a set of filter buttons "Show all", "New Applications", "Update Requests" and "On Hold" - The "Show all" button is highlighted - step: click on the "New Applications" filter button results: - You can see 14 applications in your priority list - - The update requests which were on the previous screen are no longer visible + - The update requests and "on hold" items which were on the previous screen are no longer visible - The "New Applications" filter button is now highlighted - step: click on the "Update Request" filter button results: - - You can see 12application in your priority list + - You can see 2 applications in your priority list - Your highest priority item (first in the list) is for an update request which was submitted last month - Your lowest priority item (last in the list) is for an update request which was submitted this month - The "Update Request" filter button is now highlighted + - step: click on the "On Hold" filter button + results: + - You can see 1 application in your priority list + - The "On Hold" filter button is now highlighted - step: click the "Show all" filter button results: - You are back to the original display, containing both applications and update requests \ No newline at end of file From 3aa6803b57bf2b8f1e32dda55d0114e79c0e8dd5 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 19 Jul 2024 13:27:38 +0100 Subject: [PATCH 04/17] remove on hold constraint from update requests (should be irrelevant) --- portality/bll/services/todo.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 258094b0c1..38a0225dd2 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -274,8 +274,7 @@ def maned_last_month_update_requests(cls, size, maned_of): must_nots=[ TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, - constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_ON_HOLD + constants.APPLICATION_STATUS_REJECTED ]) # TodoQuery.exists("admin.editor") ], @@ -298,8 +297,7 @@ def maned_new_update_requests(cls, size, maned_of): must_nots=[ TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, - constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_ON_HOLD + constants.APPLICATION_STATUS_REJECTED ]) # TodoQuery.exists("admin.editor") ], From b7052394c1f3a7a1280a8dcc8ad0ac75b511a9b9 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 22 Aug 2024 16:33:53 +0100 Subject: [PATCH 05/17] add withdrawn journals as delete records in oai --- portality/crosswalks/oaipmh.py | 7 +++++++ portality/models/oaipmh.py | 15 ++++++++------- portality/view/oaipmh.py | 12 ++++++++---- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py index 26bab8229c..a4c5149881 100644 --- a/portality/crosswalks/oaipmh.py +++ b/portality/crosswalks/oaipmh.py @@ -246,6 +246,10 @@ def crosswalk(self, record): idel = etree.SubElement(oai_dc, self.DC + "identifier") set_text(idel, identifier.get("id")) + # beyond this point, only include the metadata if the record is not "deleted" + if not record.is_in_doaj(): + return metadata + # our internal identifier url = app.config["BASE_URL"] + "/toc/" + record.toc_id idel = etree.SubElement(oai_dc, self.DC + "identifier") @@ -293,6 +297,9 @@ def header(self, record): bibjson = record.bibjson() head = etree.Element(self.PMH + "header", nsmap=self.NSMAP) + if not record.is_in_doaj(): + head.set("status", "deleted") + identifier = etree.SubElement(head, self.PMH + "identifier") set_text(identifier, make_oai_identifier(record.id, "journal")) diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py index 4350cb66e7..39a6ee4f2b 100644 --- a/portality/models/oaipmh.py +++ b/portality/models/oaipmh.py @@ -42,7 +42,7 @@ class OAIPMHRecord(object): "query": { "bool": { "must": [ - { "term": { "admin.in_doaj": True } } + # { "term": { "admin.in_doaj": True } } ] } }, @@ -126,15 +126,16 @@ def pull(self, identifier): return record return None + class OAIPMHJournal(OAIPMHRecord, Journal): def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=None, start_after=None): total, results = super(OAIPMHJournal, self).list_records(from_date=from_date, until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after) return total, [Journal(**r) for r in results] - def pull(self, identifier): - # override the default pull, as we care about whether the item is in_doaj - record = super(OAIPMHJournal, self).pull(identifier) - if record is not None and record.is_in_doaj(): - return record - return None + # def pull(self, identifier): + # # override the default pull, as we care about whether the item is in_doaj + # record = super(OAIPMHJournal, self).pull(identifier) + # if record is not None and record.is_in_doaj(): + # return record + # return None diff --git a/portality/view/oaipmh.py b/portality/view/oaipmh.py index 947d5d5f28..3575c4cf10 100644 --- a/portality/view/oaipmh.py +++ b/portality/view/oaipmh.py @@ -288,13 +288,16 @@ def get_record(dao, base_url, specified_oai_endpoint, identifier=None, metadata_ return IdDoesNotExist(base_url) # do the crosswalk xwalk = get_crosswalk(f.get("metadataPrefix"), dao.__type__) - metadata = xwalk.crosswalk(record) + header = xwalk.header(record) - # make the response oai_id = make_oai_identifier(identifier, dao.__type__) gr = GetRecord(base_url, oai_id, metadata_prefix) - gr.metadata = metadata gr.header = header + + if record.is_in_doaj(): + metadata = xwalk.crosswalk(record) + gr.metadata = metadata + return gr # if we have not returned already, this means we can't disseminate this format @@ -556,7 +559,8 @@ def get_element(self): record = etree.SubElement(gr, self.PMH + "record") record.append(self.header) - record.append(self.metadata) + if self.metadata is not None: + record.append(self.metadata) return gr From c0eef4e6ad0d3879ead2bc9bba4cf3a4b01057c5 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Tue, 27 Aug 2024 10:37:56 +0100 Subject: [PATCH 06/17] add withdrawn articles as delete records in oai --- portality/crosswalks/oaipmh.py | 82 +++++++++++++++++++++------------- portality/models/oaipmh.py | 12 ++--- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py index a4c5149881..6c844a3c3a 100644 --- a/portality/crosswalks/oaipmh.py +++ b/portality/crosswalks/oaipmh.py @@ -106,15 +106,28 @@ def crosswalk(self, record): oai_dc.set(self.XSI + "schemaLocation", "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd") - if bibjson.title is not None: - title = etree.SubElement(oai_dc, self.DC + "title") - set_text(title, bibjson.title) - # all the external identifiers (ISSNs, etc) for identifier in bibjson.get_identifiers(): idel = etree.SubElement(oai_dc, self.DC + "identifier") set_text(idel, identifier.get("id")) + # beyond this point, only include the metadata if the record is not "deleted" + if not record.is_in_doaj(): + # include the ft url only, so the client can identify the article this way if needed + ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) + if ftobj and "url" in ftobj: + urlel = etree.SubElement(oai_dc, self.DC + "relation") + set_text(urlel, ftobj.get("url")) + return metadata + + for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN): + journallink = etree.SubElement(oai_dc, self.DC + "relation") + set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier) + + if bibjson.title is not None: + title = etree.SubElement(oai_dc, self.DC + "title") + set_text(title, bibjson.title) + # our internal identifier url = app.config['BASE_URL'] + "/article/" + record.id idel = etree.SubElement(oai_dc, self.DC + "identifier") @@ -130,10 +143,6 @@ def crosswalk(self, record): urlel = etree.SubElement(oai_dc, self.DC + "relation") set_text(urlel, url.get("url")) - for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN): - journallink = etree.SubElement(oai_dc, self.DC + "relation") - set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier) - if bibjson.abstract is not None: abstract = etree.SubElement(oai_dc, self.DC + "description") set_text(abstract, bibjson.abstract) @@ -171,6 +180,9 @@ def header(self, record): bibjson = record.bibjson() head = etree.Element(self.PMH + "header", nsmap=self.NSMAP) + if not record.is_in_doaj(): + head.set("status", "deleted") + identifier = etree.SubElement(head, self.PMH + "identifier") set_text(identifier, make_oai_identifier(record.id, "article")) @@ -329,6 +341,34 @@ def crosswalk(self, record): oai_doaj_article.set(self.XSI + "schemaLocation", "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd http://doaj.org/features/oai_doaj/1.0/ https://doaj.org/static/doaj/doajArticles.xsd") + # all the external identifiers (ISSNs, etc) + if bibjson.get_one_identifier(bibjson.P_ISSN): + issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn") + set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN)) + + if bibjson.get_one_identifier(bibjson.E_ISSN): + eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn") + set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN)) + + if bibjson.get_one_identifier(bibjson.DOI): + doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi") + set_text(doi, bibjson.get_one_identifier(bibjson.DOI)) + + ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) + if ftobj: + attrib = {} + if "content_type" in ftobj: + attrib['format'] = ftobj['content_type'] + + fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib) + + if "url" in ftobj: + set_text(fulltext_url_elem, ftobj['url']) + + # beyond this point, only include the metadata if the record is not "deleted" + if not record.is_in_doaj(): + return metadata + # look up the journal's language jlangs = bibjson.journal_language # first, if there are any languages recorded, get the 3-char code @@ -355,14 +395,6 @@ def crosswalk(self, record): journtitel = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "journalTitle") set_text(journtitel, bibjson.journal_title) - # all the external identifiers (ISSNs, etc) - if bibjson.get_one_identifier(bibjson.P_ISSN): - issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn") - set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN)) - - if bibjson.get_one_identifier(bibjson.E_ISSN): - eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn") - set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN)) # work out the date of publication date = bibjson.get_publication_date() @@ -396,9 +428,7 @@ def crosswalk(self, record): end_page = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "endPage") set_text(end_page, bibjson.end_page) - if bibjson.get_one_identifier(bibjson.DOI): - doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi") - set_text(doi, bibjson.get_one_identifier(bibjson.DOI)) + if record.publisher_record_id(): pubrecid = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "publisherRecordId") @@ -443,17 +473,6 @@ def crosswalk(self, record): abstract = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "abstract") set_text(abstract, bibjson.abstract) - ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) - if ftobj: - attrib = {} - if "content_type" in ftobj: - attrib['format'] = ftobj['content_type'] - - fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib) - - if "url" in ftobj: - set_text(fulltext_url_elem, ftobj['url']) - if bibjson.keywords: keywords_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + 'keywords') for keyword in bibjson.keywords: @@ -466,6 +485,9 @@ def header(self, record): bibjson = record.bibjson() head = etree.Element(self.PMH + "header", nsmap=self.NSMAP) + if not record.is_in_doaj(): + head.set("status", "deleted") + identifier = etree.SubElement(head, self.PMH + "identifier") set_text(identifier, make_oai_identifier(record.id, "article")) diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py index 39a6ee4f2b..461cd97598 100644 --- a/portality/models/oaipmh.py +++ b/portality/models/oaipmh.py @@ -119,12 +119,12 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size= until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after) return total, [Article(**r) for r in results] - def pull(self, identifier): - # override the default pull, as we care about whether the item is in_doaj - record = super(OAIPMHArticle, self).pull(identifier) - if record is not None and record.is_in_doaj(): - return record - return None + # def pull(self, identifier): + # # override the default pull, as we care about whether the item is in_doaj + # record = super(OAIPMHArticle, self).pull(identifier) + # if record is not None and record.is_in_doaj(): + # return record + # return None class OAIPMHJournal(OAIPMHRecord, Journal): From 9f0540315971e03e6f76f50ef788df598c0edce5 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 29 Aug 2024 11:52:16 +0100 Subject: [PATCH 07/17] tighten up deleted record implementation for spec compliance --- portality/crosswalks/oaipmh.py | 89 +++++++++++++++------------------- portality/view/oaipmh.py | 3 +- 2 files changed, 41 insertions(+), 51 deletions(-) diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py index 6c844a3c3a..cd09e60890 100644 --- a/portality/crosswalks/oaipmh.py +++ b/portality/crosswalks/oaipmh.py @@ -99,6 +99,9 @@ class OAI_DC_Article(OAI_DC): ~~->OAIDC:Crosswalk~~ """ def crosswalk(self, record): + if not record.is_in_doaj(): + return None + bibjson = record.bibjson() metadata = etree.Element(self.PMH + "metadata") @@ -106,28 +109,15 @@ def crosswalk(self, record): oai_dc.set(self.XSI + "schemaLocation", "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd") + if bibjson.title is not None: + title = etree.SubElement(oai_dc, self.DC + "title") + set_text(title, bibjson.title) + # all the external identifiers (ISSNs, etc) for identifier in bibjson.get_identifiers(): idel = etree.SubElement(oai_dc, self.DC + "identifier") set_text(idel, identifier.get("id")) - # beyond this point, only include the metadata if the record is not "deleted" - if not record.is_in_doaj(): - # include the ft url only, so the client can identify the article this way if needed - ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) - if ftobj and "url" in ftobj: - urlel = etree.SubElement(oai_dc, self.DC + "relation") - set_text(urlel, ftobj.get("url")) - return metadata - - for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN): - journallink = etree.SubElement(oai_dc, self.DC + "relation") - set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier) - - if bibjson.title is not None: - title = etree.SubElement(oai_dc, self.DC + "title") - set_text(title, bibjson.title) - # our internal identifier url = app.config['BASE_URL'] + "/article/" + record.id idel = etree.SubElement(oai_dc, self.DC + "identifier") @@ -143,6 +133,10 @@ def crosswalk(self, record): urlel = etree.SubElement(oai_dc, self.DC + "relation") set_text(urlel, url.get("url")) + for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN): + journallink = etree.SubElement(oai_dc, self.DC + "relation") + set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier) + if bibjson.abstract is not None: abstract = etree.SubElement(oai_dc, self.DC + "description") set_text(abstract, bibjson.abstract) @@ -243,6 +237,9 @@ class OAI_DC_Journal(OAI_DC): ~~->OAIDC:Crosswalk~~ """ def crosswalk(self, record): + if not record.is_in_doaj(): + return None + bibjson = record.bibjson() metadata = etree.Element(self.PMH + "metadata") @@ -258,10 +255,6 @@ def crosswalk(self, record): idel = etree.SubElement(oai_dc, self.DC + "identifier") set_text(idel, identifier.get("id")) - # beyond this point, only include the metadata if the record is not "deleted" - if not record.is_in_doaj(): - return metadata - # our internal identifier url = app.config["BASE_URL"] + "/toc/" + record.toc_id idel = etree.SubElement(oai_dc, self.DC + "identifier") @@ -334,6 +327,9 @@ class OAI_DOAJ_Article(OAI_Crosswalk): NSMAP.update({"oai_doaj": OAI_DOAJ_NAMESPACE}) def crosswalk(self, record): + if not record.is_in_doaj(): + return None + bibjson = record.bibjson() metadata = etree.Element(self.PMH + "metadata") @@ -341,34 +337,6 @@ def crosswalk(self, record): oai_doaj_article.set(self.XSI + "schemaLocation", "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd http://doaj.org/features/oai_doaj/1.0/ https://doaj.org/static/doaj/doajArticles.xsd") - # all the external identifiers (ISSNs, etc) - if bibjson.get_one_identifier(bibjson.P_ISSN): - issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn") - set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN)) - - if bibjson.get_one_identifier(bibjson.E_ISSN): - eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn") - set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN)) - - if bibjson.get_one_identifier(bibjson.DOI): - doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi") - set_text(doi, bibjson.get_one_identifier(bibjson.DOI)) - - ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) - if ftobj: - attrib = {} - if "content_type" in ftobj: - attrib['format'] = ftobj['content_type'] - - fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib) - - if "url" in ftobj: - set_text(fulltext_url_elem, ftobj['url']) - - # beyond this point, only include the metadata if the record is not "deleted" - if not record.is_in_doaj(): - return metadata - # look up the journal's language jlangs = bibjson.journal_language # first, if there are any languages recorded, get the 3-char code @@ -395,6 +363,14 @@ def crosswalk(self, record): journtitel = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "journalTitle") set_text(journtitel, bibjson.journal_title) + # all the external identifiers (ISSNs, etc) + if bibjson.get_one_identifier(bibjson.P_ISSN): + issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn") + set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN)) + + if bibjson.get_one_identifier(bibjson.E_ISSN): + eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn") + set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN)) # work out the date of publication date = bibjson.get_publication_date() @@ -428,7 +404,9 @@ def crosswalk(self, record): end_page = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "endPage") set_text(end_page, bibjson.end_page) - + if bibjson.get_one_identifier(bibjson.DOI): + doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi") + set_text(doi, bibjson.get_one_identifier(bibjson.DOI)) if record.publisher_record_id(): pubrecid = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "publisherRecordId") @@ -473,6 +451,17 @@ def crosswalk(self, record): abstract = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "abstract") set_text(abstract, bibjson.abstract) + ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False) + if ftobj: + attrib = {} + if "content_type" in ftobj: + attrib['format'] = ftobj['content_type'] + + fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib) + + if "url" in ftobj: + set_text(fulltext_url_elem, ftobj['url']) + if bibjson.keywords: keywords_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + 'keywords') for keyword in bibjson.keywords: diff --git a/portality/view/oaipmh.py b/portality/view/oaipmh.py index 3575c4cf10..b1c5ec7ac5 100644 --- a/portality/view/oaipmh.py +++ b/portality/view/oaipmh.py @@ -739,7 +739,8 @@ def get_element(self): for metadata, header in self.records: r = etree.SubElement(lr, self.PMH + "record") r.append(header) - r.append(metadata) + if metadata is not None: + r.append(metadata) if self.resumption is not None: rt = etree.SubElement(lr, self.PMH + "resumptionToken") From 0aaa3db0f83b3e64eee34efaf520a38dfb0b5625 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 29 Aug 2024 13:11:16 +0100 Subject: [PATCH 08/17] Update oai tests --- doajtest/unit/test_oaipmh.py | 71 +++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py index 0a6540eece..a12f9736b4 100644 --- a/doajtest/unit/test_oaipmh.py +++ b/doajtest/unit/test_oaipmh.py @@ -43,7 +43,7 @@ def test_01_oai_ListMetadataFormats(self): assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix', namespaces=self.oai_ns)[0].text == 'oai_dc' def test_02_oai_journals(self): - """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ""" + """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ, marking withdrawn ones as deleted""" journal_sources = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True) j_public = models.Journal(**journal_sources[0]) j_public.save(blocking=True) @@ -52,6 +52,7 @@ def test_02_oai_journals(self): j_private = models.Journal(**journal_sources[1]) j_private.set_in_doaj(False) j_private.save(blocking=True) + deleted_id = j_private.id with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: @@ -61,11 +62,24 @@ def test_02_oai_journals(self): t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns) - # Check we only have one journal returned - assert len(records[0].xpath('//oai:record', namespaces=self.oai_ns)) == 1 - - # Check we have the correct journal - assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title + # Check we only have two journals returned + assert len(records[0].xpath('//oai:record', namespaces=self.oai_ns)) == 2 + + seen_deleted = False + seen_public = False + records = records[0].getchildren() + for r in records: + header = r.xpath('oai:header', namespaces=self.oai_ns)[0] + status = header.get("status") + if status == "deleted": + seen_deleted = True + else: + # Check we have the correct journal + seen_public = True + assert r.xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title + + assert seen_deleted + assert seen_public resp = t_client.get(url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format(public_id)) assert resp.status_code == 200 @@ -306,23 +320,22 @@ def test_08_list_sets(self): def test_09_article(self): - """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ""" + """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ, showing the others as deleted""" article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=False) - """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ""" a_private = models.Article(**article_source) + a_private.set_id(a_private.makeid()) ba = a_private.bibjson() ba.title = "Private Article" a_private.save(blocking=True) article_source = ArticleFixtureFactory.make_article_source(eissn='4321-4321', pissn='8765-8765,', in_doaj=True) a_public = models.Article(**article_source) + a_public.set_id(a_public.makeid()) ba = a_public.bibjson() ba.title = "Public Article" a_public.save(blocking=True) public_id = a_public.id - time.sleep(1) - with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) @@ -331,23 +344,39 @@ def test_09_article(self): t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns) - # Check we only have one journal returned + # Check we only have two articles returned r = records[0].xpath('//oai:record', namespaces=self.oai_ns) - assert len(r) == 1 - - # Check we have the correct journal - title = r[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text - # check orcid_id xwalk - assert str(records[0].xpath('//dc:creator/@id', namespaces=self.oai_ns)[0]) == a_public.bibjson().author[0].get("orcid_id") - assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == a_public.bibjson().title - - resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=abcdefghijk_article') + assert len(r) == 2 + + seen_deleted = False + seen_public = False + records = records[0].getchildren() + for r in records: + header = r.xpath('oai:header', namespaces=self.oai_ns)[0] + status = header.get("status") + if status == "deleted": + seen_deleted = True + else: + seen_public = True + # Check we have the correct article + title = r[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text + + # check orcid_id xwalk + assert str(records[0].xpath('//dc:creator/@id', namespaces=self.oai_ns)[0]) == \ + a_public.bibjson().author[0].get("orcid_id") + assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[ + 0].text == a_public.bibjson().title + + assert seen_deleted + assert seen_public + + resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id) assert resp.status_code == 200 t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:GetRecord', namespaces=self.oai_ns) - # Check we only have one journal returnedt + # Check we only have one article returned kids = records[0].getchildren() r = records[0].xpath('//oai:record', namespaces=self.oai_ns) assert len(r) == 1 From d6b2565821559b2b36c5d98509089c96deaddaff Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 29 Aug 2024 13:33:08 +0100 Subject: [PATCH 09/17] timing fix to oai test --- doajtest/unit/test_oaipmh.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py index a12f9736b4..d1f6133b46 100644 --- a/doajtest/unit/test_oaipmh.py +++ b/doajtest/unit/test_oaipmh.py @@ -336,6 +336,8 @@ def test_09_article(self): a_public.save(blocking=True) public_id = a_public.id + time.sleep(1) + with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) From d8974c9b3273315af524b1aa6a96fc129a348857 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Tue, 3 Sep 2024 16:53:06 +0100 Subject: [PATCH 10/17] update on hold dashboard rule to cover maned or assigned user --- doajtest/testbook/dashboards/maned_todo.yml | 18 +++++++------ .../testdrive/todo_maned_editor_associate.py | 16 +++++++++--- portality/bll/services/todo.py | 26 +++++++++++++------ 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml index 017965bc75..06ce8921c9 100644 --- a/doajtest/testbook/dashboards/maned_todo.yml +++ b/doajtest/testbook/dashboards/maned_todo.yml @@ -27,7 +27,7 @@ tests: - step: Go to the maned dashboard page path: /dashboard results: - - You can see 17 applications in your priority list + - You can see 18 applications in your priority list - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), editor items (actions related to teams you are the editor for) and associate items (actions related to applications which are assigned specifically to you for review). @@ -43,25 +43,25 @@ tests: - step: Change the application status to "Accepted" and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 16 applications in your priority list + - You can see 17 applications in your priority list - The application you have just edited has disappeared from your priority list - step: click on the [in progress] stalled managing editor's application - step: make any minor adjustment to the metadata and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 15 applications in your priority list + - You can see 16 applications in your priority list - The application you just edited has disappeared from your priority list - step: click on the "completed" maned application - step: Change the application to "ready" status - step: close the tab, return to the dashboard and reload the page results: - - You can still see 13 applications in your priority list + - You can still see 15 applications in your priority list - The completed application you just moved to ready is now in your priority list as a ready application - step: click on the pending managing editor's application - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select) - step: close the tab, return to the dashboard and reload the page results: - - You have 12 applications left in your todo list + - You have 14 applications left in your todo list - The pending application you just edited is no longer visible - title: Filtering the todo list @@ -75,14 +75,14 @@ tests: - step: Go to the maned dashboard page path: /dashboard results: - - You can see 17 applications in your priority list + - You can see 18 applications in your priority list - Your highest priority item (first in the list) is for an update request which was submitted last month - Your lowest priority item (last in the list) is for an update request which was submitted this month - On the top right of the todo list are a set of filter buttons "Show all", "New Applications", "Update Requests" and "On Hold" - The "Show all" button is highlighted - step: click on the "New Applications" filter button results: - - You can see 14 applications in your priority list + - You can see 16 applications in your priority list - The update requests and "on hold" items which were on the previous screen are no longer visible - The "New Applications" filter button is now highlighted - step: click on the "Update Request" filter button @@ -93,8 +93,10 @@ tests: - The "Update Request" filter button is now highlighted - step: click on the "On Hold" filter button results: - - You can see 1 application in your priority list + - You can see 2 application in your priority list - The "On Hold" filter button is now highlighted + - One of the "On Hold" items is for an application which is not assigned to you, but belongs to a group you are the managing editor for + - The other "On Hold" item is for an application which is assigned to you, in a group for which you are not the managing editor - step: click the "Show all" filter button results: - You are back to the original display, containing both applications and update requests \ No newline at end of file diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py index 0ca722c55e..f6a4d62875 100644 --- a/doajtest/testdrive/todo_maned_editor_associate.py +++ b/doajtest/testdrive/todo_maned_editor_associate.py @@ -51,7 +51,7 @@ def setup(self) -> dict: aapps = build_associate_applications(un) eapps = build_editor_applications(un, eg2) - mapps = build_maned_applications(un, eg1, owner.id, eg3) + mapps = build_maned_applications(un, eg1, owner.id, eg3, eg2) return { @@ -96,7 +96,7 @@ def teardown(self, params) -> dict: return {"status": "success"} -def build_maned_applications(un, eg, owner, eponymous_group): +def build_maned_applications(un, eg, owner, eponymous_group, other_group): w = 7 * 24 * 60 * 60 apps = {} @@ -142,14 +142,22 @@ def build_maned_applications(un, eg, owner, eponymous_group): "title": un + " Maned Pending Application" }] - app = build_application(un + " Maned On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, + app = build_application(un + " Maned (Group) On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, editor_group=eg.name, owner=owner) app.save() apps["on_hold"] = [{ "id": app.id, - "title": un + " Maned On Hold Application" + "title": un + " Maned (Group) On Hold Application" }] + app = build_application(un + " Maned (Editor) On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, + editor_group=other_group.name, editor=un, owner=owner) + app.save() + apps["on_hold"].append({ + "id": app.id, + "title": un + " Maned (Editor) On Hold Application" + }) + app = build_application(un + " Maned Low Priority Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, editor_group=eponymous_group.name, owner=owner) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 38a0225dd2..cd3723bb32 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -90,7 +90,7 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True queries.append(TodoRules.maned_last_month_update_requests(size, maned_of)) queries.append(TodoRules.maned_new_update_requests(size, maned_of)) if on_hold: - queries.append(TodoRules.maned_on_hold(size, maned_of)) + queries.append(TodoRules.maned_on_hold(size, account.id, maned_of)) if new_applications: # editor and associate editor roles only deal with new applications if account.has_role("editor"): @@ -307,14 +307,17 @@ def maned_new_update_requests(cls, size, maned_of): return constants.TODO_MANED_NEW_UPDATE_REQUEST, assign_pending, sort_date, -2 @classmethod - def maned_on_hold(cls, size, maned_of): + def maned_on_hold(cls, size, account, maned_of): sort_date = "created_date" on_holds = TodoQuery( musts=[ - TodoQuery.editor_group(maned_of), TodoQuery.is_new_application(), TodoQuery.status([constants.APPLICATION_STATUS_ON_HOLD]) ], + ors=[ + TodoQuery.editor_group(maned_of), + TodoQuery.editor(account) + ], sort=sort_date, size=size ) @@ -484,9 +487,10 @@ class TodoQuery(object): # therefore, we take a created_date sort to mean a date_applied sort cd_sort = {"admin.date_applied": {"order": "asc"}} - def __init__(self, musts=None, must_nots=None, sort="last_manual_update", size=10): + def __init__(self, musts=None, must_nots=None, ors=None, sort="last_manual_update", size=10): self._musts = [] if musts is None else musts self._must_nots = [] if must_nots is None else must_nots + self._ors = [] if ors is None else ors self._sort = sort self._size = size @@ -494,16 +498,22 @@ def query(self): sort = self.lmu_sort if self._sort == "last_manual_update" else self.cd_sort q = { "query" : { - "bool" : { - "must": self._musts, - "must_not": self._must_nots - } + "bool" : {} }, "sort" : [ sort ], "size" : self._size } + + if len(self._musts) > 0: + q["query"]["bool"]["must"] = self._musts + if len(self._must_nots) > 0: + q["query"]["bool"]["must_not"] = self._must_nots + if len(self._ors) > 0: + q["query"]["bool"]["should"] = self._ors + q["query"]["bool"]["minimum_should_match"] = 1 + return q @classmethod From eb7c05a42575c463e9f8e0496257c74556ce5001 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 16 Sep 2024 14:49:12 +0100 Subject: [PATCH 11/17] fully implement tests for pmh delete --- doajtest/helpers.py | 28 ++++++++++++++--- doajtest/unit/test_models.py | 55 ++++++++++++++++++++++++++++++++++ doajtest/unit/test_oaipmh.py | 22 ++++++++++---- portality/crosswalks/oaipmh.py | 6 ++-- portality/models/__init__.py | 2 +- portality/models/article.py | 42 ++++++++++++++++++++++++-- portality/models/oaipmh.py | 21 +++++++------ portality/settings.py | 1 + 8 files changed, 152 insertions(+), 25 deletions(-) diff --git a/doajtest/helpers.py b/doajtest/helpers.py index 1fcf47eba3..19c4629b22 100644 --- a/doajtest/helpers.py +++ b/doajtest/helpers.py @@ -21,6 +21,7 @@ from portality.lib.thread_utils import wait_until from portality.tasks.redis_huey import main_queue, long_running from portality.util import url_for +from view.admin import index def patch_config(inst, properties): @@ -67,6 +68,8 @@ def setUp(self): for im in self.warm_mappings: if im == "article": self.warmArticle() + if im == "article_tombstone": + self.warmArticleTombstone() # add more types if they are necessary def tearDown(self): @@ -82,6 +85,16 @@ def warmArticle(self): article.delete() Article.blockdeleted(article.id) + def warmArticleTombstone(self): + # push an article to initialise the mappings + from doajtest.fixtures import ArticleFixtureFactory + from portality.models import ArticleTombstone + source = ArticleFixtureFactory.make_article_source() + article = ArticleTombstone(**source) + article.save(blocking=True) + article.delete() + ArticleTombstone.blockdeleted(article.id) + CREATED_INDICES = [] @@ -91,10 +104,17 @@ def initialise_index(): def create_index(index_type): - if index_type in CREATED_INDICES: - return - core.initialise_index(app, core.es_connection, only_mappings=[index_type]) - CREATED_INDICES.append(index_type) + if "," in index_type: + # this covers a DAO that has multiple index types for searching purposes + # expressed as a comma separated list + index_types = index_type.split(",") + else: + index_types = [index_type] + for it in index_types: + if it in CREATED_INDICES: + return + core.initialise_index(app, core.es_connection, only_mappings=[it]) + CREATED_INDICES.append(it) def dao_proxy(dao_method, type="class"): diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index af07a92859..d1bae191c1 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1722,6 +1722,61 @@ def test_40_autocheck_retrieves(self): ap2 = models.Autocheck.for_journal("9876") assert ap2.journal == "9876" + def test_41_article_tombstone(self): + t = models.ArticleTombstone() + t.set_id("1234") + t.bibjson().add_subject("LCC", "Medicine", "KM22") + t.set_in_doaj(True) # should have no effect + + t.save(blocking=True) + + t2 = models.ArticleTombstone.pull("1234") + assert t2.id == "1234" + assert t2.is_in_doaj() is False + assert t2.last_updated is not None + assert t2.bibjson().subjects()[0].get("scheme") == "LCC" + assert t2.bibjson().subjects()[0].get("term") == "Medicine" + assert t2.bibjson().subjects()[0].get("code") == "KM22" + + def test_42_make_article_tombstone(self): + a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True)) + a.set_id(a.makeid()) + + t = a._tombstone() + assert t.id == a.id + assert t.bibjson().subjects() == a.bibjson().subjects() + assert t.is_in_doaj() is False + + a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True)) + a.set_id(a.makeid()) + a.delete() + time.sleep(1) + + stone = models.ArticleTombstone.pull(a.id) + assert stone is not None + + a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True)) + a.set_id(a.makeid()) + a.save(blocking=True) + + query = { + "query": { + "bool": { + "must": [ + {"term": {"id.exact": a.id}} + ] + } + } + } + models.Article.delete_selected(query) + time.sleep(1) + + stone = models.ArticleTombstone.pull(a.id) + assert stone is not None + + + + class TestAccount(DoajTestCase): def test_get_name_safe(self): diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py index d1f6133b46..55f27467c0 100644 --- a/doajtest/unit/test_oaipmh.py +++ b/doajtest/unit/test_oaipmh.py @@ -10,12 +10,14 @@ from doajtest.fixtures import ArticleFixtureFactory from doajtest.fixtures import JournalFixtureFactory from doajtest.helpers import DoajTestCase +from models import ArticleTombstone from portality import models from portality.app import app from portality.lib import dates from portality.lib.dates import FMT_DATE_STD from portality.view.oaipmh import ResumptionTokenException, decode_resumption_token +from doajtest.helpers import with_es class TestClient(DoajTestCase): @classmethod @@ -336,7 +338,14 @@ def test_09_article(self): a_public.save(blocking=True) public_id = a_public.id - time.sleep(1) + stone = models.ArticleTombstone() + stone.set_id(stone.makeid()) + stone.bibjson().add_subject("LCC", "Economic theory. Demography", "AB22") + stone.save(blocking=True) + stone_id = stone.id + + models.Article.blockall([(a_private.id, a_private.last_updated), (a_public.id, a_public.last_updated)]) + models.ArticleTombstone.blockall([(stone.id, stone.last_updated)]) with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: @@ -348,16 +357,16 @@ def test_09_article(self): # Check we only have two articles returned r = records[0].xpath('//oai:record', namespaces=self.oai_ns) - assert len(r) == 2 + assert len(r) == 3 - seen_deleted = False + seen_deleted = 0 seen_public = False records = records[0].getchildren() for r in records: header = r.xpath('oai:header', namespaces=self.oai_ns)[0] status = header.get("status") if status == "deleted": - seen_deleted = True + seen_deleted += 1 else: seen_public = True # Check we have the correct article @@ -369,7 +378,7 @@ def test_09_article(self): assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[ 0].text == a_public.bibjson().title - assert seen_deleted + assert seen_deleted == 2 assert seen_public resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id) @@ -437,7 +446,8 @@ def test_10_subjects(self): # Check we have the correct journal assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title - + @with_es(indices=[models.Article.__type__, models.ArticleTombstone.__type__], + warm_mappings=[models.Article.__type__, models.ArticleTombstone.__type__]) def test_11_oai_dc_attr(self): """test if the OAI-PMH article feed returns record with correct attributes in oai_dc element""" article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=True) diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py index cd09e60890..dfc4efff9d 100644 --- a/portality/crosswalks/oaipmh.py +++ b/portality/crosswalks/oaipmh.py @@ -490,10 +490,12 @@ def header(self, record): CROSSWALKS = { "oai_dc": { "article": OAI_DC_Article, - "journal": OAI_DC_Journal + "journal": OAI_DC_Journal, + "article,article_tombstone": OAI_DC_Article }, 'oai_doaj': { - "article": OAI_DOAJ_Article + "article": OAI_DOAJ_Article, + "article,article_tombstone": OAI_DOAJ_Article } } diff --git a/portality/models/__init__.py b/portality/models/__init__.py index 2570929105..21f1f460a1 100644 --- a/portality/models/__init__.py +++ b/portality/models/__init__.py @@ -14,7 +14,7 @@ from portality.models.uploads import FileUpload, ExistsFileQuery, OwnerFileQuery, ValidFileQuery, BulkArticles from portality.models.lock import Lock from portality.models.history import ArticleHistory, JournalHistory -from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException +from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException, ArticleTombstone from portality.models.oaipmh import OAIPMHRecord, OAIPMHJournal, OAIPMHArticle from portality.models.atom import AtomRecord from portality.models.search import JournalArticle, JournalStatsQuery, ArticleStatsQuery diff --git a/portality/models/article.py b/portality/models/article.py index d431bb5ae7..80f56fb2d6 100644 --- a/portality/models/article.py +++ b/portality/models/article.py @@ -95,19 +95,26 @@ def delete_by_issns(cls, issns, snapshot=True): cls.delete_selected(query=q.query(), snapshot=snapshot) @classmethod - def delete_selected(cls, query=None, owner=None, snapshot=True): + def delete_selected(cls, query=None, owner=None, snapshot=True, tombstone=True): if owner is not None: from portality.models import Journal issns = Journal.issns_by_owner(owner) q = ArticleQuery(issns=issns) query = q.query() - if snapshot: + if snapshot or tombstone: articles = cls.iterate(query, page_size=1000) for article in articles: - article.snapshot() + if snapshot: + article.snapshot() + if tombstone: + article._tombstone() return cls.delete_by_query(query) + def delete(self): + self._tombstone() + super(Article, self).delete() + def bibjson(self, **kwargs): if "bibjson" not in self.data: self.data["bibjson"] = {} @@ -142,6 +149,18 @@ def snapshot(self): hist.save() return hist.id + def _tombstone(self): + stone = ArticleTombstone() + stone.set_id(self.id) + sbj = stone.bibjson() + + subs = self.bibjson().subjects() + for s in subs: + sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code")) + + stone.save() + return stone + def add_history(self, bibjson, date=None): """Deprecated""" bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson @@ -565,6 +584,23 @@ def get_owner(self): return owners[0] + +class ArticleTombstone(Article): + __type__ = "article_tombstone" + + def snapshot(self): + return None + + def is_in_doaj(self): + return False + + def prep(self): + self.data['last_updated'] = dates.now_str() + + def save(self, *args, **kwargs): + return super(ArticleTombstone, self).save(*args, **kwargs) + + class ArticleBibJSON(GenericBibJSON): def __init__(self, bibjson=None, **kwargs): diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py index 461cd97598..1874266821 100644 --- a/portality/models/oaipmh.py +++ b/portality/models/oaipmh.py @@ -1,5 +1,6 @@ from copy import deepcopy -from portality.models import Journal, Article + +from portality.models import Journal, Article, ArticleTombstone from portality import constants class OAIPMHRecord(object): @@ -114,17 +115,19 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size= class OAIPMHArticle(OAIPMHRecord, Article): + __type__ = "article,article_tombstone" + def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=None, start_after=None): total, results = super(OAIPMHArticle, self).list_records(from_date=from_date, until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after) - return total, [Article(**r) for r in results] - - # def pull(self, identifier): - # # override the default pull, as we care about whether the item is in_doaj - # record = super(OAIPMHArticle, self).pull(identifier) - # if record is not None and record.is_in_doaj(): - # return record - # return None + return total, [Article(**r) if r.get("es_type") == "article" else ArticleTombstone(**r) for r in results] + + def pull(self, identifier): + # override the default pull, as we must check the tombstone record too + article = Article.pull(identifier) + if article is None: + article = ArticleTombstone.pull(identifier) + return article class OAIPMHJournal(OAIPMHRecord, Journal): diff --git a/portality/settings.py b/portality/settings.py index c9abf564c7..446fdcfa94 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -707,6 +707,7 @@ MAPPINGS['provenance'] = MAPPINGS["account"] #~~->Provenance:Model~~ MAPPINGS['preserve'] = MAPPINGS["account"] #~~->Preservation:Model~~ MAPPINGS['notification'] = MAPPINGS["account"] #~~->Notification:Model~~ +MAPPINGS['article_tombstone'] = MAPPINGS["account"] #~~->ArticleTombstone:Model~~ ######################################### # Query Routes From 01bf01175190e10d09f0f54886927f41e115ed04 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Mon, 16 Sep 2024 14:58:15 +0100 Subject: [PATCH 12/17] tidy up code for PR --- doajtest/helpers.py | 1 - doajtest/unit/test_oaipmh.py | 5 +---- portality/models/oaipmh.py | 11 +---------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/doajtest/helpers.py b/doajtest/helpers.py index 19c4629b22..fe2f585fa2 100644 --- a/doajtest/helpers.py +++ b/doajtest/helpers.py @@ -21,7 +21,6 @@ from portality.lib.thread_utils import wait_until from portality.tasks.redis_huey import main_queue, long_running from portality.util import url_for -from view.admin import index def patch_config(inst, properties): diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py index 55f27467c0..d5a3291de7 100644 --- a/doajtest/unit/test_oaipmh.py +++ b/doajtest/unit/test_oaipmh.py @@ -10,7 +10,6 @@ from doajtest.fixtures import ArticleFixtureFactory from doajtest.fixtures import JournalFixtureFactory from doajtest.helpers import DoajTestCase -from models import ArticleTombstone from portality import models from portality.app import app from portality.lib import dates @@ -54,7 +53,6 @@ def test_02_oai_journals(self): j_private = models.Journal(**journal_sources[1]) j_private.set_in_doaj(False) j_private.save(blocking=True) - deleted_id = j_private.id with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: @@ -342,7 +340,6 @@ def test_09_article(self): stone.set_id(stone.makeid()) stone.bibjson().add_subject("LCC", "Economic theory. Demography", "AB22") stone.save(blocking=True) - stone_id = stone.id models.Article.blockall([(a_private.id, a_private.last_updated), (a_public.id, a_public.last_updated)]) models.ArticleTombstone.blockall([(stone.id, stone.last_updated)]) @@ -355,7 +352,7 @@ def test_09_article(self): t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns) - # Check we only have two articles returned + # Check we only have three articles returned r = records[0].xpath('//oai:record', namespaces=self.oai_ns) assert len(r) == 3 diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py index 1874266821..113c6076e1 100644 --- a/portality/models/oaipmh.py +++ b/portality/models/oaipmh.py @@ -42,9 +42,7 @@ class OAIPMHRecord(object): "track_total_hits": True, "query": { "bool": { - "must": [ - # { "term": { "admin.in_doaj": True } } - ] + "must": [] } }, "from": 0, @@ -135,10 +133,3 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size= total, results = super(OAIPMHJournal, self).list_records(from_date=from_date, until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after) return total, [Journal(**r) for r in results] - - # def pull(self, identifier): - # # override the default pull, as we care about whether the item is in_doaj - # record = super(OAIPMHJournal, self).pull(identifier) - # if record is not None and record.is_in_doaj(): - # return record - # return None From d57f619c74f04ef8adde354ed4db6929a69d64d1 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 24 Oct 2024 13:32:08 +0100 Subject: [PATCH 13/17] exclude on-hold items from all rules, not just maned rules, as these sometimes cause maneds to see applications under 'new application' which are also 'on hold' --- portality/bll/services/todo.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index cd3723bb32..25a62a2c71 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -336,7 +336,8 @@ def editor_stalled(cls, groups, size): TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_READY + constants.APPLICATION_STATUS_READY, + constants.APPLICATION_STATUS_ON_HOLD ]) ], sort=sort_date, @@ -357,7 +358,8 @@ def editor_follow_up_old(cls, groups, size): TodoQuery.status([ constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, - constants.APPLICATION_STATUS_READY + constants.APPLICATION_STATUS_READY, + constants.APPLICATION_STATUS_ON_HOLD ]) ], sort=sort_date, @@ -410,7 +412,8 @@ def associate_stalled(cls, acc_id, size): constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, constants.APPLICATION_STATUS_READY, - constants.APPLICATION_STATUS_COMPLETED + constants.APPLICATION_STATUS_COMPLETED, + constants.APPLICATION_STATUS_ON_HOLD ]) ], sort=sort_field, @@ -432,7 +435,8 @@ def associate_follow_up_old(cls, acc_id, size): constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, constants.APPLICATION_STATUS_READY, - constants.APPLICATION_STATUS_COMPLETED + constants.APPLICATION_STATUS_COMPLETED, + constants.APPLICATION_STATUS_ON_HOLD ]) ], sort=sort_field, @@ -467,7 +471,8 @@ def associate_all_applications(cls, acc_id, size): constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED, constants.APPLICATION_STATUS_READY, - constants.APPLICATION_STATUS_COMPLETED + constants.APPLICATION_STATUS_COMPLETED, + constants.APPLICATION_STATUS_ON_HOLD ]) ], sort=sort_field, From b8dd4d24021c2f7572083a18d5873e7e9a5633c2 Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Tue, 5 Nov 2024 12:22:35 +0000 Subject: [PATCH 14/17] reinstate CI on release branches --- .circleci/config.yml | 1 - portality/settings.py | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f08608285b..a0a154163a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,5 +74,4 @@ workflows: filters: branches: ignore: - - /release\/.*/ - static_pages diff --git a/portality/settings.py b/portality/settings.py index 201d65edf1..2139617060 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "7.0.0" +DOAJ_VERSION = "7.0.1" API_VERSION = "4.0.0" ###################################### diff --git a/setup.py b/setup.py index 7682c79b23..8fbc87f66a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='7.0.0', + version='7.0.1', packages=find_packages(), install_requires=[ "awscli==1.20.50", From ebc40714f1c7b4cb20e9c4e37cb8b743d48b419c Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Tue, 5 Nov 2024 14:53:20 +0000 Subject: [PATCH 15/17] Fix leaky test for OAI-PMH endpoint --- doajtest/unit/test_oaipmh.py | 92 +++++++++++++++++++++++------------- portality/models/oaipmh.py | 15 +++--- 2 files changed, 68 insertions(+), 39 deletions(-) diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py index d5a3291de7..f74569ad6c 100644 --- a/doajtest/unit/test_oaipmh.py +++ b/doajtest/unit/test_oaipmh.py @@ -18,10 +18,15 @@ from doajtest.helpers import with_es + class TestClient(DoajTestCase): @classmethod def setUpClass(cls): app.testing = True + + # Preserve default value of OAI record page size + cls.DEFAULT_OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE = app.config.get("OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE", 25) + super(TestClient, cls).setUpClass() def setUp(self): @@ -31,7 +36,11 @@ def setUp(self): self.oai_ns = {'oai': 'http://www.openarchives.org/OAI/2.0/', 'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/', 'dc': 'http://purl.org/dc/elements/1.1/', - 'xsi' : 'http://www.w3.org/2001/XMLSchema-instance'} + 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} + + def tearDown(self): + app.config['OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE'] = self.DEFAULT_OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE + super(TestClient, self).tearDown() def test_01_oai_ListMetadataFormats(self): """ Check we get the correct response from the OAI endpoint ListMetdataFormats request""" @@ -41,7 +50,8 @@ def test_01_oai_ListMetadataFormats(self): assert resp.status_code == 200 t = etree.fromstring(resp.data) - assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix', namespaces=self.oai_ns)[0].text == 'oai_dc' + assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix', + namespaces=self.oai_ns)[0].text == 'oai_dc' def test_02_oai_journals(self): """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ, marking withdrawn ones as deleted""" @@ -81,7 +91,9 @@ def test_02_oai_journals(self): assert seen_deleted assert seen_public - resp = t_client.get(url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format(public_id)) + resp = t_client.get( + url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format( + public_id)) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -123,7 +135,7 @@ def test_03_oai_resumption_token(self): with self.app_test.test_client() as t_client: resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', metadataPrefix='oai_dc')) t = etree.fromstring(resp.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt.get('completeListSize') == '5' assert rt.get('cursor') == '2' @@ -131,7 +143,7 @@ def test_03_oai_resumption_token(self): # Get the next result resp2 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', resumptionToken=rt.text)) t = etree.fromstring(resp2.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt2 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt2.get('completeListSize') == '5' assert rt2.get('cursor') == '4' @@ -139,17 +151,18 @@ def test_03_oai_resumption_token(self): # And the final result - check we get an empty resumptionToken resp3 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', resumptionToken=rt2.text)) t = etree.fromstring(resp3.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt3 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt3.get('completeListSize') == '5' assert rt3.get('cursor') == '5' assert rt3.text is None # We should get an error if we request again with an empty resumptionToken - resp4 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers') + '&resumptionToken={0}'.format(rt3.text)) - assert resp4.status_code == 200 # fixme: should this be a real error code? + resp4 = t_client.get( + url_for('oaipmh.oaipmh', verb='ListIdentifiers') + '&resumptionToken={0}'.format(rt3.text)) + assert resp4.status_code == 200 # fixme: should this be a real error code? t = etree.fromstring(resp4.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) err = t.xpath('//oai:error', namespaces=self.oai_ns)[0] assert 'the resumptionToken argument is invalid or expired' in err.text @@ -171,9 +184,11 @@ def test_04_oai_changing_index(self): yesterday = (dates.now() - timedelta(days=1)).strftime(FMT_DATE_STD) with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday)) + resp = t_client.get( + url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format( + yesterday)) t = etree.fromstring(resp.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt.get('completeListSize') == '3' assert rt.get('cursor') == '2' @@ -187,15 +202,17 @@ def test_04_oai_changing_index(self): resp2 = t_client.get('/oai?verb=ListRecords&resumptionToken={0}'.format(rt.text)) resp2 = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', resumptionToken=rt.text)) t = etree.fromstring(resp2.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt2 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt2.get('completeListSize') == '3' assert rt2.get('cursor') == '3' # Start a new request - we should see the new journal - resp3 = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday)) + resp3 = t_client.get( + url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format( + yesterday)) t = etree.fromstring(resp3.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt.get('completeListSize') == '4' @@ -227,9 +244,11 @@ def test_05_date_ranges(self): with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: # Request OAI journals since yesterday (looking for today's results only) - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday.strftime(FMT_DATE_STD))) + resp = t_client.get( + url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format( + yesterday.strftime(FMT_DATE_STD))) t = etree.fromstring(resp.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt.get('completeListSize') == '2' assert rt.get('cursor') == '1' @@ -238,10 +257,11 @@ def test_05_date_ranges(self): assert title.text in [journals[2]['bibjson']['title'], journals[3]['bibjson']['title']] # Request OAI journals from 3 days ago to yesterday (expecting the 2 days ago results) - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}&until={1}'.format( + resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', + metadataPrefix='oai_dc') + '&from={0}&until={1}'.format( two_days_before_yesterday.strftime(FMT_DATE_STD), yesterday.strftime(FMT_DATE_STD))) t = etree.fromstring(resp.data) - #print etree.tostring(t, pretty_print=True) + # print etree.tostring(t, pretty_print=True) rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0] assert rt.get('completeListSize') == '2' assert rt.get('cursor') == '1' @@ -262,7 +282,8 @@ def test_06_identify(self): t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:Identify', namespaces=self.oai_ns) assert len(records) == 1 - assert records[0].xpath('//oai:repositoryName', namespaces=self.oai_ns)[0].text == 'Directory of Open Access Journals' + assert records[0].xpath('//oai:repositoryName', namespaces=self.oai_ns)[ + 0].text == 'Directory of Open Access Journals' assert records[0].xpath('//oai:adminEmail', namespaces=self.oai_ns)[0].text == 'helpdesk+oai@doaj.org' assert records[0].xpath('//oai:granularity', namespaces=self.oai_ns)[0].text == 'YYYY-MM-DDThh:mm:ssZ' @@ -278,15 +299,17 @@ def test_07_bad_verb(self): assert resp.status_code == 200 t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH', namespaces=self.oai_ns) - assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.' + assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[ + 0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.' assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].get("code") == 'badVerb' - #invalid verb + # invalid verb resp = t_client.get(url_for('oaipmh.oaipmh', verb='InvalidVerb', metadataPrefix='oai_dc')) assert resp.status_code == 200 t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH', namespaces=self.oai_ns) - assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.' + assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[ + 0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.' assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].get("code") == 'badVerb' def test_08_list_sets(self): @@ -311,14 +334,14 @@ def test_08_list_sets(self): # check that we can retrieve a record with one of those sets with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set=set0[0].text)) + resp = t_client.get( + url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set=set0[0].text)) assert resp.status_code == 200 t = etree.fromstring(resp.data) records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns) results = records[0].getchildren() assert len(results) == 1 - def test_09_article(self): """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ, showing the others as deleted""" article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=False) @@ -346,7 +369,8 @@ def test_09_article(self): with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) + resp = t_client.get( + url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -378,7 +402,8 @@ def test_09_article(self): assert seen_deleted == 2 assert seen_public - resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id) + resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord', + metadataPrefix='oai_dc') + '&identifier=' + public_id) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -402,7 +427,8 @@ def test_10_subjects(self): with self.app_test.test_request_context(): # Check whether the journal is found for its specific set: Veterinary Medicine (TENDOlZldGVyaW5hcnkgbWVkaWNpbmU) with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOlZldGVyaW5hcnkgbWVkaWNpbmU~')) + resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', + set='TENDOlZldGVyaW5hcnkgbWVkaWNpbmU~')) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -414,7 +440,7 @@ def test_10_subjects(self): # Check we have the correct journal assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title - #check we have expected subjects (Veterinary Medicine but not Agriculture) + # check we have expected subjects (Veterinary Medicine but not Agriculture) subjects = records[0].xpath('//dc:subject', namespaces=self.oai_ns) assert len(subjects) != 0 @@ -426,7 +452,8 @@ def test_10_subjects(self): with self.app_test.test_request_context(): # Check whether the journal is found for more general set: Agriculture (TENDOkFncmljdWx0dXJl) with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOkFncmljdWx0dXJl~')) + resp = t_client.get( + url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOkFncmljdWx0dXJl~')) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -457,7 +484,8 @@ def test_11_oai_dc_attr(self): with self.app_test.test_request_context(): with self.app_test.test_client() as t_client: - resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) + resp = t_client.get( + url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc')) assert resp.status_code == 200 t = etree.fromstring(resp.data) @@ -483,7 +511,7 @@ def test_11_oai_dc_attr(self): t = etree.fromstring(resp.data) # find metadata element of our record elem = t.xpath('/oai:OAI-PMH/oai:ListRecords/oai:record/oai:metadata', namespaces=self.oai_ns) - #metadata element should have only one child, "dc" with correct nsmap + # metadata element should have only one child, "dc" with correct nsmap oai_dc = elem[0].getchildren() assert len(oai_dc) == 1 assert oai_dc[0].tag == "{%s}" % self.oai_ns["oai_dc"] + "dc" @@ -499,4 +527,4 @@ def test_decode_resumption_token__fail(self): def test_decode_resumption_token(self): params = decode_resumption_token(base64.urlsafe_b64encode(b'{"m":1}').decode('utf-8')) - assert params == {"metadata_prefix": 1} \ No newline at end of file + assert params == {"metadata_prefix": 1} diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py index 113c6076e1..e7c050fe68 100644 --- a/portality/models/oaipmh.py +++ b/portality/models/oaipmh.py @@ -3,17 +3,18 @@ from portality.models import Journal, Article, ArticleTombstone from portality import constants + class OAIPMHRecord(object): earliest = { "query": { "bool": { "must": [ - { "term": { "admin.in_doaj": True } } + {"term": {"admin.in_doaj": True}} ] } }, "size": 1, - "sort" : [ + "sort": [ {"last_updated": {"order": "asc"}} ] } @@ -22,7 +23,7 @@ class OAIPMHRecord(object): "query": { "bool": { "must": [ - { "term": { "admin.in_doaj": True } } + {"term": {"admin.in_doaj": True}} ] } }, @@ -31,7 +32,7 @@ class OAIPMHRecord(object): "sets": { "terms": { "field": "index.schema_subject.exact", - "order": {"_key" : "asc"}, + "order": {"_key": "asc"}, "size": 100000 } } @@ -49,9 +50,9 @@ class OAIPMHRecord(object): "size": 25 } - set_limit = {"term" : { "index.classification.exact" : "" }} - range_limit = { "range" : { "last_updated" : {"gte" : "", "lte" : ""} } } - created_sort = [{"last_updated" : {"order" : "desc"}}, {"id.exact" : "desc"}] + set_limit = {"term": {"index.classification.exact": ""}} + range_limit = {"range": {"last_updated": {"gte": "", "lte": ""}}} + created_sort = [{"last_updated": {"order": "desc"}}, {"id.exact": "desc"}] def earliest_datestamp(self): result = self.query(q=self.earliest) From 270a655a7a2a30cb6292bad6262860ce7fef20f0 Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Tue, 5 Nov 2024 20:49:17 +0000 Subject: [PATCH 16/17] add CI badge to readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index bb04445ecb..7b21709af2 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ This repository provides the software which drives the DOAJ website and the DOAJ directory. +## CI Status + +**develop** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop) +**master** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master) + ## Reporting issues Please feel free to use the issue tracker on https://github.com/DOAJ/doaj/issues for any bug From 74ee3a4c0b5f2488bfd0644bf14fbee0c89c521a Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Tue, 5 Nov 2024 20:51:54 +0000 Subject: [PATCH 17/17] A bit for formatting for README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7b21709af2..e8c8c4877a 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ directory. ## CI Status -**develop** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop) -**master** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master) +**develop**   [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop) + +**master**   [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master) ## Reporting issues