From 8713a1dd34a173cd878e01901dd5c0fce710c418 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 19 Oct 2023 11:41:12 +0100 Subject: [PATCH 1/2] read all journals first before processing --- portality/bll/services/journal.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portality/bll/services/journal.py b/portality/bll/services/journal.py index cd8bc766b0..1e32447486 100644 --- a/portality/bll/services/journal.py +++ b/portality/bll/services/journal.py @@ -254,8 +254,12 @@ def _get_article_kvs(journal): return kvs # ~~!JournalCSV:Feature->Journal:Model~~ - cols = {} + journals = [] for j in models.Journal.all_in_doaj(page_size=1000): #Fixme: limited by ES, this may not be sufficient + journals.append(j) + + cols = {} + for j in journals: export_start = datetime.utcnow() logger("Exporting journal {x}".format(x=j.id)) From 80e91080f0062b1664edfcc023120eca31ff54d3 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 19 Oct 2023 11:59:06 +0100 Subject: [PATCH 2/2] try pull by id model --- portality/bll/services/journal.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/portality/bll/services/journal.py b/portality/bll/services/journal.py index 1e32447486..e7298ad483 100644 --- a/portality/bll/services/journal.py +++ b/portality/bll/services/journal.py @@ -254,14 +254,18 @@ def _get_article_kvs(journal): return kvs # ~~!JournalCSV:Feature->Journal:Model~~ - journals = [] + logger("Loading journal ids") + journal_ids = [] for j in models.Journal.all_in_doaj(page_size=1000): #Fixme: limited by ES, this may not be sufficient - journals.append(j) + journal_ids.append(j.id) + logger("Journal ids loaded: {x}".format(x=len(journal_ids))) cols = {} - for j in journals: + for jid in journal_ids: export_start = datetime.utcnow() - logger("Exporting journal {x}".format(x=j.id)) + logger("Exporting journal {x}".format(x=jid)) + + j = models.Journal.pull(jid) time_log = [] bj = j.bibjson()