From 1d3e8bbdb84c96a22b4ee3faf368305ddf5c79a8 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 10:40:08 +0100 Subject: [PATCH 01/34] add check --- portality/bll/services/article.py | 4 ++++ portality/ui/messages.py | 1 + 2 files changed, 5 insertions(+) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 7b55894d24..1e0e2ac76d 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,6 +170,10 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) + journals = models.Journal.find_by_issn([pissn,eissn], True) + if journals is None or len(journals) > 1: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): diff --git a/portality/ui/messages.py b/portality/ui/messages.py index 094751c97c..cf8092e212 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -61,6 +61,7 @@ class Messages(object): EXCEPTION_NO_CONTRIBUTORS_EXPLANATION = "DOAJ requires at least one author for each article." EXCEPTION_TOO_MANY_ISSNS = "Too many ISSNs. Only 2 ISSNs are allowed: one Print ISSN and one Online ISSN." + EXCEPTION_MISMATCHED_ISSNS = "Issns provided don't match any journal." EXCEPTION_ISSNS_OF_THE_SAME_TYPE = "Both ISSNs have the same type: {type}" EXCEPTION_IDENTICAL_PISSN_AND_EISSN = "The Print and Online ISSNs supplied are identical. If you supply 2 ISSNs they must be different." EXCEPTION_NO_ISSNS = "Neither Print ISSN nor Online ISSN has been supplied. DOAJ requires at least one ISSN." From 83b6fded639bba06090b38c8569f81f5da819e00 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 14:01:38 +0100 Subject: [PATCH 02/34] add check and unit tests --- ...issn_validation_against_journal.matrix.csv | 17 +++ ...sn_validation_against_journal.settings.csv | 19 +++ ...n_validation_against_journal.settings.json | 119 ++++++++++++++++++ ...test_article_acceptable_and_permissions.py | 56 ++++++++- portality/bll/services/article.py | 10 +- 5 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv create mode 100644 doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv b/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv new file mode 100644 index 0000000000..0d2f704aba --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv @@ -0,0 +1,17 @@ +test_id,eissn,pissn,validated +1,eissn_in_doaj,pissn_in_doaj,yes +2,eissn_in_doaj,eissn_not_in_doaj, +3,eissn_in_doaj,pissn_not_in_doaj, +4,eissn_in_doaj,!eissn_in_doaj, +5,pissn_in_doaj,eissn_in_doaj, +6,pissn_in_doaj,eissn_not_in_doaj, +7,pissn_in_doaj,pissn_not_in_doaj, +8,pissn_in_doaj,!pissn_in_doaj, +9,eissn_not_in_doaj,eissn_in_doaj, +10,eissn_not_in_doaj,pissn_in_doaj, +11,eissn_not_in_doaj,pissn_not_in_doaj, +12,eissn_not_in_doaj,!eissn_not_in_doaj, +13,pissn_not_in_doaj,eissn_in_doaj, +14,pissn_not_in_doaj,pissn_in_doaj, +15,pissn_not_in_doaj,eissn_not_in_doaj, +16,pissn_not_in_doaj,!pissn_not_in_doaj, diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv new file mode 100644 index 0000000000..a8eab3f4ce --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv @@ -0,0 +1,19 @@ +field,test_id,eissn,pissn,validated +type,index,generated,generated,conditional +deafult,,,,no +,,,, +values,,eissn_in_doaj,eissn_in_doaj,yes +values,,pissn_in_doaj,pissn_in_doaj,no +values,,eissn_not_in_doaj,eissn_not_in_doaj, +values,,pissn_not_in_doaj,pissn_not_in_doaj, +,,,, +,,,, +conditional validated,,eissn_in_doaj,pissn_in_doaj,yes +constraint eissn,,eissn_in_doaj,!eissn_in_doaj, +constraint eissn,,eissn_not_in_doaj,!eissn_not_in_doaj, +constraint eissn,,pissn_not_in_doaj,!pissn_not_in_doaj, +constraint eissn,,pissn_in_doaj,!pissn_in_doaj, +constraint pissn,,eissn_in_doaj,!eissn_in_doaj, +constraint pissn,,eissn_not_in_doaj,!eissn_not_in_doaj, +constraint pissn,,pissn_not_in_doaj,!pissn_not_in_doaj, +constraint pissn,,pissn_in_doaj,!pissn_in_doaj, \ No newline at end of file diff --git a/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json new file mode 100644 index 0000000000..11d1012a96 --- /dev/null +++ b/doajtest/matrices/article_create_article/issn_validation_against_journal.settings.json @@ -0,0 +1,119 @@ +{ + "parameters": [ + { + "name": "test_id", + "type": "index" + }, + { + "name": "eissn", + "type": "generated", + "values": { + "eissn_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "eissn_in_doaj" + ] + } + } + }, + "pissn_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "pissn_in_doaj" + ] + } + } + }, + "eissn_not_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "eissn_not_in_doaj" + ] + } + } + }, + "pissn_not_in_doaj": { + "constraints": { + "pissn": { + "nor": [ + "pissn_not_in_doaj" + ] + } + } + } + } + }, + { + "name": "pissn", + "type": "generated", + "values": { + "eissn_in_doaj": {}, + "pissn_in_doaj": {}, + "eissn_not_in_doaj": {}, + "pissn_not_in_doaj": {}, + "!eissn_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "eissn_in_doaj" + ] + } + } + }, + "!eissn_not_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "eissn_not_in_doaj" + ] + } + } + }, + "!pissn_not_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "pissn_not_in_doaj" + ] + } + } + }, + "!pissn_in_doaj": { + "constraints": { + "eissn": { + "or": [ + "pissn_in_doaj" + ] + } + } + } + } + }, + { + "name": "validated", + "type": "conditional", + "values": { + "yes": { + "conditions": [ + { + "eissn": { + "or": [ + "eissn_in_doaj" + ] + }, + "pissn": { + "or": [ + "pissn_in_doaj" + ] + } + } + ] + }, + "no": {} + } + } + ] +} \ No newline at end of file diff --git a/doajtest/unit/test_article_acceptable_and_permissions.py b/doajtest/unit/test_article_acceptable_and_permissions.py index eb4c04d4fb..881323ba92 100644 --- a/doajtest/unit/test_article_acceptable_and_permissions.py +++ b/doajtest/unit/test_article_acceptable_and_permissions.py @@ -14,6 +14,11 @@ def is_acceptable_load_cases(): "test_id", {"test_id": []}) +def issn_validation_against_journal_load_sets(): + return load_parameter_sets(rel2abs(__file__, "..", "matrices", "article_create_article"), "issn_validation_against_journal", + "test_id", + {"test_id": []}) + class TestBLLPrepareUpdatePublisher(DoajTestCase): @@ -110,4 +115,53 @@ def test_has_permissions(self): assert failed_result["unowned"].sort() == [pissn, eissn].sort() # assert failed_result == {'success': 0, 'fail': 1, 'update': 0, 'new': 0, 'shared': [], # 'unowned': [pissn, eissn], - # 'unmatched': []}, "received: {}".format(failed_result) \ No newline at end of file + # 'unmatched': []}, "received: {}".format(failed_result) + + + @parameterized.expand(issn_validation_against_journal_load_sets) + def test_issn_validation_against_journal_load_sets(self, value, kwargs): + kwpissn = kwargs.get("pissn") + kweissn = kwargs.get("eissn") + validated = kwargs.get("validated") + + js = JournalFixtureFactory.make_many_journal_sources(2) + journal_in_doaj = Journal(**js[0]) + journal_in_doaj.set_in_doaj(True) + journal_in_doaj.bibjson().pissn = "1111-1111" + journal_in_doaj.bibjson().eissn = "2222-2222" + journal_in_doaj.save(blocking=True) + + journal_not_in_doaj = Journal(**js[1]) + journal_not_in_doaj.set_in_doaj(False) + journal_not_in_doaj.bibjson().pissn = "3333-3333" + journal_not_in_doaj.bibjson().eissn = "4444-4444" + journal_not_in_doaj.save(blocking=True) + + if (kwpissn == "pissn_in_doaj"): + pissn = journal_in_doaj.bibjson().pissn + elif (kwpissn == "eissn_in_doaj"): + pissn = journal_in_doaj.bibjson().eissn + elif (kwpissn == "pissn_not_in_doaj"): + pissn = journal_not_in_doaj.bibjson().pissn + else: + pissn = journal_not_in_doaj.bibjson().eissn + + if (kweissn == "pissn_in_doaj"): + eissn = journal_in_doaj.bibjson().pissn + elif (kweissn == "eissn_in_doaj"): + eissn = journal_in_doaj.bibjson().eissn + elif (kweissn == "pissn_not_in_doaj"): + eissn = journal_not_in_doaj.bibjson().pissn + else: + eissn = journal_not_in_doaj.bibjson().eissn + + + art_source = ArticleFixtureFactory.make_article_source(pissn=pissn, eissn=eissn) + article = Article(**art_source) + + if validated: + self.assertIsNone(self.svc.is_acceptable(article)) + + else: + with self.assertRaises(exceptions.ArticleNotAcceptable): + self.svc.is_acceptable(article) \ No newline at end of file diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 1e0e2ac76d..5430b5f3ef 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,8 +170,14 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journals = models.Journal.find_by_issn([pissn,eissn], True) - if journals is None or len(journals) > 1: + journalp = models.Journal.find_by_issn([pissn], True) + journale = models.Journal.find_by_issn([eissn], True) + + # check if only one and the same journal matches pissn and eissn and if they are in the correct fields + if len(journalp) != 1 or \ + len(journale) != 1 or \ + journale[0].id != journalp[0].id or \ + journale[0].bibjson().pissn != pissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, From c5392bd7aeb2f6c9198c037062d52619bdea4900 Mon Sep 17 00:00:00 2001 From: Aga Date: Thu, 8 Jun 2023 14:05:12 +0100 Subject: [PATCH 03/34] one more unit test --- ...test_article_acceptable_and_permissions.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doajtest/unit/test_article_acceptable_and_permissions.py b/doajtest/unit/test_article_acceptable_and_permissions.py index 881323ba92..5e0328635f 100644 --- a/doajtest/unit/test_article_acceptable_and_permissions.py +++ b/doajtest/unit/test_article_acceptable_and_permissions.py @@ -164,4 +164,24 @@ def test_issn_validation_against_journal_load_sets(self, value, kwargs): else: with self.assertRaises(exceptions.ArticleNotAcceptable): - self.svc.is_acceptable(article) \ No newline at end of file + self.svc.is_acceptable(article) + + def test_check_validation_for_2_journals(self): + + js = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True) + journal_in_doaj = Journal(**js[0]) + journal_in_doaj.bibjson().pissn = "1111-1111" + journal_in_doaj.bibjson().eissn = "2222-2222" + journal_in_doaj.save(blocking=True) + + journal_not_in_doaj = Journal(**js[1]) + journal_not_in_doaj.bibjson().pissn = "3333-3333" + journal_not_in_doaj.bibjson().eissn = "4444-4444" + journal_not_in_doaj.save(blocking=True) + + + art_source = ArticleFixtureFactory.make_article_source(pissn="1111-1111", eissn="4444-4444") + article = Article(**art_source) + + with self.assertRaises(exceptions.ArticleNotAcceptable): + self.svc.is_acceptable(article) \ No newline at end of file From bca84c303821ac5323afbc2c2140acada6d04f1c Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 12:52:30 +0100 Subject: [PATCH 04/34] add new query and unit tests --- doajtest/unit/test_models.py | 27 ++++++++++++++++++++++++ portality/bll/services/article.py | 15 ++++++-------- portality/models/v2/journal.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index 09fb0d0205..1459fab2ab 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1641,3 +1641,30 @@ def test_get_name_safe(self): # account does not exist assert models.Account.get_name_safe('not existing account id') == '' + def test_11_find_by_issn(self): + js = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True) + j1 = models.Journal(**js[0]) + j1.bibjson().pissn = "1111-1111" + j1.bibjson().eissn = "2222-2222" + j1.save(blocking=True) + + j2 = models.Journal(**js[1]) + j2.bibjson().pissn = "3333-3333" + j2.bibjson().eissn = "4444-4444" + j2.save(blocking=True) + + journals = models.Journal.find_by_issn(["1111-1111", "2222-2222"], True) + assert len(journals) == 1 + assert journals[0].id == j1.id + + journals = models.Journal.find_by_issn(["1111-1111", "3333-3333"], True) + assert len(journals) == 2 + assert journals[0].id == j1.id + assert journals[1].id == j2.id + + journals = models.Journal.find_by_issn_exact(["1111-1111", "2222-2222"], True) + assert len(journals) == 1 + assert journals[0].id == j1.id + + journals = models.Journal.find_by_issn_exact(["1111-1111", "3333-3333"], True) + assert len(journals) == 0 \ No newline at end of file diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 5430b5f3ef..d9fb6701e9 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,20 +170,17 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journalp = models.Journal.find_by_issn([pissn], True) - journale = models.Journal.find_by_issn([eissn], True) - - # check if only one and the same journal matches pissn and eissn and if they are in the correct fields - if len(journalp) != 1 or \ - len(journale) != 1 or \ - journale[0].id != journalp[0].id or \ - journale[0].bibjson().pissn != pissn: + journal = models.Journal.find_by_issn_exact([pissn,eissn], True) + + # check if only one journal matches pissn and eissn and if they are in the correct fields + # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too + if len(journal) != 1 or journal[0].bibjson().pissn != pissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): - """ + """# no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too Create an individual article in the database This method will check and merge any duplicates, and report back on successes and failures in a manner consistent with diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 41c50a7ce9..5a4997a0de 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -70,6 +70,22 @@ def find_by_issn(cls, issns, in_doaj=None, max=10): records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] return records + @classmethod + def find_by_issn_exact(cls, issns, in_doaj=None, max=2): + """ + Finds journal that matches given issns exactly - if no data problems should always be only 1 + """ + if not isinstance(issns, list): + issns = [issns] + if len(issns) > 2: + return [] + q = JournalQuery() + q.find_by_issn_exact(issns, in_doaj=in_doaj, max=max) + result = cls.query(q=q.query) + # create an array of objects, using cls rather than Journal, which means subclasses can use it too + records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] + return records + @classmethod def issns_by_owner(cls, owner, in_doaj=None): q = IssnQuery(owner, in_doaj=in_doaj) @@ -922,6 +938,16 @@ class JournalQuery(object): } } + must_query = { + "track_total_hits": True, + "query": { + "bool": { + "must": [ + ] + } + } + } + all_doaj = { "track_total_hits": True, "query": { @@ -947,6 +973,14 @@ def find_by_issn(self, issns, in_doaj=None, max=10): self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}}) self.query["size"] = max + def find_by_issn_exact(self, issns, in_doaj=None, max=10): + self.query = deepcopy(self.must_query) + for issn in issns: + self.query["query"]["bool"]["must"].append({"term": {"index.issn.exact": issn}}) + if in_doaj is not None: + self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}}) + self.query["size"] = max + def all_in_doaj(self): q = deepcopy(self.all_doaj) if self.minified: From 0122262cd79df3e1c803812114a1a8c81541253a Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 13:32:25 +0100 Subject: [PATCH 05/34] remove duplicated code --- portality/bll/services/article.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index d9fb6701e9..feebf2b481 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -159,9 +159,6 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if len(pissn) > 1 or len(eissn) > 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_TOO_MANY_ISSNS) - pissn = article_bibjson.get_one_identifier("pissn") - eissn = article_bibjson.get_one_identifier("eissn") - # no pissn or eissn if not pissn and not eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) From e0db3f581df83e5cc958f96865d073f05e5b11c3 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:21:06 +0100 Subject: [PATCH 06/34] add script to find all articles with invalid issns --- ...230609_find_articles_with_invalid_issns.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 portality/scripts/230609_find_articles_with_invalid_issns.py diff --git a/portality/scripts/230609_find_articles_with_invalid_issns.py b/portality/scripts/230609_find_articles_with_invalid_issns.py new file mode 100644 index 0000000000..acebc0ae7a --- /dev/null +++ b/portality/scripts/230609_find_articles_with_invalid_issns.py @@ -0,0 +1,68 @@ +from portality import models +from portality.bll.services import article as articlesvc +from portality.bll import exceptions +from portality.core import es_connection +from portality.util import ipt_prefix +import esprit +import csv + +IN_DOAJ = { + "query": { + "bool": { + "must": [ + {"term" : {"admin.in_doaj":True}} + ] + } + } +} + + +if __name__ == "__main__": + + # import argparse + # + # parser = argparse.ArgumentParser() + # parser.add_argument("-o", "--out", help="output file path") + # args = parser.parse_args() + # + # if not args.out: + # print("Please specify an output file path with the -o option") + # parser.print_help() + # exit() + + out = "out.csv" + + with open(out, "w", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["ID", "PISSN", "EISSN", "Journals found with article's PISSN", "In doaj?", "Journals found with article's EISSN", "In doaj?", "Error"]) + + for a in models.Article.iterate(q=IN_DOAJ, page_size=100, keepalive='5m'): + print(a["id"]) + article = models.Article(_source=a) + bibjson = article.bibjson() + + try: + articlesvc.ArticleService._validate_issns(bibjson) + except exceptions.ArticleNotAcceptable as e: + id = article.id + pissn = bibjson.get_identifiers("pissn") + eissn = bibjson.get_identifiers("eissn") + j_p = [j["id"] for j in models.Journal.find_by_issn(pissn)] + j_p_in_doaj = [] + if (j_p): + for j in j_p: + jobj = models.Journal.pull(j) + if (jobj): + j_p_in_doaj.append(jobj.is_in_doaj()) + else: + j_p_in_doaj.append("n/a") + j_e = [j["id"] for j in models.Journal.find_by_issn(eissn)] + j_e_in_doaj = [] + if (j_e): + for j in j_e: + jobj = models.Journal.pull(j) + if (jobj): + j_e_in_doaj.append(jobj.is_in_doaj()) + else: + j_e_in_doaj.append("n/a") + writer.writerow([id, pissn, eissn, j_p, j_p_in_doaj, j_e, j_e_in_doaj, str(e)]) \ No newline at end of file From 3911cff2657ede8ddd966898ca287cb990569308 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:25:01 +0100 Subject: [PATCH 07/34] revert mistakenly removed code --- portality/bll/services/article.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index feebf2b481..17f872582d 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -159,6 +159,9 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if len(pissn) > 1 or len(eissn) > 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_TOO_MANY_ISSNS) + pissn = article_bibjson.get_one_identifier("pissn") + eissn = article_bibjson.get_one_identifier("eissn") + # no pissn or eissn if not pissn and not eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) @@ -167,12 +170,24 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - journal = models.Journal.find_by_issn_exact([pissn,eissn], True) + issns = [] + if pissn is not None: + issns.append(pissn) + if eissn is not None: + issns.append(eissn) + + journal = models.Journal.find_by_issn_exact(issns, True) # check if only one journal matches pissn and eissn and if they are in the correct fields # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too - if len(journal) != 1 or journal[0].bibjson().pissn != pissn: + if len(journal) != 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if pissn is not None: + if journal[0].bibjson().pissn != pissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if eissn is not None: + if journal[0].bibjson().eissn != eissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): From 27878361116f77a3ae1e2656b599001aa522006b Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 9 Jun 2023 14:29:15 +0100 Subject: [PATCH 08/34] remove unnecessary print --- portality/scripts/230609_find_articles_with_invalid_issns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/portality/scripts/230609_find_articles_with_invalid_issns.py b/portality/scripts/230609_find_articles_with_invalid_issns.py index acebc0ae7a..c45b8c5d63 100644 --- a/portality/scripts/230609_find_articles_with_invalid_issns.py +++ b/portality/scripts/230609_find_articles_with_invalid_issns.py @@ -37,10 +37,8 @@ writer.writerow(["ID", "PISSN", "EISSN", "Journals found with article's PISSN", "In doaj?", "Journals found with article's EISSN", "In doaj?", "Error"]) for a in models.Article.iterate(q=IN_DOAJ, page_size=100, keepalive='5m'): - print(a["id"]) article = models.Article(_source=a) bibjson = article.bibjson() - try: articlesvc.ArticleService._validate_issns(bibjson) except exceptions.ArticleNotAcceptable as e: From 97dd518a6268d8460e3c6cdf78b072e17172d2bf Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 26 Jun 2023 14:21:19 +0100 Subject: [PATCH 09/34] capitalisation on issns in messages --- portality/ui/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/ui/messages.py b/portality/ui/messages.py index cf8092e212..2164502784 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -61,7 +61,7 @@ class Messages(object): EXCEPTION_NO_CONTRIBUTORS_EXPLANATION = "DOAJ requires at least one author for each article." EXCEPTION_TOO_MANY_ISSNS = "Too many ISSNs. Only 2 ISSNs are allowed: one Print ISSN and one Online ISSN." - EXCEPTION_MISMATCHED_ISSNS = "Issns provided don't match any journal." + EXCEPTION_MISMATCHED_ISSNS = "ISSNs provided don't match any journal." EXCEPTION_ISSNS_OF_THE_SAME_TYPE = "Both ISSNs have the same type: {type}" EXCEPTION_IDENTICAL_PISSN_AND_EISSN = "The Print and Online ISSNs supplied are identical. If you supply 2 ISSNs they must be different." EXCEPTION_NO_ISSNS = "Neither Print ISSN nor Online ISSN has been supplied. DOAJ requires at least one ISSN." From 318a3f3db5621c31e23b3fed23a295eb4c2ede37 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 26 Jun 2023 14:37:49 +0100 Subject: [PATCH 10/34] create separate function for validation issns against journal rather than in validate issn --- portality/bll/services/article.py | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 17f872582d..2782538a0f 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,25 +170,6 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if pissn == eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_IDENTICAL_PISSN_AND_EISSN) - issns = [] - if pissn is not None: - issns.append(pissn) - if eissn is not None: - issns.append(eissn) - - journal = models.Journal.find_by_issn_exact(issns, True) - - # check if only one journal matches pissn and eissn and if they are in the correct fields - # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too - if len(journal) != 1: - raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) - if pissn is not None: - if journal[0].bibjson().pissn != pissn: - raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) - if eissn is not None: - if journal[0].bibjson().eissn != eissn: - raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) - def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): @@ -285,6 +266,7 @@ def is_acceptable(self, article: models.Article): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_DOI_NO_FULLTEXT) self._validate_issns(bj) + self.does_article_match_journal(bj) # is journal in doaj (we do this check last as it has more performance impact) journal = article.get_journal() @@ -292,6 +274,28 @@ def is_acceptable(self, article: models.Article): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_ADDING_ARTICLE_TO_WITHDRAWN_JOURNAL) @staticmethod + def does_article_match_journal(article_bibjson: models.ArticleBibJSON): + pissn = article_bibjson.get_one_identifier("pissn") + eissn = article_bibjson.get_one_identifier("eissn") + + if pissn is not None: + issns.append(pissn) + if eissn is not None: + issns.append(eissn) + + journal = models.Journal.find_by_issn_exact(issns, True) + + # check if only one journal matches pissn and eissn and if they are in the correct fields + # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too + if len(journal) != 1: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if pissn is not None: + if journal[0].bibjson().pissn != pissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + if eissn is not None: + if journal[0].bibjson().eissn != eissn: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + @staticmethod def is_legitimate_owner(article, owner): """ Determine if the owner id is the owner of the article From 1b6624ac7ae6b362ffa3e243197469f225468742 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 26 Jun 2023 15:09:57 +0100 Subject: [PATCH 11/34] naming bugs --- portality/bll/services/article.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 2782538a0f..733788d682 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -295,6 +295,8 @@ def does_article_match_journal(article_bibjson: models.ArticleBibJSON): if eissn is not None: if journal[0].bibjson().eissn != eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) + + return journal[0] @staticmethod def is_legitimate_owner(article, owner): """ From a8353b775598762854083c563df679ba152a97c9 Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 26 Jun 2023 15:10:33 +0100 Subject: [PATCH 12/34] return journal after match with validation and use it in is_acceptable method --- portality/bll/services/article.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 733788d682..93b48376f9 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -266,10 +266,10 @@ def is_acceptable(self, article: models.Article): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_DOI_NO_FULLTEXT) self._validate_issns(bj) - self.does_article_match_journal(bj) + journal = self.does_article_match_journal(bj) # is journal in doaj (we do this check last as it has more performance impact) - journal = article.get_journal() + # journal = article.get_journal() if journal is None or not journal.is_in_doaj(): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_ADDING_ARTICLE_TO_WITHDRAWN_JOURNAL) From 6331c1e39f5f150c8bd319cbb5d69f1d28eb66e6 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 28 Jun 2023 11:08:21 +0100 Subject: [PATCH 13/34] refactoring --- portality/bll/services/article.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 93b48376f9..982aa6d505 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -266,7 +266,7 @@ def is_acceptable(self, article: models.Article): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_DOI_NO_FULLTEXT) self._validate_issns(bj) - journal = self.does_article_match_journal(bj) + journal = self.match_journal_with_validation(bj) # is journal in doaj (we do this check last as it has more performance impact) # journal = article.get_journal() From 900ecf7b92e283c4884553d2e30ee64140a05149 Mon Sep 17 00:00:00 2001 From: Aga Date: Wed, 28 Jun 2023 11:24:43 +0100 Subject: [PATCH 14/34] fix errors found by UT --- portality/bll/services/article.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 982aa6d505..77aac68600 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -159,9 +159,6 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): if len(pissn) > 1 or len(eissn) > 1: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_TOO_MANY_ISSNS) - pissn = article_bibjson.get_one_identifier("pissn") - eissn = article_bibjson.get_one_identifier("eissn") - # no pissn or eissn if not pissn and not eissn: raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) @@ -274,10 +271,12 @@ def is_acceptable(self, article: models.Article): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_ADDING_ARTICLE_TO_WITHDRAWN_JOURNAL) @staticmethod - def does_article_match_journal(article_bibjson: models.ArticleBibJSON): + def match_journal_with_validation(article_bibjson: models.ArticleBibJSON): pissn = article_bibjson.get_one_identifier("pissn") eissn = article_bibjson.get_one_identifier("eissn") + issns = [] + if pissn is not None: issns.append(pissn) if eissn is not None: From e313ee5fcd9dd52568c4458bd9e8e9c2f71c84ab Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 4 Aug 2023 14:33:58 +0100 Subject: [PATCH 15/34] minor code layout tweaks --- portality/bll/services/article.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 77aac68600..777d499636 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -170,7 +170,7 @@ def _validate_issns(article_bibjson: models.ArticleBibJSON): def create_article(self, article, account, duplicate_check=True, merge_duplicate=True, limit_to_account=True, add_journal_info=False, dry_run=False, update_article_id=None): - """# no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too + """ Create an individual article in the database This method will check and merge any duplicates, and report back on successes and failures in a manner consistent with @@ -249,7 +249,8 @@ def has_permissions(self, account, article, limit_to_account): def is_acceptable(self, article: models.Article): """ conduct some deep validation on the article to make sure we will accept it - or the moment, this just means making sure it has a DOI and a fulltext + this just means making sure it has a DOI and a fulltext, and that its ISSNs + match a single journal """ try: bj = article.bibjson() @@ -266,7 +267,6 @@ def is_acceptable(self, article: models.Article): journal = self.match_journal_with_validation(bj) # is journal in doaj (we do this check last as it has more performance impact) - # journal = article.get_journal() if journal is None or not journal.is_in_doaj(): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_ADDING_ARTICLE_TO_WITHDRAWN_JOURNAL) @@ -296,6 +296,7 @@ def match_journal_with_validation(article_bibjson: models.ArticleBibJSON): raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_MISMATCHED_ISSNS) return journal[0] + @staticmethod def is_legitimate_owner(article, owner): """ From c8e6e919285ddd5ff4414f020e6322d3e83d066c Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 21 Sep 2023 11:32:58 +0100 Subject: [PATCH 16/34] Validate supplied owner in csv ingest --- portality/forms/application_processors.py | 10 +++++----- portality/scripts/journals_update_via_csv.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/portality/forms/application_processors.py b/portality/forms/application_processors.py index 13a294d14d..1cd426c1f6 100644 --- a/portality/forms/application_processors.py +++ b/portality/forms/application_processors.py @@ -198,8 +198,11 @@ def _patch_target_note_id(self): for note in self.target.notes: note_date = dates.parse(note['date']) if not note.get('author_id') and note_date > dates.before_now(60): - note['author_id'] = current_user.id - + try: + note['author_id'] = current_user.id + except AttributeError: + # Skip if we don't have a current_user + pass class NewApplication(ApplicationProcessor): @@ -307,7 +310,6 @@ def patch_target(self): if (self.target.owner is None or self.target.owner == "") and (self.source.owner is not None): self.target.set_owner(self.source.owner) - def finalise(self, account, save_target=True, email_alert=True): """ account is the administrator account carrying out the action @@ -326,7 +328,6 @@ def finalise(self, account, save_target=True, email_alert=True): elif not j.is_in_doaj(): raise Exception(Messages.EXCEPTION_EDITING_WITHDRAWN_JOURNAL) - # if we are allowed to finalise, kick this up to the superclass super(AdminApplication, self).finalise() @@ -813,7 +814,6 @@ def patch_target(self): if (self.target.owner is None or self.target.owner == "") and (self.source.owner is not None): self.target.set_owner(self.source.owner) - def finalise(self): # FIXME: this first one, we ought to deal with outside the form context, but for the time being this # can be carried over from the old implementation diff --git a/portality/scripts/journals_update_via_csv.py b/portality/scripts/journals_update_via_csv.py index 298b7c817b..c696068a85 100644 --- a/portality/scripts/journals_update_via_csv.py +++ b/portality/scripts/journals_update_via_csv.py @@ -82,6 +82,7 @@ reader = csv.DictReader(g, fieldnames=header_row) # verify header row with current CSV headers, report errors + # TODO: Include 'Owner' field - but we should probably base this process off the AdminCSV too. expected_headers = JournalFixtureFactory.csv_headers() # Always perform a match check on supplied headers, not counting order @@ -155,6 +156,14 @@ if len(updates) > 0: [print(upd) for upd in updates] + # Check we have the expected owner (if supplied) before proceeding to create an update request + own = row.get('Owner') + if own is not None: + if own.strip().lower() != j.owner.strip().lower(): + print('ABORTING - supplied owner {0} mismatches journal owner {1}.'.format(own, j.owner)) + writer.writerow([j.id, ' | '.join(updates), 'COULD NOT UPDATE - Owner mismatch. Expected {0} Got {1}'.format(own, j.owner)]) + continue + # Create an update request for this journal update_req = None jlock = None @@ -204,7 +213,7 @@ # Add note to UR if supplied if note: - fc.target.add_note(note) + fc.target.add_note(note, author_id=sys_acc.id) if not args.manual_review: # This is the update request, in 'update request' state From 0b197f8fc86b3e13d86ddfd9f5970f19ed1478f2 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 21 Sep 2023 14:51:44 +0100 Subject: [PATCH 17/34] Reduce frequency of journalcsv to every 2 hours --- portality/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/settings.py b/portality/settings.py index f2bdd5479f..7301701e2b 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -427,7 +427,7 @@ HUEY_SCHEDULE = { "sitemap": {"month": "*", "day": "*", "day_of_week": "*", "hour": "8", "minute": "0"}, "reporting": {"month": "*", "day": "1", "day_of_week": "*", "hour": "0", "minute": "0"}, - "journal_csv": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "35"}, + "journal_csv": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*/2", "minute": "20"}, "read_news": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "30"}, "article_cleanup_sync": {"month": "*", "day": "2", "day_of_week": "*", "hour": "0", "minute": "0"}, "async_workflow_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "0"}, From 3ae615156b160692d0e48c80096d7b225ecea16d Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Fri, 22 Sep 2023 09:21:42 +0200 Subject: [PATCH 18/34] Removed John Dove --- cms/data/ambassadors.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cms/data/ambassadors.yml b/cms/data/ambassadors.yml index 3f797bcd2d..f2a949b4f0 100644 --- a/cms/data/ambassadors.yml +++ b/cms/data/ambassadors.yml @@ -62,13 +62,6 @@ photo: "ivonne.jpg" coi: 2022: https://drive.google.com/file/d/1HnGhYbvbzL34guWOmIqcthcwAN8NADX1/view?usp=sharing - -- name: John G. Dove - region: North America - bio: "John has had a career in executive management, and is now an independent consultant and open access advocate who works with organisations seeking to accelerate their transition to open access. He advises both for-profits and non-profits, and has a particular interest in identifying the steps necessary to flip an entire discipline’s scholarly record to open access. His ambassador activities focus on increasing the support to DOAJ from the community. He served for six years on NISO’s Information Discovery and Interchange Topic Committee, and has written for Learned Publishing, Against the Grain, and Scholarly Kitchen. John serves on the Board of Trustees of his local public library in Revere, Massachusetts. He has a B.A. in Mathematics from Oberlin College." - photo: "johndove.jpg" - coi: - 2022: https://drive.google.com/file/d/1cWijl2xdmVjshsvaGTABOvC_chIIfuVA/view?usp=sharing - name: Mahmoud Khalifa region: Middle East and Persian Gulf From d2baf88bd77751a61922f35f86ee93e5e8389324 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Fri, 22 Sep 2023 09:22:50 +0200 Subject: [PATCH 19/34] Added 61-66 --- cms/data/team.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cms/data/team.yml b/cms/data/team.yml index fe599e2120..09de2d0931 100644 --- a/cms/data/team.yml +++ b/cms/data/team.yml @@ -58,6 +58,13 @@ coi: 2022: https://drive.google.com/file/d/1-3xzwkHMclREgLhj_XNF5n6Nr4q2_bnw/view?usp=sharing +- name: John G. Dove + role: DOAJ Advisor + photo: johndove.jpg + bio: "John has had a career in executive management, and is now an independent consultant and open access advocate who works with organisations seeking to accelerate their transition to open access. He advises both for-profits and non-profits, and has a particular interest in identifying the steps necessary to flip an entire discipline’s scholarly record to open access. His ambassador activities focus on increasing the support to DOAJ from the community. He served for six years on NISO’s Information Discovery and Interchange Topic Committee, and has written for Learned Publishing, Against the Grain, and Scholarly Kitchen. John serves on the Board of Trustees of his local public library in Revere, Massachusetts. He has a B.A. in Mathematics from Oberlin College." + coi: + 2022: https://drive.google.com/file/d/1cWijl2xdmVjshsvaGTABOvC_chIIfuVA/view?usp=sharing + - name: Judith Barnsby role: Head of Editorial photo: judith.jpg From ed1b3adb8661cdca21ea287b7792a40ac3ab8442 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Fri, 22 Sep 2023 09:23:40 +0200 Subject: [PATCH 20/34] Delete cms/assets/img/ambassadors/johndove.jpg --- cms/assets/img/ambassadors/johndove.jpg | Bin 11492 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cms/assets/img/ambassadors/johndove.jpg diff --git a/cms/assets/img/ambassadors/johndove.jpg b/cms/assets/img/ambassadors/johndove.jpg deleted file mode 100644 index 1d268e60720fe4ed9d48ddb6fe14fc9684c8e2b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11492 zcmbulWmH^2(=9xMySuv$9^56ky9aj(I(UKvcXxuj!{7u6?rwtw53WHH?DD+wt?&N6 zU471@RaLuJ)jn^lZ`%MYMOg(|01yZO0N)Mpwhs6PfO{YCaPSBS@Q6t7hJu2O zjDmuNiuP_;M3~r^?~@3h7$5ijVW1!(p26z7MIr3^8)_AZt(UKfQWonGB?tV411haIvTRf)o1Zj3E16KcOq07Q0n2KwN zbS^QcxKX&$EUI5Misz8FCrT}nv*7w<<59W4ZjUsQtSX&KIX)0CTI8F%*1krJo)&4C zOFmIUX%5LM(|O`*3|h&u{k;|5w*lPW^V1s-`{!Qjj?Hc zv|HBUp*aUpen_SiR&px!8HimVpDw9+T6JnH^GHSNr%vTOxyPaU7RsnQtF@|VonViy zaA|&@$EgGtef{x##w1F!b+YG!xX(Jw*r4P-QdApGsJdEM zBDtc^4@vUS(PWveiIYgUa05=+yMMi;k@*{#z~dFi|r zQ03Zx7eCcQ0_Z|fwMraJI1$FyoHb7FX;-|cwL9?_tR|bFny()d$(4Lh@YLn}&w>ecR@d!p5>rJQX3TxPZXC(-8eFc_ljphjV0j`@Ml_LxRZ21UrB@Aey>2=b zgugV_f|ES3J9`}|KN1$ho2So5G%-(BI=bCJ_%c^hpnp#>i)@^!wA1Eb(I%LsU-1qJ zAOcVO*XQ|GQs;;XD33aYejYzLC+xoV=l@*Z-J;54tc>jzXEo9Qu=jWtc&QIG4Y)ii zAZWK4`y|^NM=^W1kQo7Vbs7(TxV?ZN(E?K)+tc<#o!4-wa>$%ACuZrzI-U27l-JX% z|F}6rF-ju+*?;_n|Dg)`9i%~T;dUO^Va069%Y5KGeV&A6)ruDLIaLGRfn9LHWL^4$ z4W)RqGkWOl8{lI>^g(g+!|Y@OTDT%-t;{dEgm8`#`Wpl0!!&ynx5)3rW!l@bgDm;te;PgJ ziz1D8B$xQN>))+0NO^~Qa;qZU%?b!T<96A#Ngc6p>9@6=`kOQ2p84^5wr zSIblWHxwZJ|Ahnqgn@;7r^^3=0{b5*07TIH%PVW^UuqBlwJoDm$fg1SWHgl%tyV~z zlmHO$iniylUDybx0|;An4QSMCk^2GYyscxiQF%xrXFyyx(e{5jCXU|^w|RJ9jIN=9 zM(t`2gA=S-a_Y+$p|Ax*~<+_r8th&+xaknA&qS1COSk_2#h z<-kr!r~wJ@g-o_z^lutiEIUu3j-NVqi0>%qu->Wk9S7k5qtpL*=-nY$*f>;f>NcWp^Ot+t=>T)i#Q_&McAaPhe6TDvyJ`JzGp+h_HqONwKmbxBX#thY|g~o zgT#5icd<)d#`nUZ<1@U~BXh;sC6H4^BuR!PJmc>wXF{#!K~x@VT9NCp67F|+hwH8h zjkPVqYzCr})G7tm$LB!hxJgUn*_DNn9kXg44QbqC&c6=A!hI@wj%zN=i&}W#Uko;^ zYyqWwnpQ5~V^mT(E%m{|-@x8M*$|YhS)ul@UcrMbEjKRNUckGP_StmE zl0Kf$M@xby_R5a&*bnKsvXrwYtq@1$Y<*G1Ke<;57(5w!pIhX4ohK7h^jp}DS6hq= zlm8SJ9+j91MkScj9aGoAvj|>#LgY8z8KMOx2;EW${JdYw2V`syXc1@ZzFTUUfYsFqS=T;#fTq@ ztP($2687<|TBzEmNew~TeD}5aHdV+9{_$GXJpo4zn4!t^!+zR$I^%V#PY9BnSzdED zdJq!2n$AJ;Qcg7qk%fP341UOBYr%KeEUK1xEY;QXpW6TcY-x0y)#vb|#;30>cW6XY zB0GOkneIwHRD>W=Av%Ep{1tX~b}p9sR@uXXX_hbG!M*qeFT3lUvHgp0q-A$`s0VfS zb{MM#K>!{ee&M4$GS}HAzf*O`pISzf;fYfU@e7aw zSc~Qk<+N8)NQhD^VlGANfWyZJo3C^SUbx?zA3L)U`Z{bSynXQ<{JN50k)(1K$9sXwG)A zR2f|DE{Iq)R`XAv!PQIx)xw*h)oVR695P$YJqG6N3>p#g+5y_iy;YUv9b)z}udRW{ zD-yyOXWNnR({RCa#9{ILa^Np0?(&$W!gjR{PMu0;=fly0pqg^NS9nRzKkmF63DweW z{psnQiVM0MF1NCpCZOs*+pYRU!1X4y0J9L&fBH&?)6vhEOYG05qhmu$(;nGXV=F5w z-QYppl3gu(rCKs1J$PATk1>ser1(0bdu9WBr;ISt#99m%RC#_3vC8UrC1G2xGm-^< z3bK@3u;HwT1W?T0%M#}+FK90N+{X9SoB2fANNKusme~{z17m=4+D?X6-o8sqV@4ze zKK*0-CXfu+G#R%rOU6oRoNU*CJB!qNZ#i6F6S|ixKTY0mGfc5 zBK_*T0ZPB|VQ8~!Nfap4yuj0=uMW^MCbU+^EV3k9@549BMfhfV(mvb>Q6W=!)3Q@mg^$AOiK5ERqfH#zA>>udh7%2bmOq4#&s!P=X-$wtagL;)-*}9?3*Wy1Jvr z(vc3aCHwr@*PI#f%8eAE>{-l|#M!I0BU`dOGA)byz*hK9KZ)zd!vOUm?Omzpw7yi( zoR|Q^oHh&i9zQT`JIb;$bD~3N;&Z&@T%Aa?Qxar=M(oo;v;gSkPb%#ZKTcuPG@{+& zw$R`!ia|c=jO&?pKR*ddnmUVr2y#xs5bvNxQ2`W%x|;#fN()1v#&^Fp`j5={X7-;! zOe^!@=syTTf5a{`zJ7;cIzH=>;i77W3?GJvMWtMr6<47}JdkMKH=q^yd|`R# zdpXa)j;f|R@13otD>)WdmF~~N*-g>7Oyv%9?cq>jh|KRW z7M=WS$C?}d3z@t24bX+m1v^ab7AR6G6TcPjlPit2LIKA=s)H?Utk)`ONP?Gi8yWXs zC#m#6YWmVT*Dvka3m7KF(Mz&$`i(hJ${wnMF59i3uVO^6DE`tcbpCD=@U+)olE`AD zQwF#BHc_K^pynwk>tI}Xs~kdjM*{_d6}ouTu<&AKs$I9ghH4w5Wh*5j$v5k{Yl zJy2$J8in7Wn1}B*!eME*O_X_9d8&BS?~;8cK?$SX*a83XLQGD^b|-01cDPQw+^tPU z$oy9NKpy8el73(pJ)_rTlelb*-_7a5TaQ!f6|YMu*iI6?f>nzdy~mai*_x3*ytv|1 zNXJ|R86ZnvQ6)QxnFM6Rc)O}vDp=Oz&Y!7fPq}inU55H6Lk!q6BDZ*5@b2p=n$o9t zIY_&igH(id@G9k|Ei)&YmQ5T1eUuI*c_j`jD7qK4*&GlFv6QImzYh3S^We-j(~AAC z$-X}nMx)J#{1hzjw_VAGSNe_P)~=rCP9r+%x=8_`oGk&ul3TC zYz?lm*2aaTD(77(CB;6rY;WiIsIORAXkTS(h7L0Ky`E`BvrqTtJJinfIRB`uA?zs- z?FTyNkioKutxUACQCW%l3yTg%3XMLPE8{bmZ&w;~>h%q}D?4*bMUdR1X#`~Cx^4D> z;!5#kXV)M44-Lv&(J0Z|F5cKJ8j_7 zNmF^2;U>uP^Ye1bJ-A2B#5{g+4q_{8dfauzTsj>!f|%gv`N8-r#|*D;$%Tgep(7DG zy8ijOh$Jy6z&R>uy&Q)7{XYJum+z=LLYlp*B6Cr^e*VhveFX0S4aOTltH-usqW{wN z5+$L-?*nNYs4Ztbq}ncf#9CVbPLdSVprglEV2J3@34MTf`-U4ebifNGfU=>N=gFRg zc4UCR;``ASm3^UdWPMIh_>O&I6NqeIP+e3~d}e#`QM3-Fge=J-6+h_UG1+Il@fq6)585pk(mAEY%tFIN5!dGE2mS}oWeB3{E_7Yt8N3;GFnJjP31(OPSyPS$n2oIo6FihhRMm0 z1)>hcdZ!wAVRQ03h&pPUZ+?zH&ToQQJ|MZ0DGLz~^86 z1EIlFcI#fNwQWud{-!(^uT>R@@g}|W1)eI0#E{#B?C|)P-&O><^TBhCugj!=i z?oMBjPe|Jf*LXZbJr)zszE@q&1H*=ci@9B4d%num0Q=N_W0LE;QT8M+km&wV_RcYw znHqMZJr?$>r|%`hQx%yLOZ#^AF zz|xU6mMd-)4e&yAmhQduAmME61NY=rr;w-4Z5fz;^{jDNX|gd}_6eR}PZ=C38+q`L z#8hT8w%62_lIPYiP|wAXQJK@XyCoxlqkk_0ve6}aoHY2CVtbiKXWFd*wk6FXn8+#M)3!$#o`q=~Z5LPszdo@%n>Y>I%=f+H=&NA4Z zDmHSXw?zEf-}cqJ8k`LgGWc{jaI3Co9rxJPHuk*y27tfRUl>yN{dPRN%EY)Xk+O+sK_gj(yh~kTX5#xy< zJJ>`msCebY^9hbR&7lNlwD?fJsi7J?|*Q8B5u8_0ior@;IG**k|t51HwmeM)}6gx&yyLr)x~Z-BX& zH-Muq?u*QmsPk|BEBKdc#1~`J!QRHRH-Mqqu73STUykJ#@1>l_5a=a^GGdCgvvAIj zyQ8O{Gs%PAta9Edm!_1A?`0hW)_WNT1AOnCzxU(*t04yjU{iBRVriJW1t%50*Kqw* zJOA7J5yz4Q0YSKP2C7%0u6^1y#n@T$kwe1cgPtp8jW|i1PQS2(++!K^Q?XET8-k2E z!OP~)h#(GeKJp!j3%vV*%Nt)Jd(tr#j)`|8Yysa zQ`P%l7Rg$`hpmEkdYAFh?SeC#)*T(+ZkN2#_(_tUq? z(oyeh0pTom$5{~1hV(;qS8EL&szzw>Kct6#nN?n3k%XzEBNxb~3LkBNue&3{N#^eS zkj|^c8GKv}OY*yb=mBTj0glAMjAxnZ`f=Vl#+Th`sS)2hU0++$i@QYnSHC)4mBwTT z%!pfL0KZvxde42hK40SyU|6&%=g)Cc11XF6A$)Vg&P;$z9~)C~cbI^*8-EH`r8gft zZZjc1O~0oOcLYvB#eSbZg@UXfGLClTbjNYcTpEv=?;D`c0lAo{Pg;qyWRaaCM7?Mz zuVCL(uIIojpO$cf$R$X<=ki-OeR9bW=fMekii9AhpHA{#LH2SO=g*?>TS|wB)qad05U(iie(Ymc}W8drQ&WGSMJfk)yq>0!9GA=c|nLxAxv#UX9 zRcm%eHibN!Ha)db$jv!%kIFuNmmEK(tQNe*gU=z&UcZlRmV0TBHA`$nlnkr-ia%>% zIG8;e49zN{fnSF9rae%so$37)@y>*ky`QKJU%-b-Y; z6Q(Yw?r?~5fuyH%yp|pLX=p7X*qsCs?S83SKU2-lOWYS~5Q1gi0MW$ivZd?6*1OyE zy1IzOQ%$|PqI5W~`v9^QN4_WPH3ryQCyBzp1k(Wn?88jXcN}QFh(zY$9Bb+_0Cf28 zsz|LyO3`Rr)h+b!_kYaIaY=OO(+TBZ$H;|of7z&6VbNk+hI?mZeD_fOMveTDQzjw+ zP2SWT<#rXD>oq(KV26%#jOVBal8_9fl-7FWo zGtm_Fh{5azlDHK$pTMPjNNs)-l&N#4J(83JrlB>=jHp`o;oLe?hD{8LoVgY;lwG5J z$ZuYB(P16LTAJH&dBPB?Rw2YFfJc;OMJcU+s886v>_r$j<2VgMUu+k&AKHWH_oexD z`x8qKJ5SqTwm)hEI9E7i212F&wUVPriLXx4S66gI;@TNx3%K~cCVBE8q;)?_=x-tX znKx{TP;qzzxB*?1mv?p1e8JQB5pYT%7!B|joHk!f*WmBHI^Fey7 z&7_$|oyS4FYio}H%hh(>Rdp%cz6dBd*yyynED4g*W$WSnM1^3*#PlG8y%QrC({yY< zk(n3T;e)8ori2uLy{N;NQA#p3KFmv=nI(xK_RpvSOJb^;!$oj~eEYP&7`_05$pN1R z?GW}GVT?noisugv#my zdE@lCD*W+Gpm_XFUq9G#Uv)NR8SPB)jxbq7m5|5FmW-H<11QrG=B!Y0+9HXD9_Gx% zU6*WGQHb?Us0m1FhzO!2)i^>igxauED-4gzCESo)48*8nZY8nXuetzCHlp@UGFZw+ zIWe%t+$@a>hvZc;8Q?yOqp6;#R04k4+;^F@ibqG18|zOf8(=lDGLy4=j`xI#ZG$>< zLt}QESeE^I|8wMUXL`0(68X+DgM`&Kb!=zvlxG6g$ z-?BB#Lk$T!;T@4{W73PJ?U*96jMME;U}PK3hQkXh;O%tiA?4^;_{)33t!sDIA?(4_iwjQnll*iGW=jkD zyxxuaIHNSXVV>Ponl&RiqY-Zbd64@TL=Zl<=_*SM9WlrZwhm>nr7H|5`By8rc`qG(o-!zM^TATMw(wb_l4iEH+d;xn$KYvQQ*an1ta zTEW#h-De#Ydme_Xq6Qd)sj;3ex1XFbdwv|-A62g+ghfEkr-QR$iT087Gg-<=1;whZ zo|bE@1YvdGi>3Q`}k2D9l-vWO#m56g0H7TzWsvqF5_Y8WPiee^v9Cn7YJ0k^TW<9E@t;In{c7D zl~R-OJg^4t;u>qV7r=Ac@WKD0MJ7mLGUZif%u>qG<@)M|`{{K}X@aC+b7|IXS9c-0 zT6jDZ(Of1Lmgwew+zpQV4hF5XeUeei!{xf7Dz>c;4IY#@M2sq5?waCT5=1qhOT?)p zNiK^povlLObC`HUoO!Y}e4?T4Cs5OWGT37B!cWW6S?j;1sh62NB)F2~M;OjduiNC3 z(?Ass-)8REML~4qu;#uNT};PhTqC^6xn$9evQ(?X70iqkSp?CqA(dl_RVLL+M+V#;={fkCTemtQi6jxq7&bi!UXnYy$(3#A#yrcd4gEpV8RYG}*K9!JgV9PLH3aa`1%-p~z)r)-roo12 zK zLFA7JhOk&mhdTUvFNx37nnU~-YZX-w1Y=eH(s1ed8y4!BqFjet}pGqzr6KMOqrr4;Rl60 zGlS~F6hk?=aIx0w;--UMAJkGPT|K+Gf7&YvRKOcz$AIK&+A|^oH~T&=4(x^u|8`yb}<&xk@UFS zu$f3$Cirm_W(LJwC~C=k&YzSw`$f3I#}UY^&L&MT*T&oi*i2(4X>EVPX^ML#>$|+P zH)FNR`K*sOghVnVZ|lKMun@x%?%PfdJCT(a{~Dy4824}|7|-YW`VxOv@?=rmnfe1z zAjyecV)ko8pB$lrbDWyq06Pje-^9~Qc-P%i`Rz6PzEN4@5_?dRpi*0FQyJD08`P{3 z6m_!g?z3C>!u>nHsZl>U^GgF;aHDbDmudN_%huGM5tg_{BA^#Q=>BEe6t2CocFpNe z1C_}B*!KXgv##*dz$SZIrXAm&`g(Ssv4&eN&3&6mQSwdLh{8c~#7TwD(NpbT&YhjG zS)bP{Or35;o{f^g$_Kx{qA-8b<)n8evw0Gv;an*3Y**G^<`FegbX8fvw@SluJI@a zVTNP3v3;q}i(r&?pm;cXTbf|;@`bUwx*DO2T#M!i7aT@pt$&O!2L;)qKI(wp2JQlj zKasj<^9NP0Q$K0gNsYC;D9V1!@ohNu7n%I|N=6Qsfx$9*hxYBbCw^AfFJ^0H@xt|B z&@DXM?i)Zom*b}||903LK<=epCDr8(0F&HjhO6C=&IMVv8mYS=Y}d}=a3ixLQVb^f zSA@X?8!)pmKAy1uGKdoQY34Z6jMde={Mfn}gQez~6M&Pnv|fCrJgoCPfuEk+SSqEi z^YySGvSYDD@tTY~9w&}QQ&fxtU8+NPnKTXWJkm5K+O|z?=?$U=<2Hn%A`;?P%cW7+zlb{opG5wl z+cg_WJzZiih605=k{;c0cUA z_AL41>!gOeCF0UY*v)4Lv*y4oHJN|r0~_Mhywtze#H|+i{cZQZ{&tT>ey#VG?(o{n z?p^*Tj+8xe&-l2xDgA2v= zNvd~Ul~dzgR&@*J(R|lalM8pQ{;Lv)0RqJVe>R*}N8k%)Vg@Aoy8GvLn{@NYy__8qDmVWQWP;-`XGA-@4$+_o6(34$^f!RQ@`GuG3Lp?(5_Im6L%6V&s(?Q!tEx9Kf%0r6mQxjf!BiKm7 z=V4jBT`v1X-dhqo&l+-f{y&o(miB)ZEB{K+s(-SPSxXzV9Lh9M%j6Q%l5gWYf(cL= z%WQnv145l-T(^RRa1edmao5XwiAQq0Ly3g54ML@a_g`0Gs7j+tTErt1V^ZVM;P_?D zsK*0TU;c40*dQmpY)uWm0hVoehD)lVUnI|muK9*dao$DBdp@f1*@vUXBH!$Xv9jSc zH~bC4(I1CpL3K6{kd_3%^|cz89Nln;@1Hlod*JcC2Obax4i^6XpR4~3JXpYgtNB=x zxK!_fhsUkymPGqOD!8y8LCZWOx$(b)@4ZhX4!E?kU?+$c*bG?G9R8UOU_uuJ%kOPg za&~y(w?>SI@!>VrEmBY3e}Yh=(qVby<+5w@lRM&u5F?XD51Rjrd(IqgtHEP1U}g4k z;9OCRK!js>%AN)m(j>z35pf?I%b_OqKnLa0>QDEj+Ukf2Bs1q6os}_VYh`CNKD+8K zy7bk$WsVtdJL^N>k^kIq_WBeR19c?o<~eyC3>ME?7J>@G3Zr06s>lqvP#nDU!ulT_ z@!{rWKr~#iTYoVgH*`6WXtHrh?CfomWtknV%4L-rCwJ2oBqoao9#b!GD<;4Ou}<&` zuE?lgr{Krz&`eZkiJ0 z?q^KqsXJ*%R|ha0o! z@iD+CcBj)Ws{5qcN1E0$u%V|;xqn6KC|HK`s!{IXyx}jx{XZnr$1pWiNRX1t3C;m34Q!w$+X_TKp>p9{w*`=G(bR|Cq@?i##cE-s3! z-mO&YMs+`J91#0m8cLGV8q2CWb$0tN+_=4ayX-ywxL$`OCl`^l4Q+8pIJFZ9t`Iqs zg%!*ihnA%#eRoZT7)4oQu8n;t-50nQ6@(tce_G6+?xG6Pe`UZA(j{gZxm`S0#@e-k zA=QLKfd`?_31`e%o6R4LZ(s&;dL_kwUCNH2~DqsQvagTm^6+4L*R6T>|MJF$2P zt8X+`k`A5hMx Date: Fri, 22 Sep 2023 09:24:03 +0200 Subject: [PATCH 21/34] Added johndove.jpg --- cms/assets/img/team/johndove.jpg | Bin 0 -> 11492 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cms/assets/img/team/johndove.jpg diff --git a/cms/assets/img/team/johndove.jpg b/cms/assets/img/team/johndove.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1d268e60720fe4ed9d48ddb6fe14fc9684c8e2b3 GIT binary patch literal 11492 zcmbulWmH^2(=9xMySuv$9^56ky9aj(I(UKvcXxuj!{7u6?rwtw53WHH?DD+wt?&N6 zU471@RaLuJ)jn^lZ`%MYMOg(|01yZO0N)Mpwhs6PfO{YCaPSBS@Q6t7hJu2O zjDmuNiuP_;M3~r^?~@3h7$5ijVW1!(p26z7MIr3^8)_AZt(UKfQWonGB?tV411haIvTRf)o1Zj3E16KcOq07Q0n2KwN zbS^QcxKX&$EUI5Misz8FCrT}nv*7w<<59W4ZjUsQtSX&KIX)0CTI8F%*1krJo)&4C zOFmIUX%5LM(|O`*3|h&u{k;|5w*lPW^V1s-`{!Qjj?Hc zv|HBUp*aUpen_SiR&px!8HimVpDw9+T6JnH^GHSNr%vTOxyPaU7RsnQtF@|VonViy zaA|&@$EgGtef{x##w1F!b+YG!xX(Jw*r4P-QdApGsJdEM zBDtc^4@vUS(PWveiIYgUa05=+yMMi;k@*{#z~dFi|r zQ03Zx7eCcQ0_Z|fwMraJI1$FyoHb7FX;-|cwL9?_tR|bFny()d$(4Lh@YLn}&w>ecR@d!p5>rJQX3TxPZXC(-8eFc_ljphjV0j`@Ml_LxRZ21UrB@Aey>2=b zgugV_f|ES3J9`}|KN1$ho2So5G%-(BI=bCJ_%c^hpnp#>i)@^!wA1Eb(I%LsU-1qJ zAOcVO*XQ|GQs;;XD33aYejYzLC+xoV=l@*Z-J;54tc>jzXEo9Qu=jWtc&QIG4Y)ii zAZWK4`y|^NM=^W1kQo7Vbs7(TxV?ZN(E?K)+tc<#o!4-wa>$%ACuZrzI-U27l-JX% z|F}6rF-ju+*?;_n|Dg)`9i%~T;dUO^Va069%Y5KGeV&A6)ruDLIaLGRfn9LHWL^4$ z4W)RqGkWOl8{lI>^g(g+!|Y@OTDT%-t;{dEgm8`#`Wpl0!!&ynx5)3rW!l@bgDm;te;PgJ ziz1D8B$xQN>))+0NO^~Qa;qZU%?b!T<96A#Ngc6p>9@6=`kOQ2p84^5wr zSIblWHxwZJ|Ahnqgn@;7r^^3=0{b5*07TIH%PVW^UuqBlwJoDm$fg1SWHgl%tyV~z zlmHO$iniylUDybx0|;An4QSMCk^2GYyscxiQF%xrXFyyx(e{5jCXU|^w|RJ9jIN=9 zM(t`2gA=S-a_Y+$p|Ax*~<+_r8th&+xaknA&qS1COSk_2#h z<-kr!r~wJ@g-o_z^lutiEIUu3j-NVqi0>%qu->Wk9S7k5qtpL*=-nY$*f>;f>NcWp^Ot+t=>T)i#Q_&McAaPhe6TDvyJ`JzGp+h_HqONwKmbxBX#thY|g~o zgT#5icd<)d#`nUZ<1@U~BXh;sC6H4^BuR!PJmc>wXF{#!K~x@VT9NCp67F|+hwH8h zjkPVqYzCr})G7tm$LB!hxJgUn*_DNn9kXg44QbqC&c6=A!hI@wj%zN=i&}W#Uko;^ zYyqWwnpQ5~V^mT(E%m{|-@x8M*$|YhS)ul@UcrMbEjKRNUckGP_StmE zl0Kf$M@xby_R5a&*bnKsvXrwYtq@1$Y<*G1Ke<;57(5w!pIhX4ohK7h^jp}DS6hq= zlm8SJ9+j91MkScj9aGoAvj|>#LgY8z8KMOx2;EW${JdYw2V`syXc1@ZzFTUUfYsFqS=T;#fTq@ ztP($2687<|TBzEmNew~TeD}5aHdV+9{_$GXJpo4zn4!t^!+zR$I^%V#PY9BnSzdED zdJq!2n$AJ;Qcg7qk%fP341UOBYr%KeEUK1xEY;QXpW6TcY-x0y)#vb|#;30>cW6XY zB0GOkneIwHRD>W=Av%Ep{1tX~b}p9sR@uXXX_hbG!M*qeFT3lUvHgp0q-A$`s0VfS zb{MM#K>!{ee&M4$GS}HAzf*O`pISzf;fYfU@e7aw zSc~Qk<+N8)NQhD^VlGANfWyZJo3C^SUbx?zA3L)U`Z{bSynXQ<{JN50k)(1K$9sXwG)A zR2f|DE{Iq)R`XAv!PQIx)xw*h)oVR695P$YJqG6N3>p#g+5y_iy;YUv9b)z}udRW{ zD-yyOXWNnR({RCa#9{ILa^Np0?(&$W!gjR{PMu0;=fly0pqg^NS9nRzKkmF63DweW z{psnQiVM0MF1NCpCZOs*+pYRU!1X4y0J9L&fBH&?)6vhEOYG05qhmu$(;nGXV=F5w z-QYppl3gu(rCKs1J$PATk1>ser1(0bdu9WBr;ISt#99m%RC#_3vC8UrC1G2xGm-^< z3bK@3u;HwT1W?T0%M#}+FK90N+{X9SoB2fANNKusme~{z17m=4+D?X6-o8sqV@4ze zKK*0-CXfu+G#R%rOU6oRoNU*CJB!qNZ#i6F6S|ixKTY0mGfc5 zBK_*T0ZPB|VQ8~!Nfap4yuj0=uMW^MCbU+^EV3k9@549BMfhfV(mvb>Q6W=!)3Q@mg^$AOiK5ERqfH#zA>>udh7%2bmOq4#&s!P=X-$wtagL;)-*}9?3*Wy1Jvr z(vc3aCHwr@*PI#f%8eAE>{-l|#M!I0BU`dOGA)byz*hK9KZ)zd!vOUm?Omzpw7yi( zoR|Q^oHh&i9zQT`JIb;$bD~3N;&Z&@T%Aa?Qxar=M(oo;v;gSkPb%#ZKTcuPG@{+& zw$R`!ia|c=jO&?pKR*ddnmUVr2y#xs5bvNxQ2`W%x|;#fN()1v#&^Fp`j5={X7-;! zOe^!@=syTTf5a{`zJ7;cIzH=>;i77W3?GJvMWtMr6<47}JdkMKH=q^yd|`R# zdpXa)j;f|R@13otD>)WdmF~~N*-g>7Oyv%9?cq>jh|KRW z7M=WS$C?}d3z@t24bX+m1v^ab7AR6G6TcPjlPit2LIKA=s)H?Utk)`ONP?Gi8yWXs zC#m#6YWmVT*Dvka3m7KF(Mz&$`i(hJ${wnMF59i3uVO^6DE`tcbpCD=@U+)olE`AD zQwF#BHc_K^pynwk>tI}Xs~kdjM*{_d6}ouTu<&AKs$I9ghH4w5Wh*5j$v5k{Yl zJy2$J8in7Wn1}B*!eME*O_X_9d8&BS?~;8cK?$SX*a83XLQGD^b|-01cDPQw+^tPU z$oy9NKpy8el73(pJ)_rTlelb*-_7a5TaQ!f6|YMu*iI6?f>nzdy~mai*_x3*ytv|1 zNXJ|R86ZnvQ6)QxnFM6Rc)O}vDp=Oz&Y!7fPq}inU55H6Lk!q6BDZ*5@b2p=n$o9t zIY_&igH(id@G9k|Ei)&YmQ5T1eUuI*c_j`jD7qK4*&GlFv6QImzYh3S^We-j(~AAC z$-X}nMx)J#{1hzjw_VAGSNe_P)~=rCP9r+%x=8_`oGk&ul3TC zYz?lm*2aaTD(77(CB;6rY;WiIsIORAXkTS(h7L0Ky`E`BvrqTtJJinfIRB`uA?zs- z?FTyNkioKutxUACQCW%l3yTg%3XMLPE8{bmZ&w;~>h%q}D?4*bMUdR1X#`~Cx^4D> z;!5#kXV)M44-Lv&(J0Z|F5cKJ8j_7 zNmF^2;U>uP^Ye1bJ-A2B#5{g+4q_{8dfauzTsj>!f|%gv`N8-r#|*D;$%Tgep(7DG zy8ijOh$Jy6z&R>uy&Q)7{XYJum+z=LLYlp*B6Cr^e*VhveFX0S4aOTltH-usqW{wN z5+$L-?*nNYs4Ztbq}ncf#9CVbPLdSVprglEV2J3@34MTf`-U4ebifNGfU=>N=gFRg zc4UCR;``ASm3^UdWPMIh_>O&I6NqeIP+e3~d}e#`QM3-Fge=J-6+h_UG1+Il@fq6)585pk(mAEY%tFIN5!dGE2mS}oWeB3{E_7Yt8N3;GFnJjP31(OPSyPS$n2oIo6FihhRMm0 z1)>hcdZ!wAVRQ03h&pPUZ+?zH&ToQQJ|MZ0DGLz~^86 z1EIlFcI#fNwQWud{-!(^uT>R@@g}|W1)eI0#E{#B?C|)P-&O><^TBhCugj!=i z?oMBjPe|Jf*LXZbJr)zszE@q&1H*=ci@9B4d%num0Q=N_W0LE;QT8M+km&wV_RcYw znHqMZJr?$>r|%`hQx%yLOZ#^AF zz|xU6mMd-)4e&yAmhQduAmME61NY=rr;w-4Z5fz;^{jDNX|gd}_6eR}PZ=C38+q`L z#8hT8w%62_lIPYiP|wAXQJK@XyCoxlqkk_0ve6}aoHY2CVtbiKXWFd*wk6FXn8+#M)3!$#o`q=~Z5LPszdo@%n>Y>I%=f+H=&NA4Z zDmHSXw?zEf-}cqJ8k`LgGWc{jaI3Co9rxJPHuk*y27tfRUl>yN{dPRN%EY)Xk+O+sK_gj(yh~kTX5#xy< zJJ>`msCebY^9hbR&7lNlwD?fJsi7J?|*Q8B5u8_0ior@;IG**k|t51HwmeM)}6gx&yyLr)x~Z-BX& zH-Muq?u*QmsPk|BEBKdc#1~`J!QRHRH-Mqqu73STUykJ#@1>l_5a=a^GGdCgvvAIj zyQ8O{Gs%PAta9Edm!_1A?`0hW)_WNT1AOnCzxU(*t04yjU{iBRVriJW1t%50*Kqw* zJOA7J5yz4Q0YSKP2C7%0u6^1y#n@T$kwe1cgPtp8jW|i1PQS2(++!K^Q?XET8-k2E z!OP~)h#(GeKJp!j3%vV*%Nt)Jd(tr#j)`|8Yysa zQ`P%l7Rg$`hpmEkdYAFh?SeC#)*T(+ZkN2#_(_tUq? z(oyeh0pTom$5{~1hV(;qS8EL&szzw>Kct6#nN?n3k%XzEBNxb~3LkBNue&3{N#^eS zkj|^c8GKv}OY*yb=mBTj0glAMjAxnZ`f=Vl#+Th`sS)2hU0++$i@QYnSHC)4mBwTT z%!pfL0KZvxde42hK40SyU|6&%=g)Cc11XF6A$)Vg&P;$z9~)C~cbI^*8-EH`r8gft zZZjc1O~0oOcLYvB#eSbZg@UXfGLClTbjNYcTpEv=?;D`c0lAo{Pg;qyWRaaCM7?Mz zuVCL(uIIojpO$cf$R$X<=ki-OeR9bW=fMekii9AhpHA{#LH2SO=g*?>TS|wB)qad05U(iie(Ymc}W8drQ&WGSMJfk)yq>0!9GA=c|nLxAxv#UX9 zRcm%eHibN!Ha)db$jv!%kIFuNmmEK(tQNe*gU=z&UcZlRmV0TBHA`$nlnkr-ia%>% zIG8;e49zN{fnSF9rae%so$37)@y>*ky`QKJU%-b-Y; z6Q(Yw?r?~5fuyH%yp|pLX=p7X*qsCs?S83SKU2-lOWYS~5Q1gi0MW$ivZd?6*1OyE zy1IzOQ%$|PqI5W~`v9^QN4_WPH3ryQCyBzp1k(Wn?88jXcN}QFh(zY$9Bb+_0Cf28 zsz|LyO3`Rr)h+b!_kYaIaY=OO(+TBZ$H;|of7z&6VbNk+hI?mZeD_fOMveTDQzjw+ zP2SWT<#rXD>oq(KV26%#jOVBal8_9fl-7FWo zGtm_Fh{5azlDHK$pTMPjNNs)-l&N#4J(83JrlB>=jHp`o;oLe?hD{8LoVgY;lwG5J z$ZuYB(P16LTAJH&dBPB?Rw2YFfJc;OMJcU+s886v>_r$j<2VgMUu+k&AKHWH_oexD z`x8qKJ5SqTwm)hEI9E7i212F&wUVPriLXx4S66gI;@TNx3%K~cCVBE8q;)?_=x-tX znKx{TP;qzzxB*?1mv?p1e8JQB5pYT%7!B|joHk!f*WmBHI^Fey7 z&7_$|oyS4FYio}H%hh(>Rdp%cz6dBd*yyynED4g*W$WSnM1^3*#PlG8y%QrC({yY< zk(n3T;e)8ori2uLy{N;NQA#p3KFmv=nI(xK_RpvSOJb^;!$oj~eEYP&7`_05$pN1R z?GW}GVT?noisugv#my zdE@lCD*W+Gpm_XFUq9G#Uv)NR8SPB)jxbq7m5|5FmW-H<11QrG=B!Y0+9HXD9_Gx% zU6*WGQHb?Us0m1FhzO!2)i^>igxauED-4gzCESo)48*8nZY8nXuetzCHlp@UGFZw+ zIWe%t+$@a>hvZc;8Q?yOqp6;#R04k4+;^F@ibqG18|zOf8(=lDGLy4=j`xI#ZG$>< zLt}QESeE^I|8wMUXL`0(68X+DgM`&Kb!=zvlxG6g$ z-?BB#Lk$T!;T@4{W73PJ?U*96jMME;U}PK3hQkXh;O%tiA?4^;_{)33t!sDIA?(4_iwjQnll*iGW=jkD zyxxuaIHNSXVV>Ponl&RiqY-Zbd64@TL=Zl<=_*SM9WlrZwhm>nr7H|5`By8rc`qG(o-!zM^TATMw(wb_l4iEH+d;xn$KYvQQ*an1ta zTEW#h-De#Ydme_Xq6Qd)sj;3ex1XFbdwv|-A62g+ghfEkr-QR$iT087Gg-<=1;whZ zo|bE@1YvdGi>3Q`}k2D9l-vWO#m56g0H7TzWsvqF5_Y8WPiee^v9Cn7YJ0k^TW<9E@t;In{c7D zl~R-OJg^4t;u>qV7r=Ac@WKD0MJ7mLGUZif%u>qG<@)M|`{{K}X@aC+b7|IXS9c-0 zT6jDZ(Of1Lmgwew+zpQV4hF5XeUeei!{xf7Dz>c;4IY#@M2sq5?waCT5=1qhOT?)p zNiK^povlLObC`HUoO!Y}e4?T4Cs5OWGT37B!cWW6S?j;1sh62NB)F2~M;OjduiNC3 z(?Ass-)8REML~4qu;#uNT};PhTqC^6xn$9evQ(?X70iqkSp?CqA(dl_RVLL+M+V#;={fkCTemtQi6jxq7&bi!UXnYy$(3#A#yrcd4gEpV8RYG}*K9!JgV9PLH3aa`1%-p~z)r)-roo12 zK zLFA7JhOk&mhdTUvFNx37nnU~-YZX-w1Y=eH(s1ed8y4!BqFjet}pGqzr6KMOqrr4;Rl60 zGlS~F6hk?=aIx0w;--UMAJkGPT|K+Gf7&YvRKOcz$AIK&+A|^oH~T&=4(x^u|8`yb}<&xk@UFS zu$f3$Cirm_W(LJwC~C=k&YzSw`$f3I#}UY^&L&MT*T&oi*i2(4X>EVPX^ML#>$|+P zH)FNR`K*sOghVnVZ|lKMun@x%?%PfdJCT(a{~Dy4824}|7|-YW`VxOv@?=rmnfe1z zAjyecV)ko8pB$lrbDWyq06Pje-^9~Qc-P%i`Rz6PzEN4@5_?dRpi*0FQyJD08`P{3 z6m_!g?z3C>!u>nHsZl>U^GgF;aHDbDmudN_%huGM5tg_{BA^#Q=>BEe6t2CocFpNe z1C_}B*!KXgv##*dz$SZIrXAm&`g(Ssv4&eN&3&6mQSwdLh{8c~#7TwD(NpbT&YhjG zS)bP{Or35;o{f^g$_Kx{qA-8b<)n8evw0Gv;an*3Y**G^<`FegbX8fvw@SluJI@a zVTNP3v3;q}i(r&?pm;cXTbf|;@`bUwx*DO2T#M!i7aT@pt$&O!2L;)qKI(wp2JQlj zKasj<^9NP0Q$K0gNsYC;D9V1!@ohNu7n%I|N=6Qsfx$9*hxYBbCw^AfFJ^0H@xt|B z&@DXM?i)Zom*b}||903LK<=epCDr8(0F&HjhO6C=&IMVv8mYS=Y}d}=a3ixLQVb^f zSA@X?8!)pmKAy1uGKdoQY34Z6jMde={Mfn}gQez~6M&Pnv|fCrJgoCPfuEk+SSqEi z^YySGvSYDD@tTY~9w&}QQ&fxtU8+NPnKTXWJkm5K+O|z?=?$U=<2Hn%A`;?P%cW7+zlb{opG5wl z+cg_WJzZiih605=k{;c0cUA z_AL41>!gOeCF0UY*v)4Lv*y4oHJN|r0~_Mhywtze#H|+i{cZQZ{&tT>ey#VG?(o{n z?p^*Tj+8xe&-l2xDgA2v= zNvd~Ul~dzgR&@*J(R|lalM8pQ{;Lv)0RqJVe>R*}N8k%)Vg@Aoy8GvLn{@NYy__8qDmVWQWP;-`XGA-@4$+_o6(34$^f!RQ@`GuG3Lp?(5_Im6L%6V&s(?Q!tEx9Kf%0r6mQxjf!BiKm7 z=V4jBT`v1X-dhqo&l+-f{y&o(miB)ZEB{K+s(-SPSxXzV9Lh9M%j6Q%l5gWYf(cL= z%WQnv145l-T(^RRa1edmao5XwiAQq0Ly3g54ML@a_g`0Gs7j+tTErt1V^ZVM;P_?D zsK*0TU;c40*dQmpY)uWm0hVoehD)lVUnI|muK9*dao$DBdp@f1*@vUXBH!$Xv9jSc zH~bC4(I1CpL3K6{kd_3%^|cz89Nln;@1Hlod*JcC2Obax4i^6XpR4~3JXpYgtNB=x zxK!_fhsUkymPGqOD!8y8LCZWOx$(b)@4ZhX4!E?kU?+$c*bG?G9R8UOU_uuJ%kOPg za&~y(w?>SI@!>VrEmBY3e}Yh=(qVby<+5w@lRM&u5F?XD51Rjrd(IqgtHEP1U}g4k z;9OCRK!js>%AN)m(j>z35pf?I%b_OqKnLa0>QDEj+Ukf2Bs1q6os}_VYh`CNKD+8K zy7bk$WsVtdJL^N>k^kIq_WBeR19c?o<~eyC3>ME?7J>@G3Zr06s>lqvP#nDU!ulT_ z@!{rWKr~#iTYoVgH*`6WXtHrh?CfomWtknV%4L-rCwJ2oBqoao9#b!GD<;4Ou}<&` zuE?lgr{Krz&`eZkiJ0 z?q^KqsXJ*%R|ha0o! z@iD+CcBj)Ws{5qcN1E0$u%V|;xqn6KC|HK`s!{IXyx}jx{XZnr$1pWiNRX1t3C;m34Q!w$+X_TKp>p9{w*`=G(bR|Cq@?i##cE-s3! z-mO&YMs+`J91#0m8cLGV8q2CWb$0tN+_=4ayX-ywxL$`OCl`^l4Q+8pIJFZ9t`Iqs zg%!*ihnA%#eRoZT7)4oQu8n;t-50nQ6@(tce_G6+?xG6Pe`UZA(j{gZxm`S0#@e-k zA=QLKfd`?_31`e%o6R4LZ(s&;dL_kwUCNH2~DqsQvagTm^6+4L*R6T>|MJF$2P zt8X+`k`A5hMx Date: Fri, 22 Sep 2023 09:26:32 +0200 Subject: [PATCH 22/34] Change role Leena, Cenyu --- cms/data/team.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/cms/data/team.yml b/cms/data/team.yml index 09de2d0931..e3a0b3df8e 100644 --- a/cms/data/team.yml +++ b/cms/data/team.yml @@ -11,7 +11,7 @@ 2022: https://drive.google.com/file/d/1E45ycyctDfYkh65ZCM8PMtQsYym_eP6L/view?usp=sharing - name: Cenyu Shen - role: Quality Team Lead and Managing Editor + role: Deputy Head of Editorial (Quality) photo: cenyu.jpg bio: 'Cenyu holds a PhD in Information Systems Science at Hanken School of Economics in Finland. She has spent around seven years on Open Access research with a particular focus on gold OA publishing concerning journals and publishers outside the mainstream. She was one of three DOAJ ambassadors for China from 2016 to 2017.' coi: @@ -59,7 +59,7 @@ 2022: https://drive.google.com/file/d/1-3xzwkHMclREgLhj_XNF5n6Nr4q2_bnw/view?usp=sharing - name: John G. Dove - role: DOAJ Advisor + role: Advisor photo: johndove.jpg bio: "John has had a career in executive management, and is now an independent consultant and open access advocate who works with organisations seeking to accelerate their transition to open access. He advises both for-profits and non-profits, and has a particular interest in identifying the steps necessary to flip an entire discipline’s scholarly record to open access. His ambassador activities focus on increasing the support to DOAJ from the community. He served for six years on NISO’s Information Discovery and Interchange Topic Committee, and has written for Learned Publishing, Against the Grain, and Scholarly Kitchen. John serves on the Board of Trustees of his local public library in Revere, Massachusetts. He has a B.A. in Mathematics from Oberlin College." coi: @@ -100,7 +100,7 @@ 2022: https://drive.google.com/file/d/1fRJtvci2_j4vad0C5N1pfqm2sHZQkFz3/view?usp=sharing - name: Leena Shah - role: Managing Editor and Ambassador + role: Deputy Head of Editorial (Workflow) and Ambassador photo: leena.jpg bio: "Leena joined the DOAJ team in 2016 as an Ambassador for India before becoming a Managing Editor. Prior to joining DOAJ she worked as a science librarian at Nanyang Technological University, Singapore, where she developed a keen interest in scholarly communication & open science. A recent addition to her interests is artificial intelligence in scholarly communication. Leena holds a Master’s degree in Information Studies and lives in Singapore. She loves watching sci-fi shows and is enthusiastic about travelling to new places." coi: @@ -108,13 +108,6 @@ 2020: https://drive.google.com/file/d/1zU-lLB5W54E_QUm5uto5tqB6cZl83TAJ/view?usp=sharing 2022: https://drive.google.com/file/d/19rw-naMJqHkI5T7aDIDPUkwPutBdDpDm/view?usp=sharing -- name: Luis Montilla - role: Managing Editor - photo: luis.jpeg - bio: "Luis is a marine ecologist with a passion for improving the quality of scientific publishing. After finishing his Masters in Venezuela, he spent three years in Italy completing his PhD studying marine microbial symbioses in seagrass beds. In his free time, he enjoys reading and watching movies." - coi: - 2023: https://drive.google.com/file/d/1IJhnV2Ht5t5jilaCAFzpuFdYk7UMOjN3/view?usp=sharing - - name: Mahmoud Khalifa role: Managing Editor and Ambassador photo: mahmoud-new.jpg From e5cdcbdf4b97cc14a9eea95e342425cca0e8bfb9 Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Sun, 24 Sep 2023 18:51:21 +0100 Subject: [PATCH 23/34] Reduce gunicorn worker count --- deploy/doaj_gunicorn_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/doaj_gunicorn_config.py b/deploy/doaj_gunicorn_config.py index f9425de5e5..a08dd6ef62 100644 --- a/deploy/doaj_gunicorn_config.py +++ b/deploy/doaj_gunicorn_config.py @@ -1,7 +1,7 @@ import multiprocessing bind = "0.0.0.0:5050" -workers = multiprocessing.cpu_count() * 8 + 1 +workers = multiprocessing.cpu_count() * 6 + 1 proc_name = 'doaj' max_requests = 1000 @@ -13,4 +13,4 @@ max_requests_jitter = 100 timeout = 40 -graceful_timeout = 40 \ No newline at end of file +graceful_timeout = 40 From 33188dbf09a2baf89eb8788525a1cfa92b224490 Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Mon, 25 Sep 2023 09:30:31 +0200 Subject: [PATCH 24/34] Edits: 26-31 --- portality/templates/publisher/preservation.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portality/templates/publisher/preservation.html b/portality/templates/publisher/preservation.html index c87a2fcb97..611eb380af 100644 --- a/portality/templates/publisher/preservation.html +++ b/portality/templates/publisher/preservation.html @@ -25,10 +25,10 @@

Guidance before uploading your file

  1. Only the full text of articles whose metadata is already uploaded to DOAJ can be sent to us. Check that your article metadata appears in DOAJ first.
  2. Only articles for journals indexed in DOAJ can be uploaded. -
  3. Collect the full texts into a package consisting of folders and files.
    4. -
  4. Compress the package into a ZIP file.
    5. -
  5. Upload the zipped package (on this page).
    6. -
  6. Check that the file has uploaded correctly in the History of Uploads section and is not bigger than 50MB.
    7. +
  7. Collect the full text files into a package containing folders and files.
    8. +
  8. Compress the package into a ZIP file. Keep the name of the file simple: avoid spaces, hyphens, underscores, special characters, etc
    9. +
  9. Upload the zipped package (on this page). It may not be bigger than 50MB.
    10. +
  10. Check that the file has uploaded correctly in the History of Uploads section.

The package must have the following structure:

From e60cbfa9a85be762e5c3129a9df684657bcf7b6a Mon Sep 17 00:00:00 2001 From: Dom Mitchell Date: Tue, 26 Sep 2023 12:21:37 +0200 Subject: [PATCH 25/34] Corrected license link --- cms/pages/legal/terms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/pages/legal/terms.md b/cms/pages/legal/terms.md index ad872453eb..1b27b66e99 100644 --- a/cms/pages/legal/terms.md +++ b/cms/pages/legal/terms.md @@ -32,7 +32,7 @@ DOAJ uses a variety of licenses for the different parts of its website and the c + In our [OAI-PMH feed](/docs/oai-pmh) + In the [full data dump of all article metadata](/docs/public-data-dump/). -4. The *open source software* that DOAJ is built with is licensed under [an Apache license Version 2](https://github.com/DOAJ/doaj/blob/a6fc2bee499b5a8a1f24fb098acfb8e10bd72503/portality/static/vendor/select2-3.5.4/LICENSE). +4. The *open source software* that DOAJ is built with is licensed under [an Apache license Version 2](https://github.com/DOAJ/doaj/blob/develop/LICENSE). --- From 9179c74d558bf82334cd5186be0ecf45335b6649 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 28 Sep 2023 15:16:09 +0100 Subject: [PATCH 26/34] Version bump for static pages release --- portality/settings.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 7301701e2b..3dcc0ec592 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.3.17" +DOAJ_VERSION = "6.4.0" API_VERSION = "3.0.1" ###################################### diff --git a/setup.py b/setup.py index fbcac71bc0..2c36468273 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='6.3.17', + version='6.4.0', packages=find_packages(), install_requires=[ "awscli==1.20.50", From 1f46ae902418212325d2861e5efcda13c95147f8 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Thu, 28 Sep 2023 15:17:23 +0100 Subject: [PATCH 27/34] Reinstate hourly journalcsv --- portality/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/settings.py b/portality/settings.py index 3dcc0ec592..0d1edcc409 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -427,7 +427,7 @@ HUEY_SCHEDULE = { "sitemap": {"month": "*", "day": "*", "day_of_week": "*", "hour": "8", "minute": "0"}, "reporting": {"month": "*", "day": "1", "day_of_week": "*", "hour": "0", "minute": "0"}, - "journal_csv": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*/2", "minute": "20"}, + "journal_csv": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "20"}, "read_news": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "30"}, "article_cleanup_sync": {"month": "*", "day": "2", "day_of_week": "*", "hour": "0", "minute": "0"}, "async_workflow_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "0"}, From 3c644ae361aeaf69de851f19e8ba132bb3c1d5b4 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Fri, 29 Sep 2023 00:37:25 +0100 Subject: [PATCH 28/34] Some test fixes for ArticleAcceptible rules --- .../test_bll_article_batch_create_article.py | 35 ++++++++++++------- portality/bll/services/article.py | 3 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/doajtest/unit/test_bll_article_batch_create_article.py b/doajtest/unit/test_bll_article_batch_create_article.py index 6cda9ee82c..e397725089 100644 --- a/doajtest/unit/test_bll_article_batch_create_article.py +++ b/doajtest/unit/test_bll_article_batch_create_article.py @@ -5,7 +5,7 @@ from doajtest.helpers import DoajTestCase from portality.bll import DOAJ from portality.bll import exceptions -from portality.models import Article, Account,Journal +from portality.models import Article, Account, Journal from portality.lib.paths import rel2abs from doajtest.mocks.bll_article import BLLArticleMockFactory from doajtest.mocks.model_Article import ModelArticleMockFactory @@ -37,12 +37,14 @@ def setUp(self): self._get_duplicate = self.svc.get_duplicate self._issn_ownership_status = self.svc.issn_ownership_status self._get_journal = Article.get_journal + self._find_by_issn_exact = Journal.find_by_issn_exact def tearDown(self): self.svc.is_legitimate_owner = self._is_legitimate_owner self.svc.get_duplicate = self._get_duplicate self.svc.issn_ownership_status = self._issn_ownership_status Article.get_journal = self._get_journal + Journal.find_by_issn = self._find_by_issn_exact super(TestBLLArticleBatchCreateArticle, self).tearDown() @parameterized.expand(load_cases) @@ -118,8 +120,8 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "0", "pissn" : "0000-0000", "eissn" : "0000-0001"}) + # We always need a journal to exist for an article to be created + journal_specs.append({"title" : "0", "pissn" : "0000-0000", "eissn" : "0000-0001"}) # another with a DOI and no fulltext source = ArticleFixtureFactory.make_article_source( @@ -132,8 +134,7 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "1", "pissn" : "1111-1112", "eissn" : "1111-1111"}) + journal_specs.append({"title" : "1", "pissn" : "1111-1112", "eissn" : "1111-1111"}) # one with a fulltext and no DOI source = ArticleFixtureFactory.make_article_source( @@ -146,8 +147,7 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "2", "pissn" : "2222-2222", "eissn" : "2222-2223"}) + journal_specs.append({"title" : "2", "pissn" : "2222-2222", "eissn" : "2222-2223"}) # another one with a fulltext and no DOI source = ArticleFixtureFactory.make_article_source( @@ -160,8 +160,7 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "3", "pissn" : "3333-3333", "eissn" : "3333-3334"}) + journal_specs.append({"title" : "3", "pissn" : "3333-3333", "eissn" : "3333-3334"}) last_issn = "3333-3333" last_doi = "10.123/abc/1" @@ -180,8 +179,7 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "4", "pissn" : "4444-4444", "eissn" : "4444-4445"}) + journal_specs.append({"title" : "4", "pissn" : "4444-4444", "eissn" : "4444-4445"}) # one with a duplicated Fulltext source = ArticleFixtureFactory.make_article_source( @@ -194,8 +192,7 @@ def test_01_batch_create_article(self, name, kwargs): article = Article(**source) article.set_id() articles.append(article) - if add_journal_info: - journal_specs.append({"title" : "5", "pissn" : "5555-5555", "eissn" : "5555-5556"}) + journal_specs.append({"title" : "5", "pissn" : "5555-5555", "eissn" : "5555-5556"}) ilo_mock = None if account_arg == "owner": @@ -224,6 +221,18 @@ def test_01_batch_create_article(self, name, kwargs): gj_mock = ModelArticleMockFactory.get_journal(journal_specs, in_doaj=journal_in_doaj) Article.get_journal = gj_mock + # We need the journal to be in the index for the ArticleAcceptable checks FIXME: too slow, mock this + #[Journal(**js['instance']).save(blocking=True) for js in journal_specs] + + # We need to retrieve the correct Journal by its ISSNs + def mock_find(issns: list, in_doaj=None, max=2): + for j in journal_specs: + if sorted([j['eissn'], j['pissn']]) == sorted(issns): + return [j['instance']] + return [] + + Journal.find_by_issn_exact = mock_find + ########################################################### # Execution diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 777d499636..63d73055f3 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -282,7 +282,8 @@ def match_journal_with_validation(article_bibjson: models.ArticleBibJSON): if eissn is not None: issns.append(eissn) - journal = models.Journal.find_by_issn_exact(issns, True) + # Find an exact match, whether in_doaj or not + journal = models.Journal.find_by_issn_exact(issns) # check if only one journal matches pissn and eissn and if they are in the correct fields # no need to check eissn, if pissn matches, pissn and eissn are different and only 1 journal has been found - then eissn matches too From aadea74c57a3cbcbe041ad88a1b285487c4b72c9 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Fri, 29 Sep 2023 02:45:02 +0100 Subject: [PATCH 29/34] Some more test fixes for ArticleAcceptible rules --- .../unit/test_tasks_ingestDOAJarticles.py | 14 +++++------ portality/bll/services/article.py | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/doajtest/unit/test_tasks_ingestDOAJarticles.py b/doajtest/unit/test_tasks_ingestDOAJarticles.py index 2872124a47..a2eb5f2be9 100644 --- a/doajtest/unit/test_tasks_ingestDOAJarticles.py +++ b/doajtest/unit/test_tasks_ingestDOAJarticles.py @@ -1260,10 +1260,10 @@ def test_40_doaj_2_journals_different_owners_issn_each_fail(self): found = [a for a in models.Article.find_by_issns(["1234-5678", "9876-5432"])] assert len(found) == 0 - def test_41_doaj_2_journals_same_owner_issn_each_success(self): + def test_41_doaj_2_journals_same_owner_issn_each_fail(self): # Create 2 journals with the same owner, each with one different issn. The article's 2 issns # match each of these issns - # We expect a successful article ingest + # We expect a failed article ingest - articles must match only ONE journal j1 = models.Journal() j1.set_owner("testowner") bj1 = j1.bibjson() @@ -1301,18 +1301,18 @@ def test_41_doaj_2_journals_same_owner_issn_each_success(self): fu = models.FileUpload.pull(id) assert fu is not None - assert fu.status == "processed" - assert fu.imported == 1 + assert fu.status == "failed" + assert fu.imported == 0 assert fu.updates == 0 - assert fu.new == 1 + assert fu.new == 0 fr = fu.failure_reasons assert len(fr.get("shared", [])) == 0 assert len(fr.get("unowned", [])) == 0 - assert len(fr.get("unmatched", [])) == 0 + assert len(fr.get("unmatched", [])) == 2 # error message for each article found = [a for a in models.Article.find_by_issns(["1234-5678", "9876-5432"])] - assert len(found) == 1 + assert len(found) == 0 def test_42_doaj_2_journals_different_owners_different_issns_mixed_article_fail(self): # Create 2 different journals with different owners and different issns (2 each). diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 63d73055f3..7737be124b 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -56,6 +56,9 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d all_unowned = set() all_unmatched = set() + # Generic error message if we don't have a specific reason + error_msg = Messages.EXCEPTION_ARTICLE_BATCH_FAIL + for article in articles: try: # ~~!ArticleBatchCreate:Feature->ArticleCreate:Feature~~ @@ -67,6 +70,10 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d dry_run=True) except (exceptions.ArticleMergeConflict, exceptions.ConfigurationException): raise exceptions.IngestException(message=Messages.EXCEPTION_ARTICLE_BATCH_CONFLICT) + except exceptions.ArticleNotAcceptable as e: + # The ArticleNotAcceptable exception is a superset of reasons we can't match a journal to this article + error_msg = e.message + result = {'fail': 1, 'unmatched': set(article.bibjson().issns())} success += result.get("success", 0) fail += result.get("fail", 0) @@ -90,7 +97,7 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d # return some stats on the import return report else: - raise exceptions.IngestException(message=Messages.EXCEPTION_ARTICLE_BATCH_FAIL, result=report) + raise exceptions.IngestException(message=error_msg, result=report) @staticmethod def _batch_contains_duplicates(articles): @@ -201,18 +208,18 @@ def create_article(self, article, account, duplicate_check=True, merge_duplicate {"arg": update_article_id, "instance": str, "allow_none": True, "arg_name": "update_article_id"} ], exceptions.ArgumentException) - # quickly validate that the article is acceptable - it must have a DOI and/or a fulltext - # this raises an exception if the article is not acceptable, containing all the relevant validation details + has_permissions_result = self.has_permissions(account, article, limit_to_account) + if isinstance(has_permissions_result, dict): + return has_permissions_result + # Validate that the article is acceptable: it must have a DOI and/or a fulltext & match only one in_doaj journal + # this raises an exception if the article is not acceptable, containing all the relevant validation details + # We do this after the permissions check because that gives a detailed result whereas this throws an exception try: self.is_acceptable(article) except Exception as e: raise e - has_permissions_result = self.has_permissions(account, article, limit_to_account) - if isinstance(has_permissions_result,dict): - return has_permissions_result - is_update = 0 if duplicate_check: # ~~!ArticleCreate:Feature->ArticleDeduplication:Feature~~ @@ -395,6 +402,10 @@ def issn_ownership_status(article, owner): issns = b.get_identifiers(b.P_ISSN) issns += b.get_identifiers(b.E_ISSN) + # FIXME: Duplicate check due to inconsistent control flow (result vs exception) + if len(issns) == 0: + raise exceptions.ArticleNotAcceptable(message=Messages.EXCEPTION_NO_ISSNS) + owned = [] shared = [] unowned = [] From 07a975bf6a2d7b181b309b9ff74c72383304cb5b Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Fri, 29 Sep 2023 15:55:29 +0100 Subject: [PATCH 30/34] Delay raising the ArticleNotAcceptible error to similar location to the IngestExeption --- portality/bll/services/article.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/portality/bll/services/article.py b/portality/bll/services/article.py index 7737be124b..b5e829cd24 100644 --- a/portality/bll/services/article.py +++ b/portality/bll/services/article.py @@ -56,8 +56,8 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d all_unowned = set() all_unmatched = set() - # Generic error message if we don't have a specific reason - error_msg = Messages.EXCEPTION_ARTICLE_BATCH_FAIL + # Hold on to the exception so we can raise it later + e_not_acceptable = None for article in articles: try: @@ -72,7 +72,7 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d raise exceptions.IngestException(message=Messages.EXCEPTION_ARTICLE_BATCH_CONFLICT) except exceptions.ArticleNotAcceptable as e: # The ArticleNotAcceptable exception is a superset of reasons we can't match a journal to this article - error_msg = e.message + e_not_acceptable = e result = {'fail': 1, 'unmatched': set(article.bibjson().issns())} success += result.get("success", 0) @@ -97,7 +97,9 @@ def batch_create_articles(self, articles, account, duplicate_check=True, merge_d # return some stats on the import return report else: - raise exceptions.IngestException(message=error_msg, result=report) + if e_not_acceptable is not None: + raise exceptions.ArticleNotAcceptable(message=e_not_acceptable.message, result=report) + raise exceptions.IngestException(message=Messages.EXCEPTION_ARTICLE_BATCH_FAIL, result=report) @staticmethod def _batch_contains_duplicates(articles): From 220aa856bfa54f942eafeb43517dd0851a421059 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Mon, 2 Oct 2023 09:31:28 +0100 Subject: [PATCH 31/34] A bit more work on control flow for ArticleNotAcceptable --- doajtest/unit/test_bll_article_create_article.py | 1 - .../unit/test_tasks_ingestCrossref442Articles.py | 16 ++++++++-------- .../unit/test_tasks_ingestCrossref531Articles.py | 3 ++- portality/bll/exceptions.py | 1 + portality/tasks/ingestarticles.py | 15 ++++++++++----- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/doajtest/unit/test_bll_article_create_article.py b/doajtest/unit/test_bll_article_create_article.py index f595a1b96e..d9d524efe7 100644 --- a/doajtest/unit/test_bll_article_create_article.py +++ b/doajtest/unit/test_bll_article_create_article.py @@ -35,7 +35,6 @@ def setUp(self): self.prepare_update_admin = self.svc._prepare_update_admin self.prepare_update_publisher = self.svc._prepare_update_publisher - def tearDown(self): super(TestBLLArticleCreateArticle, self).tearDown() diff --git a/doajtest/unit/test_tasks_ingestCrossref442Articles.py b/doajtest/unit/test_tasks_ingestCrossref442Articles.py index 2714b33644..ed2236552c 100644 --- a/doajtest/unit/test_tasks_ingestCrossref442Articles.py +++ b/doajtest/unit/test_tasks_ingestCrossref442Articles.py @@ -1315,11 +1315,11 @@ def test_40_crossref_2_journals_different_owners_issn_each_fail(self): found = [a for a in models.Article.find_by_issns(["1234-5678", "9876-5432"])] assert len(found) == 0 - def test_41_crossref_2_journals_same_owner_issn_each_success(self): + def test_41_crossref_2_journals_same_owner_issn_each_fail(self): etree.XMLSchema = self.mock_load_schema # Create 2 journals with the same owner, each with one different issn. The article's 2 issns # match each of these issns - # We expect a successful article ingest + # We expect a failed ingest - an article must match with only ONE journal j1 = models.Journal() j1.set_owner("testowner") @@ -1365,19 +1365,19 @@ def test_41_crossref_2_journals_same_owner_issn_each_success(self): fu = models.FileUpload.pull(id) assert fu is not None - assert fu.status == "processed" - assert fu.imported == 1 + assert fu.status == "failed" + assert fu.imported == 0 assert fu.updates == 0 - assert fu.new == 1 + assert fu.new == 0 fr = fu.failure_reasons + assert len(fr) > 0 assert len(fr.get("shared", [])) == 0 assert len(fr.get("unowned", [])) == 0 - assert len(fr.get("unmatched", [])) == 0 + assert len(fr.get("unmatched", [])) == 2 found = [a for a in models.Article.find_by_issns(["1234-5678", "9876-5432"])] - assert len(found) == 1 - + assert len(found) == 0 def test_42_crossref_2_journals_different_owners_different_issns_mixed_article_fail(self): etree.XMLSchema = self.mock_load_schema diff --git a/doajtest/unit/test_tasks_ingestCrossref531Articles.py b/doajtest/unit/test_tasks_ingestCrossref531Articles.py index 27308a3d22..09edcf1b1d 100644 --- a/doajtest/unit/test_tasks_ingestCrossref531Articles.py +++ b/doajtest/unit/test_tasks_ingestCrossref531Articles.py @@ -624,7 +624,7 @@ def test_23_crossref_process_success(self): j.set_owner("testowner") bj = j.bibjson() bj.add_identifier(bj.P_ISSN, "1234-5678") - j.save() + j.save(blocking=True) asource = AccountFixtureFactory.make_publisher_source() account = models.Account(**asource) @@ -634,6 +634,7 @@ def test_23_crossref_process_success(self): # push an article to initialise the mappings source = ArticleFixtureFactory.make_article_source() article = models.Article(**source) + article.bibjson().add_identifier(bj.P_ISSN, "1234-5678") article.save(blocking=True) article.delete() models.Article.blockdeleted(article.id) diff --git a/portality/bll/exceptions.py b/portality/bll/exceptions.py index 3bb676f984..005ad7f31c 100644 --- a/portality/bll/exceptions.py +++ b/portality/bll/exceptions.py @@ -66,6 +66,7 @@ class ArticleNotAcceptable(Exception): """ def __init__(self, *args, **kwargs): self.message = kwargs.get("message", "") + self.result = kwargs.get("result", {}) super(ArticleNotAcceptable, self).__init__(*args) def __str__(self): diff --git a/portality/tasks/ingestarticles.py b/portality/tasks/ingestarticles.py index de6991ab40..e798f4005d 100644 --- a/portality/tasks/ingestarticles.py +++ b/portality/tasks/ingestarticles.py @@ -312,11 +312,16 @@ def _process(self, file_upload: models.FileUpload): for article in articles: article.set_upload_id(file_upload.id) result = articleService.batch_create_articles(articles, account, add_journal_info=True) - except (IngestException, CrosswalkException) as e: - job.add_audit_message("IngestException: {msg}. Inner message: {inner}. Stack: {x}" - .format(msg=e.message, inner=e.inner_message, x=e.trace())) + except (IngestException, CrosswalkException, ArticleNotAcceptable) as e: + if hasattr(e, 'inner_message'): + job.add_audit_message("{exception}: {msg}. Inner message: {inner}. Stack: {x}" + .format(exception=e.__class__.__name__, msg=e.message, inner=e.inner_message, x=e.trace())) + file_upload.failed(e.message, e.inner_message) + else: + job.add_audit_message("{exception}: {msg}.".format(exception=e.__class__.__name__, msg=e.message)) + file_upload.failed(e.message) + job.outcome_fail() - file_upload.failed(e.message, e.inner_message) result = e.result try: file_failed(path) @@ -324,7 +329,7 @@ def _process(self, file_upload: models.FileUpload): except: job.add_audit_message("Error cleaning up file which caused IngestException: {x}" .format(x=traceback.format_exc())) - except (DuplicateArticleException, ArticleNotAcceptable) as e: + except DuplicateArticleException as e: job.add_audit_message(str(e)) job.outcome_fail() file_upload.failed(str(e)) From 6f04d1a9a9dfc62ad150517bea8bdd3c370d2372 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Mon, 2 Oct 2023 10:45:58 +0100 Subject: [PATCH 32/34] Correct harvester fixtures for EISSN / PISSN the right way around --- doajtest/unit/resources/harvester_resp.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doajtest/unit/resources/harvester_resp.json b/doajtest/unit/resources/harvester_resp.json index dc24cb7dd9..133fedaf24 100644 --- a/doajtest/unit/resources/harvester_resp.json +++ b/doajtest/unit/resources/harvester_resp.json @@ -45,8 +45,8 @@ "journal": { "title": "My Journal", "medlineAbbreviation": "My Jour", - "essn": "1234-5678", - "issn": "9876-5432", + "issn": "1234-5678", + "essn": "9876-5432", "isoabbreviation": "My Jour", "nlmid": "123456789" } @@ -143,8 +143,8 @@ "journal": { "title": "My Journal", "medlineAbbreviation": "My Jour", - "essn": "1234-5678", - "issn": "9876-5432", + "issn": "1234-5678", + "essn": "9876-5432", "isoabbreviation": "My Jour", "nlmid": "123456789" } From 26e825bddf94016def9a3cfef8cfa87fde4729f4 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Mon, 2 Oct 2023 12:15:11 +0100 Subject: [PATCH 33/34] Fix mis-restored function in teardown --- doajtest/unit/test_bll_article_batch_create_article.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doajtest/unit/test_bll_article_batch_create_article.py b/doajtest/unit/test_bll_article_batch_create_article.py index e397725089..34f537c7a8 100644 --- a/doajtest/unit/test_bll_article_batch_create_article.py +++ b/doajtest/unit/test_bll_article_batch_create_article.py @@ -44,7 +44,7 @@ def tearDown(self): self.svc.get_duplicate = self._get_duplicate self.svc.issn_ownership_status = self._issn_ownership_status Article.get_journal = self._get_journal - Journal.find_by_issn = self._find_by_issn_exact + Journal.find_by_issn_exact = self._find_by_issn_exact super(TestBLLArticleBatchCreateArticle, self).tearDown() @parameterized.expand(load_cases) From 3af79038d7002312462d305641a1b38ed60b73c6 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Mon, 2 Oct 2023 12:36:57 +0100 Subject: [PATCH 34/34] Version bump for test fixes release --- portality/settings.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 0d1edcc409..f1a11269e4 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.4.0" +DOAJ_VERSION = "6.4.1" API_VERSION = "3.0.1" ###################################### diff --git a/setup.py b/setup.py index 2c36468273..9be9b324ea 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='doaj', - version='6.4.0', + version='6.4.1', packages=find_packages(), install_requires=[ "awscli==1.20.50",