Merge branch 'develop' into feature/3136_different_publisher_and_society

.circleci/config.yml

@@ -14,6 +14,7 @@ jobs:
           - xpack.security.enabled: false
           - transport.host: localhost
           - discovery.type: single-node
+    parallelism: 4
 
     steps:
       - checkout
@@ -26,7 +27,8 @@ jobs:
           echo "ELASTICSEARCH_HOSTS = [{'host': 'localhost', 'port': 9200}]" > test.cfg
           python portality/cms/build_fragments.py
           python portality/cms/build_sass.py
-          pytest -v --color=yes --code-highlight=yes --log-level=DEBUG doajtest/unit
+          TESTS=$(circleci tests glob "doajtest/unit/**/*.py" | circleci tests split)
+          pytest -v --color=yes --code-highlight=yes --log-level=DEBUG $TESTS
 
     working_directory: ~/doaj
 

cms/data/ambassadors.yml

@@ -62,13 +62,6 @@
   photo: "ivonne.jpg"
   coi:
     2022: https://drive.google.com/file/d/1HnGhYbvbzL34guWOmIqcthcwAN8NADX1/view?usp=sharing
-
-- name: John G. Dove
-  region: North America
-  bio: "John has had a career in executive management, and is now an independent consultant and open access advocate who works with organisations seeking to accelerate their transition to open access.  He advises both for-profits and non-profits, and has a particular interest in identifying the steps necessary to flip an entire discipline’s scholarly record to open access. His ambassador activities focus on increasing the support to DOAJ from the community. He served for six years on NISO’s <a href='https://www.niso.org/topic-committees/information-discovery-interchange' target='_blank'>Information Discovery and Interchange Topic Committee</a>, and has written for <a href='https://onlinelibrary.wiley.com/doi/10.1002/leap.1086' target='_blank'>Learned Publishing</a>, <a href='https://against-the-grain.com/2011/12/v23-5-atg-interviews-john-g-dove-president-credo-reference/' target='_blank'>Against the Grain</a>, and <a href='https://scholarlykitchen.sspnet.org/2019/11/07/guest-post-transparency-what-can-one-learn-from-a-trove-of-invoices/' target='_blank'>Scholarly Kitchen</a>. John serves on the Board of Trustees of his local public library in Revere, Massachusetts. He has a B.A. in Mathematics from Oberlin College."
-  photo: "johndove.jpg"
-  coi:
-    2022: https://drive.google.com/file/d/1cWijl2xdmVjshsvaGTABOvC_chIIfuVA/view?usp=sharing
 
 - name: Mahmoud Khalifa
   region: Middle East and Persian Gulf

cms/pages/legal/terms.md

@@ -32,7 +32,7 @@ DOAJ uses a variety of licenses for the different parts of its website and the c
   + In our [OAI-PMH feed](/docs/oai-pmh)
   + In the [full data dump of all article metadata](/docs/public-data-dump/).
 
-4. The *open source software* that DOAJ is built with is licensed under [an Apache license Version 2](https://github.com/DOAJ/doaj/blob/a6fc2bee499b5a8a1f24fb098acfb8e10bd72503/portality/static/vendor/select2-3.5.4/LICENSE).
+4. The *open source software* that DOAJ is built with is licensed under [an Apache license Version 2](https://github.com/DOAJ/doaj/blob/develop/LICENSE).
 
 ---
 

cms/sass/components/_form.scss

@@ -82,7 +82,7 @@
   border-left: 1px solid $sanguine;
 }
 
-.form__long-help {
+.form__long-help, .form__click-to-copy {
   cursor: pointer;
 
   &:hover {

cms/sass/components/_skip-to-main-content.scss

@@ -0,0 +1,35 @@
+/* Back to main content button */
+
+.skip-to-main {
+  position: absolute;
+  z-index: 10000;
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  min-width: min-content;
+  padding: 5px;
+  top: 10px;
+  left: 10px;
+  background-color: $grapefruit;
+
+  svg {
+    display: block;
+    margin: 0 auto;
+    stroke: $warm-black;
+    margin-rigth: 10px;
+  }
+  &:hover, &:focus {
+    svg {
+      margin-right: 10px;
+    }
+  }
+  &:hover:after, &:focus:after {
+    content: "  Skip to main content";
+    color: $warm-black;
+    vertical-align: bottom;
+    -webkit-font-feature-settings: 'liga' 1;
+    -moz-font-feature-settings: 'liga' 1;
+    font-feature-settings: 'liga' 1;
+    transition: 0.5 smooth;
+  }
+}

cms/sass/components/_tag.scss

@@ -90,3 +90,8 @@
     color: $white;
   }
 }
+
+.tag--confirmation {
+  background: $dark-green;
+  color: $white;
+}

cms/sass/main.scss

@@ -52,6 +52,7 @@
   "components/review-table",
   "components/select2",
   "components/search-results",
+  "components/skip-to-main-content",
   "components/stat",
   "components/stretch-list",
   "components/tabs",

deploy/doaj_gunicorn_config.py

@@ -1,7 +1,7 @@
 import multiprocessing
 
 bind = "0.0.0.0:5050"
-workers = multiprocessing.cpu_count() * 8 + 1
+workers = multiprocessing.cpu_count() * 6 + 1
 proc_name = 'doaj'
 max_requests = 1000
 
@@ -13,4 +13,4 @@
 max_requests_jitter = 100
 
 timeout = 40
-graceful_timeout = 40
+graceful_timeout = 40

deploy/lambda/alert_backups_missing.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-""" Steven Eardley 2020-02-07 for DOAJ - uploaded manually (todo: we should upload this in the release script) """
+""" Steven Eardley 2023-09-15 for DOAJ - uploaded manually (todo: we should upload this in the release script) """
 
 # ~~BackupsMissing:Monitoring->Lambda:Technology~~
 
@@ -8,35 +8,39 @@
 import json
 from datetime import datetime, timezone, timedelta
 
-from portality.lib.dates import FMT_DATETIME_STD
-
 s3 = boto3.client('s3')
 
 # Check the doaj elasticsearch snapshot bucket has been updated today (should happen daily at 0600 via background job)
-buckets = ['doaj-index-backups']
+buckets = ['doaj-index-ipt-backups']
+
 
 # Check the doaj-nginx logs bucket has been updated today (should happen daily at 0630 via cron logrotate)
-buckets += ['doaj-nginx-logs']
+# buckets += ['doaj-nginx-logs']
 
 
 def lambda_handler(event, context):
     """ The main function executed by Lambda"""
 
+    start = datetime.utcnow()
     summary = {'success': [], 'fail': []}
     for b in buckets:
 
+        print('Checking bucket {0} was updated today'.format(b))
+
         # First check the bucket actually exists
         try:
             s3.head_bucket(Bucket=b)
         except botocore.exceptions.ClientError as e:
             error_code = int(e.response['Error']['Code'])
             if error_code == 404:
                 send_alert_email(b, last_mod=None)
+            raise
 
         # Then check the expected entry exists in the bucket's objects.
         files = list_bucket_keys(bucket_name=b)
         old_to_new = sorted(files, key=lambda f: f['LastModified'])
         newest = old_to_new[-1]
+        print('Latest backup is', newest)
 
         # If the newest file is older than 1 day old, our backups are not up to date.
         if datetime.now(timezone.utc) - newest['LastModified'] > timedelta(days=1):
@@ -47,6 +51,8 @@ def lambda_handler(event, context):
             summary['success'].append(b)
 
     print(summary)  # For the CloudWatch logs
+    print('Completed in', str(datetime.utcnow() - start))
+
     return str(summary)
 
 
@@ -86,8 +92,8 @@ def send_alert_email(bucket, last_mod):
                 msg = 'AWS backup error: bucket {b} is missing.'.format(b=bucket)
             else:
                 msg = 'AWS backup error: bucket {b} has not been updated today - it was last modified on {t}.' \
-                      '\nYou may wish to check the corresponding logs.'.format(b=bucket,
-                                                                               t=last_mod.strftime(FMT_DATETIME_STD))
+                      '\nYou may wish to check the corresponding logs.'.format(b=bucket, t=last_mod.strftime(
+                    '%Y-%m-%dT%H:%M:%SZ'))
 
             r = botocore.vendored.requests.post('https://api.mailgun.net/v3/doaj.org/messages',
                                                 auth=('api', credentials.get('ERROR_MAIL_API_KEY', '')),

deploy/nginx/doaj

@@ -36,17 +36,27 @@ map $http_user_agent $block_ua {
     ~*curl              1;
 }
 
+# the public server (deprecated, use failover)
 upstream doaj_apps {
-    server 10.131.191.139:5050;
+    server 10.131.191.139:5050;   #doaj-public-app-1
 }
+
+# Background server runs async tasks
 upstream doaj_bg_apps {
-    #server 10.131.56.133:5050;  #old bg machine
-    server 10.131.12.33:5050;
+    server 10.131.12.33:5050;     #doaj-background-app-1
+}
+
+# Editor and admin site components
+upstream doaj_ed_failover {
+    server 10.131.56.133:5050;    #doaj-editor-app-1
+    server 10.131.12.33:5050 backup;     #doaj-background-app-1
 }
+
+# For public site components, try all servers
 upstream doaj_apps_failover {
-    server 10.131.191.139:5050;
-    #server 10.131.56.133:5050 backup;  #old bg machine
-    server 10.131.12.33:5050 backup;
+    server 10.131.191.139:5050;        #doaj-public-app-1
+    server 10.131.12.33:5050 backup;   #doaj-background-app-1
+    server 10.131.56.133:5050 backup;  #doaj-editor-app-1
 }
 upstream doaj_index {
     server 10.131.191.132:9200;
@@ -121,6 +131,7 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
+
     location /search {
         if ($block_ua) {return 403;}
         limit_req zone=general burst=10 nodelay;
@@ -144,9 +155,7 @@ server {
         proxy_buffering off;
     }
 
-    # for now we are going to send all login functions to the bg machine
-    # technically ONLY the routes that require file upload need to go to the bg machine
-    # but we think it is handy to separate them out, and later we could send them to other machines
+    # technically only the routes that require file upload need to go to the bg machine, but separate for consistency
     location /account {
         limit_req zone=general burst=10 nodelay;
         proxy_pass http://doaj_bg_apps;
@@ -157,6 +166,19 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
+
+    # prefer the editor machine for application form work (but application_quick_reject goes to background async)
+    location ~* /admin/application/ {
+        limit_req zone=general burst=10 nodelay;
+        proxy_pass http://doaj_ed_failover;
+        proxy_redirect off;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_buffering off;
+    }
+
     location /admin { # there are admin bulk actions that MUST go to bg machine
         limit_req zone=general burst=10 nodelay;
         proxy_pass http://doaj_bg_apps;
@@ -167,27 +189,30 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
+
     location /editor {
         limit_req zone=general burst=10 nodelay;
-        proxy_pass http://doaj_bg_apps;
+        proxy_pass http://doaj_ed_failover;
         proxy_redirect off;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
+
     location /journal/readonly {
         limit_req zone=general burst=10 nodelay;
-        proxy_pass http://doaj_bg_apps;
+        proxy_pass http://doaj_ed_failover;
         proxy_redirect off;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
-    location /publisher { # only /publisher/uploadfile MUST go to bg, and /publisher/uploadFile
+
+    location /publisher {   # only /publisher/uploadfile MUST go to background
         limit_req zone=general burst=10 nodelay;
         proxy_pass http://doaj_bg_apps;
         proxy_redirect off;
@@ -197,7 +222,8 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
-    location /service {
+
+    location /service {    # performs locks etc - handle on the background server
         limit_req zone=general burst=10 nodelay;
         proxy_pass http://doaj_bg_apps;
         proxy_redirect off;
@@ -221,6 +247,7 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_buffering off;
     }
+
     location /csv {
         limit_req zone=general burst=10 nodelay;
         proxy_pass http://doaj_bg_apps;
@@ -235,6 +262,7 @@ server {
     location =/robots.txt {
         alias /home/cloo/doaj/src/doaj/deploy/robots-production.txt;
     }
+
     location /static/ {
         alias /home/cloo/doaj/src/doaj/portality/static/;
         autoindex off;

doajtest/matrices/article_create_article/issn_validation_against_journal.matrix.csv

@@ -0,0 +1,17 @@
+test_id,eissn,pissn,validated
+1,eissn_in_doaj,pissn_in_doaj,yes
+2,eissn_in_doaj,eissn_not_in_doaj,
+3,eissn_in_doaj,pissn_not_in_doaj,
+4,eissn_in_doaj,!eissn_in_doaj,
+5,pissn_in_doaj,eissn_in_doaj,
+6,pissn_in_doaj,eissn_not_in_doaj,
+7,pissn_in_doaj,pissn_not_in_doaj,
+8,pissn_in_doaj,!pissn_in_doaj,
+9,eissn_not_in_doaj,eissn_in_doaj,
+10,eissn_not_in_doaj,pissn_in_doaj,
+11,eissn_not_in_doaj,pissn_not_in_doaj,
+12,eissn_not_in_doaj,!eissn_not_in_doaj,
+13,pissn_not_in_doaj,eissn_in_doaj,
+14,pissn_not_in_doaj,pissn_in_doaj,
+15,pissn_not_in_doaj,eissn_not_in_doaj,
+16,pissn_not_in_doaj,!pissn_not_in_doaj,

doajtest/matrices/article_create_article/issn_validation_against_journal.settings.csv

@@ -0,0 +1,19 @@
+field,test_id,eissn,pissn,validated
+type,index,generated,generated,conditional
+deafult,,,,no
+,,,,
+values,,eissn_in_doaj,eissn_in_doaj,yes
+values,,pissn_in_doaj,pissn_in_doaj,no
+values,,eissn_not_in_doaj,eissn_not_in_doaj,
+values,,pissn_not_in_doaj,pissn_not_in_doaj,
+,,,,
+,,,,
+conditional validated,,eissn_in_doaj,pissn_in_doaj,yes
+constraint eissn,,eissn_in_doaj,!eissn_in_doaj,
+constraint eissn,,eissn_not_in_doaj,!eissn_not_in_doaj,
+constraint eissn,,pissn_not_in_doaj,!pissn_not_in_doaj,
+constraint eissn,,pissn_in_doaj,!pissn_in_doaj,
+constraint pissn,,eissn_in_doaj,!eissn_in_doaj,
+constraint pissn,,eissn_not_in_doaj,!eissn_not_in_doaj,
+constraint pissn,,pissn_not_in_doaj,!pissn_not_in_doaj,
+constraint pissn,,pissn_in_doaj,!pissn_in_doaj,