From 26c5f35dba5cad271a4586febc30c762176ceaaf Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Tue, 26 Nov 2024 16:48:31 +0100 Subject: [PATCH 1/2] move project root from ~/doaj/src/doaj to ~/doaj and put the virtualenv inside it --- deploy/_legacy/anacrontab-harvester-apps | 11 - deploy/_legacy/anacrontab-harvester-gate | 11 - deploy/_legacy/anacrontab-production-apps | 11 - .../anacrontab-production-background-apps | 11 - deploy/_legacy/anacrontab-production-gate | 11 - deploy/_legacy/anacrontab-test-apps | 11 - .../_legacy/anacrontab-test-background-apps | 11 - deploy/_legacy/crontab-harvester-apps | 28 --- deploy/_legacy/crontab-harvester-gate | 27 --- deploy/_legacy/crontab-production-apps | 28 --- .../crontab-production-background-apps | 27 --- deploy/_legacy/crontab-production-gate | 30 --- deploy/_legacy/crontab-production-monitor | 27 --- deploy/_legacy/crontab-test-apps | 26 --- deploy/_legacy/crontab-test-background-apps | 1 - deploy/_legacy/deploy-apps.sh | 53 ----- deploy/_legacy/deploy-background-apps.sh | 52 ----- deploy/_legacy/deploy-gateway.sh | 90 -------- deploy/_legacy/deploy-monitor.sh | 16 -- deploy/_legacy/deploy-redis.sh | 29 --- deploy/_legacy/doaj_gunicorn_config.py | 16 -- deploy/_legacy/doaj_test_gunicorn_config.py | 16 -- deploy/_legacy/esbackup.py | 117 ---------- deploy/_legacy/install_s3fs.sh | 32 --- .../_legacy/lambda/alert_backups_missing.py | 98 -------- deploy/_legacy/logrotate/doaj-analytics | 10 - deploy/_legacy/logrotate/doaj-duplicity | 23 -- deploy/_legacy/mount_s3fs.py | 123 ---------- deploy/_legacy/nginx/block_user_agents.conf | 10 - deploy/_legacy/nginx/doaj-harvester | 28 --- deploy/_legacy/nginx/doaj-harvester-gate | 56 ----- deploy/_legacy/nginx/doaj-production | 20 -- deploy/_legacy/nginx/doaj-production-gate | 210 ------------------ .../nginx/doaj-staging-to-DOAJ-STAGING | 41 ---- deploy/_legacy/nginx/doaj-test | 28 --- deploy/_legacy/nginx/enable-cors | 4 - deploy/_legacy/nginx/gateway | 159 ------------- deploy/_legacy/nginx/proxy_pass_settings | 6 - deploy/_legacy/portality_nginx_config | 20 -- deploy/_legacy/portality_supervisord_config | 8 - deploy/_legacy/restart-apps.sh | 14 -- deploy/_legacy/restart-background-apps.sh | 13 -- deploy/_legacy/restart-gateway.sh | 4 - deploy/_legacy/restart-redis.sh | 14 -- deploy/_legacy/restart.sh | 20 -- deploy/_legacy/robots-production.txt | 9 - deploy/_legacy/robots-test.txt | 2 - .../supervisor/harvester/doaj-harvester.conf | 10 - .../huey-long-running-production.conf | 9 - .../huey-main-production.conf | 9 - .../production/doaj-production.conf | 10 - .../supervisor/staging/doaj-staging.conf | 10 - .../huey-long-running-test.conf | 9 - .../test-background/huey-main-test.conf | 9 - deploy/_legacy/supervisor/test/doaj-test.conf | 10 - deploy/ci_deploy_static.sh | 12 +- deploy/deploy.sh | 6 +- deploy/logrotate/README.md | 4 +- deploy/nginx/doaj | 16 +- deploy/nginx/doaj-test | 14 +- .../huey-long-running.conf | 4 +- .../production-background/huey-main.conf | 4 +- deploy/supervisor/production/doaj.conf | 4 +- deploy/supervisor/test/doaj-static.conf | 4 +- deploy/supervisor/test/doaj.conf | 4 +- deploy/supervisor/test/huey-long-running.conf | 2 +- deploy/supervisor/test/huey-main.conf | 2 +- 67 files changed, 38 insertions(+), 1726 deletions(-) delete mode 100644 deploy/_legacy/anacrontab-harvester-apps delete mode 100644 deploy/_legacy/anacrontab-harvester-gate delete mode 100644 deploy/_legacy/anacrontab-production-apps delete mode 100644 deploy/_legacy/anacrontab-production-background-apps delete mode 100644 deploy/_legacy/anacrontab-production-gate delete mode 100644 deploy/_legacy/anacrontab-test-apps delete mode 100644 deploy/_legacy/anacrontab-test-background-apps delete mode 100644 deploy/_legacy/crontab-harvester-apps delete mode 100644 deploy/_legacy/crontab-harvester-gate delete mode 100644 deploy/_legacy/crontab-production-apps delete mode 100644 deploy/_legacy/crontab-production-background-apps delete mode 100644 deploy/_legacy/crontab-production-gate delete mode 100644 deploy/_legacy/crontab-production-monitor delete mode 100644 deploy/_legacy/crontab-test-apps delete mode 120000 deploy/_legacy/crontab-test-background-apps delete mode 100755 deploy/_legacy/deploy-apps.sh delete mode 100755 deploy/_legacy/deploy-background-apps.sh delete mode 100755 deploy/_legacy/deploy-gateway.sh delete mode 100755 deploy/_legacy/deploy-monitor.sh delete mode 100755 deploy/_legacy/deploy-redis.sh delete mode 100644 deploy/_legacy/doaj_gunicorn_config.py delete mode 100644 deploy/_legacy/doaj_test_gunicorn_config.py delete mode 100644 deploy/_legacy/esbackup.py delete mode 100755 deploy/_legacy/install_s3fs.sh delete mode 100644 deploy/_legacy/lambda/alert_backups_missing.py delete mode 100644 deploy/_legacy/logrotate/doaj-analytics delete mode 100644 deploy/_legacy/logrotate/doaj-duplicity delete mode 100755 deploy/_legacy/mount_s3fs.py delete mode 100644 deploy/_legacy/nginx/block_user_agents.conf delete mode 100644 deploy/_legacy/nginx/doaj-harvester delete mode 100644 deploy/_legacy/nginx/doaj-harvester-gate delete mode 100644 deploy/_legacy/nginx/doaj-production delete mode 100644 deploy/_legacy/nginx/doaj-production-gate delete mode 100644 deploy/_legacy/nginx/doaj-staging-to-DOAJ-STAGING delete mode 100644 deploy/_legacy/nginx/doaj-test delete mode 100644 deploy/_legacy/nginx/enable-cors delete mode 100644 deploy/_legacy/nginx/gateway delete mode 100644 deploy/_legacy/nginx/proxy_pass_settings delete mode 100644 deploy/_legacy/portality_nginx_config delete mode 100644 deploy/_legacy/portality_supervisord_config delete mode 100755 deploy/_legacy/restart-apps.sh delete mode 100755 deploy/_legacy/restart-background-apps.sh delete mode 100755 deploy/_legacy/restart-gateway.sh delete mode 100755 deploy/_legacy/restart-redis.sh delete mode 100755 deploy/_legacy/restart.sh delete mode 100644 deploy/_legacy/robots-production.txt delete mode 100644 deploy/_legacy/robots-test.txt delete mode 100644 deploy/_legacy/supervisor/harvester/doaj-harvester.conf delete mode 100644 deploy/_legacy/supervisor/production-background/huey-long-running-production.conf delete mode 100644 deploy/_legacy/supervisor/production-background/huey-main-production.conf delete mode 100644 deploy/_legacy/supervisor/production/doaj-production.conf delete mode 100644 deploy/_legacy/supervisor/staging/doaj-staging.conf delete mode 100644 deploy/_legacy/supervisor/test-background/huey-long-running-test.conf delete mode 100644 deploy/_legacy/supervisor/test-background/huey-main-test.conf delete mode 100644 deploy/_legacy/supervisor/test/doaj-test.conf diff --git a/deploy/_legacy/anacrontab-harvester-apps b/deploy/_legacy/anacrontab-harvester-apps deleted file mode 100644 index 9cd33907ea..0000000000 --- a/deploy/_legacy/anacrontab-harvester-apps +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly diff --git a/deploy/_legacy/anacrontab-harvester-gate b/deploy/_legacy/anacrontab-harvester-gate deleted file mode 100644 index 01e90fb9cb..0000000000 --- a/deploy/_legacy/anacrontab-harvester-gate +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly \ No newline at end of file diff --git a/deploy/_legacy/anacrontab-production-apps b/deploy/_legacy/anacrontab-production-apps deleted file mode 100644 index 9cd33907ea..0000000000 --- a/deploy/_legacy/anacrontab-production-apps +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly diff --git a/deploy/_legacy/anacrontab-production-background-apps b/deploy/_legacy/anacrontab-production-background-apps deleted file mode 100644 index 9cd33907ea..0000000000 --- a/deploy/_legacy/anacrontab-production-background-apps +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly diff --git a/deploy/_legacy/anacrontab-production-gate b/deploy/_legacy/anacrontab-production-gate deleted file mode 100644 index 01e90fb9cb..0000000000 --- a/deploy/_legacy/anacrontab-production-gate +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly \ No newline at end of file diff --git a/deploy/_legacy/anacrontab-test-apps b/deploy/_legacy/anacrontab-test-apps deleted file mode 100644 index 9cd33907ea..0000000000 --- a/deploy/_legacy/anacrontab-test-apps +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly diff --git a/deploy/_legacy/anacrontab-test-background-apps b/deploy/_legacy/anacrontab-test-background-apps deleted file mode 100644 index 9cd33907ea..0000000000 --- a/deploy/_legacy/anacrontab-test-background-apps +++ /dev/null @@ -1,11 +0,0 @@ -# /etc/anacrontab: configuration file for anacron - -# See anacron(8) and anacrontab(5) for details. - -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# These replace cron's entries -1 5 cron.daily nice run-parts --report /etc/cron.daily -7 10 cron.weekly nice run-parts --report /etc/cron.weekly -@monthly 15 cron.monthly nice run-parts --report /etc/cron.monthly diff --git a/deploy/_legacy/crontab-harvester-apps b/deploy/_legacy/crontab-harvester-apps deleted file mode 100644 index 2ca4077090..0000000000 --- a/deploy/_legacy/crontab-harvester-apps +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - #0 5 * * 1 export NOW=`date +\%F_\%H\%M\%S` ; export DOAJENV=harvester ; echo -e "\n\nstart timestamp $NOW" >> /home/cloo/cron-logs/doaj-workflow-emails.log && /home/cloo/repl/harvester/doaj/bin/python /home/cloo/repl/harvester/doaj/src/doaj/portality/scripts/async_workflow_notifications.py >> /home/cloo/cron-logs/doaj-workflow-emails.log 2>&1 - diff --git a/deploy/_legacy/crontab-harvester-gate b/deploy/_legacy/crontab-harvester-gate deleted file mode 100644 index e46e086f87..0000000000 --- a/deploy/_legacy/crontab-harvester-gate +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - 0 6 * * * export NOW=`date +\%F_\%H\%M\%S` ; export SNAPSHOT="snapshot_"`date +\%F_\%H\%M`; echo -e "\n\nstart timestamp $NOW with snapshot name $SNAPSHOT" >> /home/cloo/cron-logs/es_backups.log && curl -s -XPUT "http://"`head -n 1 /home/cloo/repl/ips/index | tr -d '\n'`":9200/_snapshot/doaj_harvester_s3/$SNAPSHOT" >> /home/cloo/cron-logs/es_backups.log 2>&1 diff --git a/deploy/_legacy/crontab-production-apps b/deploy/_legacy/crontab-production-apps deleted file mode 100644 index ac5d9d7cb6..0000000000 --- a/deploy/_legacy/crontab-production-apps +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - 30 0 * * * export NOW=`date +\%F_\%H\%M\%S` ; export YESTERDAY_DATE=`date -d yesterday +\%F`; export DOAJENV=production ; echo -e "\n\nstart timestamp $NOW" >> /home/cloo/cron-logs/doaj-article-history-cleanup.log && cd /home/cloo/appdata/doaj/history/article && tar -czf "article-$YESTERDAY_DATE.tar.gz" "$YESTERDAY_DATE/" >> /home/cloo/cron-logs/doaj-article-history-cleanup.log 2>&1 && s3cmd -c ~/.s3cfg put "article-$YESTERDAY_DATE.tar.gz" "s3://doaj-sync-$DOAJENV/" >> /home/cloo/cron-logs/doaj-article-history-cleanup.log 2>&1 && rm -r "$YESTERDAY_DATE" >> /home/cloo/cron-logs/doaj-article-history-cleanup.log 2>&1 && rm "article-$YESTERDAY_DATE.tar.gz" >> /home/cloo/cron-logs/doaj-article-history-cleanup.log 2>&1 - diff --git a/deploy/_legacy/crontab-production-background-apps b/deploy/_legacy/crontab-production-background-apps deleted file mode 100644 index 7d94a491da..0000000000 --- a/deploy/_legacy/crontab-production-background-apps +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - diff --git a/deploy/_legacy/crontab-production-gate b/deploy/_legacy/crontab-production-gate deleted file mode 100644 index 9530e4da38..0000000000 --- a/deploy/_legacy/crontab-production-gate +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - 0 6 * * * export NOW=`date +\%F_\%H\%M\%S` ; export SNAPSHOT="snapshot_"`date +\%F_\%H\%M`; echo -e "\n\nstart timestamp $NOW with snapshot name $SNAPSHOT" >> /home/cloo/cron-logs/es_backups.log && curl -s -XPUT "http://gateway:9200/_snapshot/doaj_s3/$SNAPSHOT" >> /home/cloo/cron-logs/es_backups.log 2>&1 - 19 11,23 * * * export NOW=`date +\%F_\%H\%M\%S` ; echo -e "\n\nstart timestamp $NOW" >> /home/cloo/cron-logs/letsencrypt-autorenew.log && cd /opt/letsencrypt && ./letsencrypt-auto renew >> /home/cloo/cron-logs/letsencrypt-autorenew.log 2>&1 && sudo nginx -t >> /home/cloo/cron-logs/letsencrypt-autorenew.log 2>&1 && sudo nginx -s reload >> /home/cloo/cron-logs/letsencrypt-autorenew.log 2>&1 - 30 3 * * 7 . /home/cloo/.doaj_duplicity_credentials ; duplicity full --s3-use-new-style --encrypt-key=98FADACE --exclude /home/cloo/repl/.git /home/cloo/repl s3+http://doaj-duplicity >> /home/cloo/cron-logs/duplicity.log 2>&1 && duplicity remove-all-but-n-full 4 --s3-use-new-style --force s3+http://doaj-duplicity >> /home/cloo/cron-logs/duplicity.log 2>&1 - 30 3 * * 1-6 . /home/cloo/.doaj_duplicity_credentials ; duplicity incr --s3-use-new-style --encrypt-key=98FADACE --exclude /home/cloo/repl/.git /home/cloo/repl s3+http://doaj-duplicity >> /home/cloo/cron-logs/duplicity.log 2>&1 diff --git a/deploy/_legacy/crontab-production-monitor b/deploy/_legacy/crontab-production-monitor deleted file mode 100644 index 94a2a33871..0000000000 --- a/deploy/_legacy/crontab-production-monitor +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command - 0 7 * * * export NOW=`date +\%F_\%H\%M\%S` ; echo -e "\n\nstart timestamp $NOW" >> /home/cloo/cron-logs/prune_marvel.log && /home/cloo/repl/production/doaj/bin/python /home/cloo/repl/production/doaj/src/doaj/portality/scripts/prune_marvel.py >> /home/cloo/cron-logs/prune_marvel.log 2>&1 diff --git a/deploy/_legacy/crontab-test-apps b/deploy/_legacy/crontab-test-apps deleted file mode 100644 index f1aecb3d79..0000000000 --- a/deploy/_legacy/crontab-test-apps +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# m h dom mon dow command diff --git a/deploy/_legacy/crontab-test-background-apps b/deploy/_legacy/crontab-test-background-apps deleted file mode 120000 index d02d8c75fd..0000000000 --- a/deploy/_legacy/crontab-test-background-apps +++ /dev/null @@ -1 +0,0 @@ -crontab-test-apps \ No newline at end of file diff --git a/deploy/_legacy/deploy-apps.sh b/deploy/_legacy/deploy-apps.sh deleted file mode 100755 index cbd65f0c5b..0000000000 --- a/deploy/_legacy/deploy-apps.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -mkdir -p /home/cloo/appdata/doaj -mkdir -p /home/cloo/appdata/doaj/history -mkdir -p /home/cloo/appdata/doaj/history/article -mkdir -p /home/cloo/appdata/doaj/history/journal -mkdir -p /home/cloo/appdata/doaj/ga_logs - -sudo apt-get update -q -y -sudo apt-get install -q -y redis-tools - -# Restart apps services -/home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-apps.sh $ENV - -echo "Setting up crontab and anacrontab" -crontab $DIR/crontab-$ENV-apps -sudo rm -f /etc/anacrontab && sudo ln -sf $DIR/anacrontab-$ENV-apps /etc/anacrontab - -echo "Configuring logrotate" -sudo ln -sf /home/cloo/repl/$ENV/doaj/deploy/logrotate/doaj-analytics /etc/logrotate.d/doaj-analytics - -echo "Mounting S3FS permanently" -cd /home/cloo/repl/$ENV/doaj -. bin/activate -cd src/doaj -chmod 600 .s3fs-credentials # s3fs will refuse to use the credentials file if there are any Group or Other permissions on it -deploy/install_s3fs.sh -echo -echo "Attempting to unmount" -echo -DOAJENV=$1 python deploy/mount_s3fs.py -u || true # try to unmount, but don't care if that doesn't work (e.g. s3fs is not mounted) - -if [ $? -ne 0 ]; then echo; echo "Unmounting failed, but if the reason is that S3FS is already mounted then it can be ignored safely."; echo; fi - -DOAJENV=$1 python deploy/mount_s3fs.py -p - -if [ $? -eq 0 ]; then - echo - echo "Permanent mount seems to be in place. Now attempting to actually mount S3FS using the permanent settings." - echo - DOAJENV=$1 python deploy/mount_s3fs.py -fi - -mkdir -p /home/cloo/appdata/doaj/s3fs -mkdir -p /home/cloo/appdata/doaj/s3fs/cache -mkdir -p /home/cloo/appdata/doaj/s3fs/cache/csv -mkdir -p /home/cloo/appdata/doaj/s3fs/cache/sitemap -mkdir -p /home/cloo/appdata/doaj/s3fs/upload diff --git a/deploy/_legacy/deploy-background-apps.sh b/deploy/_legacy/deploy-background-apps.sh deleted file mode 100755 index 4ca6d7428f..0000000000 --- a/deploy/_legacy/deploy-background-apps.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -mkdir -p /home/cloo/appdata/doaj -mkdir -p /home/cloo/appdata/doaj/history -mkdir -p /home/cloo/appdata/doaj/history/article -mkdir -p /home/cloo/appdata/doaj/history/journal -mkdir -p /home/cloo/appdata/doaj/local_store -mkdir -p /home/cloo/appdata/doaj/local_store/main -mkdir -p /home/cloo/appdata/doaj/local_store/tmp - -sudo apt-get update -q -y -sudo apt-get install -q -y redis-tools - -echo "Mounting S3FS permanently" -cd /home/cloo/repl/$ENV/doaj -. bin/activate -cd src/doaj -chmod 600 .s3fs-credentials # s3fs will refuse to use the credentials file if there are any Group or Other permissions on it -deploy/install_s3fs.sh -echo -echo "Attempting to unmount" -echo -DOAJENV=$1 python deploy/mount_s3fs.py -u || true # try to unmount, but don't care if that doesn't work (e.g. s3fs is not mounted) - -if [ $? -ne 0 ]; then echo; echo "Unmounting failed, but if the reason is that S3FS is already mounted then it can be ignored safely."; echo; fi - -DOAJENV=$1 python deploy/mount_s3fs.py -p - -if [ $? -eq 0 ]; then - echo - echo "Permanent mount seems to be in place. Now attempting to actually mount S3FS using the permanent settings." - echo - DOAJENV=$1 python deploy/mount_s3fs.py -fi - -mkdir -p /home/cloo/appdata/doaj/s3fs -mkdir -p /home/cloo/appdata/doaj/s3fs/cache -mkdir -p /home/cloo/appdata/doaj/s3fs/cache/csv -mkdir -p /home/cloo/appdata/doaj/s3fs/cache/sitemap -mkdir -p /home/cloo/appdata/doaj/s3fs/upload - -# Restart background apps services -/home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-background-apps.sh $ENV - -echo "Setting up crontab and anacrontab" -crontab $DIR/crontab-$ENV-background-apps -sudo rm -f /etc/anacrontab && sudo ln -sf $DIR/anacrontab-$ENV-background-apps /etc/anacrontab diff --git a/deploy/_legacy/deploy-gateway.sh b/deploy/_legacy/deploy-gateway.sh deleted file mode 100755 index f4e35bbc94..0000000000 --- a/deploy/_legacy/deploy-gateway.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# TODO don't delete source code, just delete package info, move src dir out of the way, then put it back in -# otherwise we can't check out a specific tag -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -# recreate the virtualenv, so go all the way out of it -# after checking it's the right dir to delete -cd $DIR/../../../ -CHECK_DIR=`basename "$PWD"` - -[ ! "$CHECK_DIR" = "doaj" ] && echo "Wrong virtualenv name, expected 'doaj' so it could be deleted and recreated" && exit 1 - -# if everything's ok, time to move out the source code and recreate the virtualenv, and move the source code back in -rm -rf /home/cloo/tmp_deploy_workspace_$ENV -mkdir -p /home/cloo/tmp_deploy_workspace_$ENV -rm -rf src/doaj/doaj.egg-info -mv src /home/cloo/tmp_deploy_workspace_$ENV/doaj_src -cd .. -rm -rf doaj -virtualenv -p python2.7 doaj -cd doaj -. bin/activate -pip install --upgrade pip==10.0.1 -pip install --upgrade setuptools-40.6.2 -mv /home/cloo/tmp_deploy_workspace_$ENV/doaj_src src -cd src/doaj - -git submodule update --recursive --init -git submodule update --recursive - -# install app on gate -sudo apt-get update -q -y -sudo apt-get -q -y install libxml2-dev libxslt-dev python-dev lib32z1-dev zip -pip install -r requirements.txt - -# prep sym links for the app server -ln -sf $DIR/supervisor/$ENV/doaj-$ENV.conf /home/cloo/repl/$ENV/supervisor/conf.d/doaj-$ENV.conf -ln -sf $DIR/nginx/doaj-$ENV /home/cloo/repl/$ENV/nginx/sites-available/doaj-$ENV -ln -sf /home/cloo/repl/$ENV/nginx/sites-available/doaj-$ENV /home/cloo/repl/$ENV/nginx/sites-enabled/doaj-$ENV - -# prep sym links for the background app server -ln -sf $DIR/supervisor/$ENV-background/huey-main-$ENV.conf /home/cloo/repl/$ENV-background/supervisor/conf.d/huey-main-$ENV.conf -ln -sf $DIR/supervisor/$ENV-background/huey-long-running-$ENV.conf /home/cloo/repl/$ENV-background/supervisor/conf.d/huey-long-running-$ENV.conf - -# prep sym links for gateway -if [ "$ENV" = 'harvester' ] -then - GATE_ENV=$ENV -else - GATE_ENV=production -fi -ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/nginx/doaj-$GATE_ENV-gate /home/cloo/repl/gateway/nginx/sites-available/doaj-$GATE_ENV-gate -ln -sf /home/cloo/repl/gateway/nginx/sites-available/doaj-$GATE_ENV-gate /home/cloo/repl/gateway/nginx/sites-enabled/doaj-$GATE_ENV-gate -ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/nginx/gateway /home/cloo/repl/gateway/nginx/sites-available/gateway -ln -sf /home/cloo/repl/gateway/nginx/sites-available/gateway /home/cloo/repl/gateway/nginx/sites-enabled/gateway -ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/nginx/block_user_agents.conf /home/cloo/repl/gateway/nginx/conf.d/block_user_agents.conf -ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/nginx/proxy_pass_settings /home/cloo/repl/gateway/nginx/includes/proxy_pass_settings -sudo ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/logrotate/doaj-duplicity /etc/logrotate.d/doaj-duplicity - -# gateway crons -sudo ln -sf /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/anacrontab-$GATE_ENV-gate /etc/anacrontab -crontab /home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/crontab-$GATE_ENV-gate - -# Upload AWS Lambda functions -# todo: this could actually be better using an S3 client library and Lambda from S3 filesystem -# todo: populating .aws/credentials is a bit tricky at the moment. Again solved with S3 library above. (sort of) -# todo: when we have more than one function, they should each be in their own subdirectories named function-name. -if [ "$ENV" = 'production' ] -then - aws --version - cd deploy/lambda - zip upload.zip * - # Credentials must already be in ~/.aws/ - aws --profile doaj-production-lambda lambda update-function-code --function-name alertS3BackupsFailure --zip-file fileb://upload.zip --region=eu-west-1 --publish - rm upload.zip -fi - -# replicate across servers -/home/cloo/repl/replicate.sh -/home/cloo/repl/command.sh -v redis-$ENV /home/cloo/repl/$ENV/doaj/src/doaj/deploy/deploy-redis.sh $ENV -/home/cloo/repl/command.sh -v $ENV /home/cloo/repl/$ENV/doaj/src/doaj/deploy/deploy-apps.sh $ENV -/home/cloo/repl/command.sh -v $ENV-background /home/cloo/repl/$ENV/doaj/src/doaj/deploy/deploy-background-apps.sh $ENV -/home/cloo/repl/command.sh -v monitor /home/cloo/repl/$ENV/doaj/src/doaj/deploy/deploy-monitor.sh $ENV - -# Restart gateway services -/home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/restart-gateway.sh diff --git a/deploy/_legacy/deploy-monitor.sh b/deploy/_legacy/deploy-monitor.sh deleted file mode 100755 index e19375251b..0000000000 --- a/deploy/_legacy/deploy-monitor.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - - -if [ "$ENV" = 'production' ] -then - echo "Setting up crontab and anacrontab" - crontab $DIR/crontab-$ENV-monitor -else - echo "Not setting up monitor crontab in $ENV - it only runs in production." -fi \ No newline at end of file diff --git a/deploy/_legacy/deploy-redis.sh b/deploy/_legacy/deploy-redis.sh deleted file mode 100755 index 34608df127..0000000000 --- a/deploy/_legacy/deploy-redis.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -sudo apt-get update -q -y -sudo apt-get -q -y install curl linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates redis-tools -curl -fsSL https://yum.dockerproject.org/gpg | sudo apt-key add - -# TODO could use apt-key fingerprint to verify the key had fingerprint 58118E89F3A912897C070ADBF76221572C52609D -sudo add-apt-repository "deb https://apt.dockerproject.org/repo/ ubuntu-$(lsb_release -cs) main" -sudo apt-get update -q -y -sudo apt-get -q -y install docker-engine=1.12.3-0~trusty -echo -echo "Docker version should be displayed below if successfully installed" -docker -v -# might have to be done manually - script terminates at some point after the echoes below, either after adding the group, adding *to* the group, or the newgrp command -echo -echo "Setting up Docker to be used without sudo" -sudo groupadd docker -sudo gpasswd -a ${USER} docker -#newgrp docker # TODO automate this. Calling newgrp will start a subshell and not normally execute commands after this line. If we need to run commands under a different group (as we do - everything below this line), then use redirection, see http://unix.stackexchange.com/q/18897/160468 . -echo -echo "Installing Docker Compose, version information should be displayed below" -curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` | sudo tee /usr/local/bin/docker-compose > /dev/null && sudo chmod a+x /usr/local/bin/docker-compose -docker-compose --version - -# Restart redis -/home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-redis.sh $ENV diff --git a/deploy/_legacy/doaj_gunicorn_config.py b/deploy/_legacy/doaj_gunicorn_config.py deleted file mode 100644 index fd0cedb55b..0000000000 --- a/deploy/_legacy/doaj_gunicorn_config.py +++ /dev/null @@ -1,16 +0,0 @@ -import multiprocessing - -bind = "127.0.0.1:5050" -workers = multiprocessing.cpu_count() * 8 + 1 -proc_name = 'doaj' -max_requests = 1000 - -# The maximum jitter to add to the max_requests setting. -# -# The jitter causes the restart per worker to be randomized by -# randint(0, max_requests_jitter) -# This is intended to stagger worker restarts to avoid all workers restarting at the same time. -max_requests_jitter = 100 - -timeout = 40 -graceful_timeout = 40 \ No newline at end of file diff --git a/deploy/_legacy/doaj_test_gunicorn_config.py b/deploy/_legacy/doaj_test_gunicorn_config.py deleted file mode 100644 index 1803495b4a..0000000000 --- a/deploy/_legacy/doaj_test_gunicorn_config.py +++ /dev/null @@ -1,16 +0,0 @@ -import multiprocessing - -bind = "127.0.0.1:5050" -workers = multiprocessing.cpu_count() * 3 + 1 -proc_name = 'doaj (test)' -max_requests = 1000 - -# The maximum jitter to add to the max_requests setting. -# -# The jitter causes the restart per worker to be randomized by -# randint(0, max_requests_jitter) -# This is intended to stagger worker restarts to avoid all workers restarting at the same time. -max_requests_jitter = 100 - -timeout = 40 -graceful_timeout = 40 \ No newline at end of file diff --git a/deploy/_legacy/esbackup.py b/deploy/_legacy/esbackup.py deleted file mode 100644 index 345c6b527b..0000000000 --- a/deploy/_legacy/esbackup.py +++ /dev/null @@ -1,117 +0,0 @@ -#! /usr/bin/python - -# TODO: update this to query 1000 rows at a time from large indices, or it falls over - -from datetime import datetime -import os, requests, json, shutil - -from portality.lib import dates - -# make sure this script is executable, then symlink it from a cron folder, -# or trigger a schedule for it howevr you see fit - -# set the location of the elasticsearch server -# if set, queries retrieving every record from every index -# will be sent, and the JSON results will be saved to file -# do not include trailing slash -location = 'http://localhost:9200' - -# list the names of any indices to ignore -ignore = [] - -# specify the directory on the server running this script where -# the index files are stored. If set, they will be copied too. -# do not include trailing slash. this script must run with -# permissions allowing it to copy this directory, or copying will fail -directory = '' - -# set where to save the backup files to -# do not include trailing slash -backupto = '/home/BACKUPS/es' - -# specify how many days to keep backups -# TODO: implement this -deleteafter = 0 - -# specify whether or not to compress backups -# TODO: implement this -compress = False - -# set an email address for activity alerts -mailto = '' - -# set a filename to write logs to. this script must run with -# permissions allowing it to write to this file, or logging will fail -logto = '' - -# END OF SETTINGS -# ===================================================================================== - -# track the actions that are done, for the log -done = [] - -# create a folder for todays backup, and set todays backup path -time = dates.now_str(dates.FMT_TIME_SHORT) -today = dates.now_str(dates.FMT_DATE_SHORT) -done.append('backup starting at ' + time + ' on ' + today) -backuppath = backupto + '/' + today + '/' -try: - os.makedirs(backuppath) - done.append('backup directory created for today') -except: - pass - -# if a location is set, query every index -if location: - done.append('performing backups via index query') - try: - rs = requests.get(location + '/' + '_status') - indices = rs.json['indices'].keys() - for index in indices: - if index not in ignore: - try: - ts = requests.get(location + '/' + index + '/_mapping') - types = ts.json[index].keys() - for t in types: - try: - rh = requests.get(location + '/' + index + '/' + t + '/_search?q=*&size=0') - size = rh.json['hits']['total'] - r = requests.get(location + '/' + index + '/' + t + '/_search?q=*&size=' + str(size)) - recs = [i['_source'] for i in r.json['hits']['hits']] - out = open(backuppath + index + '_' + t + '.json', 'w') - out.write(json.dumps(recs,indent=4)) - out.close() - done.append(location + '/' + index + '/' + t) - except: - pass - except: - pass - except: - pass - -# if told where the index files are stored, grab a copy of them too -if directory: - try: - shutil.copytree(directory,backuppath + '/data') - done.append('performing backup of index files from ' + directory) - except: - pass - -# TODO: remove old backups based on defined rota -if deleteafter: - pass - -# write to the log, if set -if logto: - out.open(logto, 'a+') - out.write('' + json.dumps(done)) - out.close() - -# mail the log, if set -if mailto: - pass - -# uncomment to print when done -print(done) - - diff --git a/deploy/_legacy/install_s3fs.sh b/deploy/_legacy/install_s3fs.sh deleted file mode 100755 index 8556148a33..0000000000 --- a/deploy/_legacy/install_s3fs.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -echo "Installing S3FS-FUSE system software requirements" -sudo apt-get update -q -y -sudo apt-get -q -y install automake autotools-dev g++ git libcurl4-gnutls-dev libfuse-dev libssl-dev libxml2-dev make pkg-config - -S3FS_GIT_REPO_PATH=/tmp/s3fs-fuse -echo -echo "Compiling S3FS-FUSE" -echo "It is OK to see \"fatal: destination path '$S3FS_GIT_REPO_PATH' already exists and is not an empty directory\". This script deals with it." -echo - -function update_git_repo { - cd $S3FS_GIT_REPO_PATH - git checkout master - git pull - git status -} - -git clone https://github.com/s3fs-fuse/s3fs-fuse.git $S3FS_GIT_REPO_PATH || update_git_repo -cd $S3FS_GIT_REPO_PATH -./autogen.sh -./configure -make -echo -echo "Attempting to install S3FS-FUSE. This uses sudo so you may be asked for your password now." -echo -sudo make install -echo -echo "Installed S3FS-FUSE. Use the deploy/mount_s3fs.py script in this repo for easier mounting. See s3fs -h if you need to use the software directly." -echo -s3fs --version \ No newline at end of file diff --git a/deploy/_legacy/lambda/alert_backups_missing.py b/deploy/_legacy/lambda/alert_backups_missing.py deleted file mode 100644 index 0c9cceda46..0000000000 --- a/deploy/_legacy/lambda/alert_backups_missing.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -""" Steven Eardley 2018-07-09 for DOAJ """ - -import boto3 -import botocore -import json -from datetime import datetime, timezone, timedelta - -from portality.lib.dates import FMT_DATETIME_STD - -s3 = boto3.client('s3') -# doaj-es-backups is too slow currently, so not good value on Lambda (and is already covered by another check) -# buckets = ["doaj-duplicity", "doaj-es-backups", "doaj-letsencrypt"] -buckets = ["doaj-duplicity"] # 2018-11-13 stop backing up letsencrypt - - -def lambda_handler(event, context): - """ The main function executed by Lambda""" - - summary = {'success': [], 'fail': []} - for b in buckets: - - # First check the bucket actually exists - try: - s3.head_bucket(Bucket=b) - except botocore.exceptions.ClientError as e: - error_code = int(e.response['Error']['Code']) - if error_code == 404: - send_alert_email(b, last_mod=None) - - # Then check the expected entry exists in the bucket's objects. - files = list_bucket_keys(bucket_name=b) - old_to_new = sorted(files, key=lambda f: f['LastModified']) - newest = old_to_new[-1] - - # If the newest file is older than 1 day old, our backups are not up to date. - if datetime.now(timezone.utc) - newest['LastModified'] > timedelta(days=1): - # Send an alert - send_alert_email(bucket=b, last_mod=newest['LastModified']) - summary['fail'].append(b) - else: - summary['success'].append(b) - - print(summary) # For the CloudWatch logs - return str(summary) - - -def list_bucket_keys(bucket_name): - """ Read all object metadata from an S3 bucket """ - all_keys = [] - r = s3.list_objects_v2(Bucket=bucket_name) - while r.get('Contents', None): - all_keys += r['Contents'] - - if r.get('NextContinuationToken', None): - r = s3.list_objects_v2(Bucket=bucket_name, ContinuationToken=r['NextContinuationToken']) - else: - break - return all_keys - - -def send_alert_email(bucket, last_mod): - """ Use the mailgun API key stored in AWS Secrets Manager to email us errors """ - - session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name='eu-west-1', - endpoint_url='https://secretsmanager.eu-west-1.amazonaws.com' - ) - - try: - resp = client.get_secret_value(SecretId='doaj-error-email') - except botocore.exceptions.ClientError as e: - raise e - else: - if 'SecretString' in resp: - credentials = json.loads(resp['SecretString']) - - if last_mod is None: - msg = 'AWS backup error: bucket {b} is missing.'.format(b=bucket) - else: - msg = 'AWS backup error: bucket {b} has not been updated today - it was last modified on {t}.' \ - '\nYou may wish to check the corresponding logs.'.format(b=bucket, - t=last_mod.strftime(FMT_DATETIME_STD)) - - r = botocore.vendored.requests.post('https://api.mailgun.net/v3/doaj.org/messages', - auth=('api', credentials.get('ERROR_MAIL_API_KEY', '')), - data={'from': 'error@doaj.org', - 'to': [credentials.get('ERROR_LOGGING_EMAIL', - 'sysadmin@cottagelabs.com')], - 'subject': 'DOAJ backups error - AWS S3', - 'text': msg}) - - if r.status_code != 200: - raise Exception('Error sending email: status {0}'.format(r.status_code)) - else: - raise Exception('Could not send alert email - no usable credentials supplied.') diff --git a/deploy/_legacy/logrotate/doaj-analytics b/deploy/_legacy/logrotate/doaj-analytics deleted file mode 100644 index ef8728c2dc..0000000000 --- a/deploy/_legacy/logrotate/doaj-analytics +++ /dev/null @@ -1,10 +0,0 @@ -/home/cloo/appdata/doaj/ga_logs/portality.lib.analytics.log { - daily - missingok - compress - delaycompress - notifempty - dateext - dateyesterday - rotate 365 -} diff --git a/deploy/_legacy/logrotate/doaj-duplicity b/deploy/_legacy/logrotate/doaj-duplicity deleted file mode 100644 index 7b29c37859..0000000000 --- a/deploy/_legacy/logrotate/doaj-duplicity +++ /dev/null @@ -1,23 +0,0 @@ -/home/cloo/cron-logs/duplicity.log { - monthly - missingok - compress - delaycompress - notifempty - dateext - dateyesterday -} - -/home/cloo/cron-logs/letsencrypt-duplicity.log { - monthly - missingok - compress - delaycompress - notifempty - dateext - dateyesterday -} - -# rotate 52 Only keep a year's worth of logs -# delaycompress Compress the following time it's run, not immediately -# dateyesterday Logs will run up to and include the date in the filename diff --git a/deploy/_legacy/mount_s3fs.py b/deploy/_legacy/mount_s3fs.py deleted file mode 100755 index 4c19479c9a..0000000000 --- a/deploy/_legacy/mount_s3fs.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -""" -Mount S3FS-FUSE. - -Provides a way to put the various arguments in the app configuration, so they don't have to be passed around the command -line. Should also make it far easier for devs who don't have to remember the s3fs CLI syntax. - -Since it uses the app config, it's easiest to have it as a Python rather than a bash script. -""" - -import subprocess -import sys - -from portality.core import app - - -class S3FSScriptException(Exception): - pass - - -def check_permanent_mount(bucket): - try: - # we don't actually need grep's output, but this prevents grep from printing it out - subprocess.check_output('grep "s3fs#{} {}" /etc/fstab'.format(bucket, app.config['S3FS_MOUNT_DIR']), shell=True) - return True - except subprocess.CalledProcessError: - return False - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("-u", "--unmount", action="store_true", help="Unmount, rather than mount (the default action)") - parser.add_argument("-p", "--permanent", action="store_true", help="Make the mount stick on reboot. Checks /etc/fstab for a relevant entry and adds one if no entries are found.") - args = parser.parse_args() - - if args.unmount and args.permanent: - raise S3FSScriptException("You can't specify both --unmount and --permanent. --permanent only makes sense when mounting.") - - # print args - if 'S3FS_BUCKET' not in app.config: - raise S3FSScriptException( -""" - ---> Ensure S3FS_BUCKET is a defined string in your .cfg files. <-- - -Acceptable formats are s3://bucket_name and bucket_name. -The test server's bucket is s3://doaj-test-sync, can also be used on development machines to test syncing. Your files will be synced with those of the test server if you use it! -""" - ) - if 'S3FS_CREDS_PATH' not in app.config: - raise S3FSScriptException( -""" - ---> Ensure S3FS_CREDS_PATH is defined in your .cfg files. <-- - -/.s3fs-credentials is conventional and .gitignore-d. -The file you're pointing to must conform to the format "AWS ACCESS KEY:AWS SECRET KEY" on 1 line. -""" - ) - if 'S3FS_MOUNT_DIR' not in app.config: - raise S3FSScriptException( -""" - ---> Ensure S3FS_MOUNT_DIR is defined in your .cfg files. <-- - -**NOTE it's recommended you put this outside the DOAJ git repostitory, since 'git status' and other everyday commands which recurse through the directory tree may become significantly slower if S3FS is mounted. -Please use an empty directory as the mount target. -""" - ) - - bucket = app.config['S3FS_BUCKET'] - if bucket.startswith('s3://'): - bucket = bucket[5:] - - credentials_arg = 'passwd_file={}'.format(app.config['S3FS_CREDS_PATH']) - - permanently_mounted = check_permanent_mount(bucket) - - if args.permanent: - if permanently_mounted: - print ("S3FS permanent mount for bucket {} already exists, not doing anything.\n" \ - 'If you think the /etc/fstab entry is wrong, remove it manually and rerun this script with --permanent.') - sys.exit(0) - - fstab_entry = 's3fs#{} {} fuse _netdev,allow_other,{},url=https://s3.amazonaws.com 0 0'.format(bucket, app.config['S3FS_MOUNT_DIR'], credentials_arg) - if '"' in fstab_entry: - raise S3FSScriptException("You can't have \" (double quotes) in any of the environment variables that control the S3 mount.") - print() - print('Setting up permanent mount in /etc/fstab - you may be asked for your sudo password now.') - print() - subprocess.check_call('echo "{}" | sudo tee -a /etc/fstab'.format(fstab_entry), shell=True) - print() - print ('^ the above entry has been appended to your /etc/fstab') - permanently_mounted = check_permanent_mount(bucket) - - if args.unmount: - try: - subprocess.check_call(["sudo", 'umount', app.config['S3FS_MOUNT_DIR']]) - except subprocess.CalledProcessError: - print() - print ('Unmounting failed. This can happen if you try to unmount too soon after you mounted. If the error is something like "device busy", wait for 5-10 seconds and rerun the unmount.') - print() - raise - - print () - sys.exit(0) - - # let's mount then - if permanently_mounted: - print() - print ("Mounting using the permanent mount in /etc/fstab.\n" \ - "If this script completes successfully but you can't use the mountpoint as an S3 file system, " \ - "check your /etc/fstab and compare against your local S3FS_* DOAJ config vars, then report a bug.") - - subprocess.check_call(['sudo', 'mount', app.config['S3FS_MOUNT_DIR']]) - else: - print() - print ('Mounting with temporary mount.') - subprocess.check_call(["s3fs", bucket, app.config['S3FS_MOUNT_DIR'], '-o', credentials_arg, '-o', 'url=https://s3.amazonaws.com']) - - print ('S3FS mounted bucket {} at directory {} succesfully'.format(bucket, app.config['S3FS_MOUNT_DIR'])) diff --git a/deploy/_legacy/nginx/block_user_agents.conf b/deploy/_legacy/nginx/block_user_agents.conf deleted file mode 100644 index 83a096719e..0000000000 --- a/deploy/_legacy/nginx/block_user_agents.conf +++ /dev/null @@ -1,10 +0,0 @@ -map $http_user_agent $block_ua { - default 0; - ~*bot 1; - ~*spider 1; - ~*Go-http-client 1; - ~*python-requests 1; - ~*mechanize 1; - ~*wget 1; - ~*curl 1; -} diff --git a/deploy/_legacy/nginx/doaj-harvester b/deploy/_legacy/nginx/doaj-harvester deleted file mode 100644 index 07e076cd2a..0000000000 --- a/deploy/_legacy/nginx/doaj-harvester +++ /dev/null @@ -1,28 +0,0 @@ -server { - listen 80; - server_name harvesterdoaj.cottagelabs.com; - - access_log /var/log/nginx/doaj-harvester.access.log; - error_log /var/log/nginx/doaj-harvester.error.log; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - location / { - proxy_pass http://localhost:5050/; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } - - location /static/ { - include /home/cloo/repl/harvester/doaj/src/doaj/deploy/nginx/enable-cors; - - alias /home/cloo/repl/harvester/doaj/src/doaj/portality/static/; - autoindex off; - expires max; - } -} diff --git a/deploy/_legacy/nginx/doaj-harvester-gate b/deploy/_legacy/nginx/doaj-harvester-gate deleted file mode 100644 index 97780d729f..0000000000 --- a/deploy/_legacy/nginx/doaj-harvester-gate +++ /dev/null @@ -1,56 +0,0 @@ -# redirect to SSL version -#server { -# listen 80; -# server_name harvesterdoaj.cottagelabs.com; -# -# # Serve some crucial XML schemas via plain HTTP as well -# -# # Common misspelling of doajArticle**s**.xsd -# location =/static/doaj/doajArticle.xsd { -# alias /home/cloo/repl/harvester/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; -# } -# -# location =/static/doaj/doajArticles.xsd { -# alias /home/cloo/repl/harvester/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; -# } -# -# location =/static/doaj/iso_639-2b.xsd { -# alias /home/cloo/repl/harvester/doaj/src/doaj/portality/static/doaj/iso_639-2b.xsd; -# } -# -# location / { -# return 301 https://$host$request_uri; -# } -# -# access_log /var/log/nginx/doaj-harvester.access.log; -# error_log /var/log/nginx/doaj-harvester.error.log; -#} - -server { - #listen 443 ssl; - listen 80; - server_name harvesterdoaj.cottagelabs.com; - - access_log /var/log/nginx/doaj-harvester.access.log; - error_log /var/log/nginx/doaj-harvester.error.log; - - include includes/cl-wildcard-ssl.conf; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - # Common misspelling of doajArticle**s**.xsd - location =/static/doaj/doajArticle.xsd { - alias /home/cloo/repl/test/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location / { - proxy_pass http://harvesterapps; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } -} diff --git a/deploy/_legacy/nginx/doaj-production b/deploy/_legacy/nginx/doaj-production deleted file mode 100644 index 72ca28432a..0000000000 --- a/deploy/_legacy/nginx/doaj-production +++ /dev/null @@ -1,20 +0,0 @@ -server { - listen 80; - server_name doaj.org; - - access_log /var/log/nginx/doaj.access.log; - error_log /var/log/nginx/doaj.error.log; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - location / { - proxy_pass http://localhost:5050/; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } -} diff --git a/deploy/_legacy/nginx/doaj-production-gate b/deploy/_legacy/nginx/doaj-production-gate deleted file mode 100644 index 8af3fdce6d..0000000000 --- a/deploy/_legacy/nginx/doaj-production-gate +++ /dev/null @@ -1,210 +0,0 @@ -# redirect to non-www version -server { - listen 80; - listen 443 ssl; - server_name www.doaj.org; - - access_log /var/log/nginx/doaj.access.log; - error_log /var/log/nginx/doaj.error.log; - - include includes/cl-letsencrypt-doaj.conf; - - return 301 $scheme://doaj.org$request_uri; -} - -# redirect to SSL version -server { - listen 80; - server_name doaj.org; - - # Serve some crucial XML schemas via plain HTTP as well - - # Common misspelling of doajArticle**s**.xsd - location =/static/doaj/doajArticle.xsd { - alias /home/cloo/repl/production/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location =/static/doaj/doajArticles.xsd { - alias /home/cloo/repl/production/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location =/static/doaj/iso_639-2b.xsd { - alias /home/cloo/repl/production/doaj/src/doaj/portality/static/doaj/iso_639-2b.xsd; - } - - location ~ /.well-known { - root /usr/share/nginx/html/; - allow all; - } - - location =/robots.txt { - alias /home/cloo/repl/production/doaj/src/doaj/deploy/robots-production.txt; - } - - location / { - return 301 https://$host$request_uri; - } - - access_log /var/log/nginx/doaj.access.log; - error_log /var/log/nginx/doaj.error.log; -} - -# separate section for test app due to paths to the XML schemas -# being different to the live app -server { - listen 80; - server_name testdoaj.cottagelabs.com; - - # password restrict the test server (disabled 2018-11-15 since server data is anonymous) - #auth_basic "Restricted"; - #auth_basic_user_file /etc/nginx/htpasswd; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - # Common misspelling of doajArticle**s**.xsd - location =/static/doaj/doajArticle.xsd { - alias /home/cloo/repl/test/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location ~ /.well-known { - #auth_basic off; - root /usr/share/nginx/html/; - allow all; - } - - location =/robots.txt { - alias /home/cloo/repl/test/doaj/src/doaj/deploy/robots-test.txt; - } - - location / { - return 301 https://$host$request_uri; - } - - access_log /var/log/nginx/doaj.access.log; - error_log /var/log/nginx/doaj.error.log; -} - -# actually serve the app on doaj.org via SSL -# there's 2 server sections since *.cottagelabs.com is covered by a -# different SSL cert, see further down - -limit_req_zone $binary_remote_addr zone=general:100m rate=2r/s; -limit_req_zone $binary_remote_addr zone=api:100m rate=2r/s; -limit_req_status 429; -server { - listen 443 ssl; - server_name doaj.org; - - access_log /var/log/nginx/doaj.access.log; - error_log /var/log/nginx/doaj.error.log; - - include includes/cl-letsencrypt-doaj.conf; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - # Common misspelling of doajArticle**s**.xsd - location =/static/doaj/doajArticle.xsd { - alias /home/cloo/repl/production/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location ~ /.well-known { - root /usr/share/nginx/html/; - allow all; - } - - location =/robots.txt { - alias /home/cloo/repl/production/doaj/src/doaj/deploy/robots-production.txt; - } - - location /static/ { - include /home/cloo/repl/production/doaj/src/doaj/deploy/nginx/enable-cors; - - alias /home/cloo/repl/production/doaj/src/doaj/portality/static/; - autoindex off; - expires max; - } - - location /api { - if ($http_user_agent ~* (bot|spider) ) { - return 403; - } - limit_req zone=api burst=5 nodelay; - proxy_pass http://apps; - include includes/proxy_pass_settings; - } - - location /search { - if ($block_ua) {return 403;} - limit_req zone=general burst=5 nodelay; - proxy_pass http://apps; - include includes/proxy_pass_settings; - } - - location /query { - if ($block_ua) {return 403;} - limit_req zone=general burst=5 nodelay; - proxy_pass http://apps; - include includes/proxy_pass_settings; - } - - location / { - limit_req zone=general burst=5 nodelay; - proxy_pass http://apps; - include includes/proxy_pass_settings; - } - -} - -# the test app -server { - listen 443 ssl; - server_name testdoaj.cottagelabs.com; - - access_log /var/log/nginx/doaj-test.access.log; - error_log /var/log/nginx/doaj-test.error.log; - - include includes/cl-letsencrypt-doaj.conf; - - # password restrict the test server (disabled 2018-11-15 since server data is anonymous) - #auth_basic "Restricted"; - #auth_basic_user_file /etc/nginx/htpasswd; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - # Common misspelling of doajArticle**s**.xsd - location =/static/doaj/doajArticle.xsd { - alias /home/cloo/repl/test/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; - } - - location ~ /.well-known { - #auth_basic off; - root /usr/share/nginx/html/; - allow all; - } - - location =/robots.txt { - alias /home/cloo/repl/test/doaj/src/doaj/deploy/robots-test.txt; - } - - location / { - proxy_pass http://testapps; - include includes/proxy_pass_settings; - } -} - -# Metrics for metricsbeat -server { - listen [::]:80; - listen 80; - server_name 127.0.0.1; - location /server-status { - stub_status on; - access_log off; - allow ::1; - allow 127.0.0.1; - deny all; - } -} diff --git a/deploy/_legacy/nginx/doaj-staging-to-DOAJ-STAGING b/deploy/_legacy/nginx/doaj-staging-to-DOAJ-STAGING deleted file mode 100644 index addddadbce..0000000000 --- a/deploy/_legacy/nginx/doaj-staging-to-DOAJ-STAGING +++ /dev/null @@ -1,41 +0,0 @@ -server { - listen 80; - server_name staging.doaj.cottagelabs.com; - - access_log /var/log/nginx/doaj-staging-to-DOAJ-STAGING.access.log; - error_log /var/log/nginx/doaj-staging-to-DOAJ-STAGING.error.log; - - proxy_read_timeout 600s; - - location / { - proxy_pass http://95.85.48.213; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} - -server { - listen 443; - server_name staging.doaj.cottagelabs.com; - - ssl on; - ssl_certificate /etc/nginx/CERTS/cottagelabs.com.chained.crt; - ssl_certificate_key /etc/nginx/CERTS/cottagelabs.key; - - access_log /var/log/nginx/doaj-staging-to-DOAJ-STAGING.access.log; - error_log /var/log/nginx/doaj-staging-to-DOAJ-STAGING.error.log; - - proxy_read_timeout 600s; - - location / { - proxy_pass https://95.85.48.213; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - include /home/cloo/repl/staging/doaj/src/doaj/deploy/nginx/enable-cors; - } -} diff --git a/deploy/_legacy/nginx/doaj-test b/deploy/_legacy/nginx/doaj-test deleted file mode 100644 index 1a3c905bc2..0000000000 --- a/deploy/_legacy/nginx/doaj-test +++ /dev/null @@ -1,28 +0,0 @@ -server { - listen 80; - server_name testdoaj.cottagelabs.com; - - access_log /var/log/nginx/doaj-test.access.log; - error_log /var/log/nginx/doaj-test.error.log; - - proxy_read_timeout 600s; - client_max_body_size 50M; - - location / { - proxy_pass http://localhost:5050/; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } - - location /static/ { - include /home/cloo/repl/test/doaj/src/doaj/deploy/nginx/enable-cors; - - alias /home/cloo/repl/test/doaj/src/doaj/portality/static/; - autoindex off; - expires max; - } -} diff --git a/deploy/_legacy/nginx/enable-cors b/deploy/_legacy/nginx/enable-cors deleted file mode 100644 index 5f380968f6..0000000000 --- a/deploy/_legacy/nginx/enable-cors +++ /dev/null @@ -1,4 +0,0 @@ -add_header 'Access-Control-Allow-Origin' '*'; -add_header 'Access-Control-Allow-Credentials' 'true'; -add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; -add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; \ No newline at end of file diff --git a/deploy/_legacy/nginx/gateway b/deploy/_legacy/nginx/gateway deleted file mode 100644 index 861f465c73..0000000000 --- a/deploy/_legacy/nginx/gateway +++ /dev/null @@ -1,159 +0,0 @@ -# Edit this file when you want changes to the nginx gate config - it's the basis for gateway-autogenerated -# Don't edit gateway-autogenerated manually. - -# SOME GENERAL CONFIGS -ssl_session_cache shared:SSL:10m; -ssl_session_timeout 10m; - -# USE PRIVATE IP ADDRESSES FOR THE CLUSTER MACHINES -# MAKE SURE THE IP ADDRESSES USED HERE MATCH THOSE LISTED IN THE FILES IN THE IPS FOLDER - -# THE MACHINES THAT RUN THE APPS -upstream apps { - ip_hash; - #server 10.131.21.233; # doaj-app-2 - server 10.131.168.173; # doaj-app-1 -} - -upstream testapps { - ip_hash; - server 10.131.178.96; -} - -# UPSTREAM_STAGING_APPS_SECTION - -# THE MACHINES ON WHICH ELASTICSEARCH CLUSTER IS ACCEPTING INPUT -# APPS THAT CALL IT "DIRECTLY" SHOULD CALL THIS GATEWAY MACHINE AND THEN THIS ROUTES TO THE INDEX -include includes/upstream-index-production.conf; - -upstream testindex { - server 10.131.178.96:9200; -} - -# UPSTREAM_STAGING_INDEX_SECTION - -# THE INDEX - GATEWAY UFW SHOULD ALLOW ALL APPS MACHINES TO CALL 9200 VIA ETH1 -# WHICH MATCHES THIS, WHICH PASSES THROUGH TO THE INDEX UPSTREAM CLUSTER MACHINES -server { - listen 9200; - server_name gateway; - - location /_snapshot { - proxy_pass http://index; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } - - location / { - client_max_body_size 1024M; - proxy_connect_timeout 3s; - proxy_read_timeout 15s; - - proxy_pass http://index; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - } -} - -# Redirect to SSL wherever necessary -server { - listen 80; - server_name doajgate.cottagelabs.com testdoajes.cottagelabs.com monitordoaj.cottagelabs.com; - - location ~ /.well-known { - root /usr/share/nginx/html/; - allow all; - } - - return 301 https://$host$request_uri; -} - -# AN HTTPS ENDPOINT FOR ES THAT IS SECURED WITH BASIC AUTH -# SO ELASTICSEARCH HEAD CAN BE VIEWED IF NECESSARY -# SHOULD EVENTUALLY REWRITE THROUGH THE INDEX APP -server { - listen 443 ssl; - server_name doajgate.cottagelabs.com; - - auth_basic "Restricted"; - auth_basic_user_file /etc/nginx/htpasswd; - - include includes/cl-letsencrypt-doaj.conf; - - location ~ /.well-known { - auth_basic off; - root /usr/share/nginx/html/; - allow all; - } - - location / { - proxy_pass http://index; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Authorization ""; - } -} - -server { - listen 443 ssl; - server_name monitordoaj.cottagelabs.com; - - auth_basic "Restricted"; - auth_basic_user_file /etc/nginx/htpasswd; - - include includes/cl-letsencrypt-doaj.conf; - - location ~ /.well-known { - auth_basic off; - root /usr/share/nginx/html/; - allow all; - } - - location / { - proxy_pass http://10.131.137.108:9200; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Authorization ""; - } - - location =/kibana { - return 301 /_plugin/marvel/kibana/index.html; - } -} - -server { - listen 443 ssl; - server_name testdoajes.cottagelabs.com; - - auth_basic "Restricted"; - auth_basic_user_file /etc/nginx/test_doaj_es_htpasswd; - - include includes/cl-letsencrypt-doaj.conf; - - location ~ /.well-known { - auth_basic off; - root /usr/share/nginx/html/; - allow all; - } - - location / { - proxy_pass http://testindex; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Authorization ""; - } -} diff --git a/deploy/_legacy/nginx/proxy_pass_settings b/deploy/_legacy/nginx/proxy_pass_settings deleted file mode 100644 index dfd1adf4a8..0000000000 --- a/deploy/_legacy/nginx/proxy_pass_settings +++ /dev/null @@ -1,6 +0,0 @@ -proxy_redirect off; -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; -proxy_buffering off; \ No newline at end of file diff --git a/deploy/_legacy/portality_nginx_config b/deploy/_legacy/portality_nginx_config deleted file mode 100644 index ee12242900..0000000000 --- a/deploy/_legacy/portality_nginx_config +++ /dev/null @@ -1,20 +0,0 @@ -upstream portality_server { - server 127.0.0.1:5004 fail_timeout=0; -} - -server { - server_name portality.cottagelabs.com; - access_log /var/log/nginx/portality.access.log; - server_name_in_redirect off; - client_max_body_size 20M; - - location / { - proxy_connect_timeout 75s; - proxy_read_timeout 180s; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass http://portality_server; - } -} - diff --git a/deploy/_legacy/portality_supervisord_config b/deploy/_legacy/portality_supervisord_config deleted file mode 100644 index 8b561f6804..0000000000 --- a/deploy/_legacy/portality_supervisord_config +++ /dev/null @@ -1,8 +0,0 @@ -[program:portality] -command=/opt/%(program_name)s/bin/gunicorn -w 4 -b 127.0.0.1:5004 %(program_name)s.app:app -user=www-data -directory=/opt/%(program_name)s/src/%(program_name)s -stdout_logfile=/var/log/supervisor/%(program_name)s-access.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true - diff --git a/deploy/_legacy/restart-apps.sh b/deploy/_legacy/restart-apps.sh deleted file mode 100755 index 95be8b5abb..0000000000 --- a/deploy/_legacy/restart-apps.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -# Restart the python app under supervisor -sudo supervisorctl reread doaj-$ENV -sudo supervisorctl update doaj-$ENV -kill -HUP $(sudo supervisorctl pid doaj-$ENV) - -# Restart nginx, which the app is running behind. -sudo nginx -t && sudo nginx -s reload diff --git a/deploy/_legacy/restart-background-apps.sh b/deploy/_legacy/restart-background-apps.sh deleted file mode 100755 index 2a17f5056f..0000000000 --- a/deploy/_legacy/restart-background-apps.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -sudo supervisorctl reread huey-main-$ENV -sudo supervisorctl reread huey-long-running-$ENV -sudo supervisorctl update huey-main-$ENV -sudo supervisorctl update huey-long-running-$ENV -sudo supervisorctl restart huey-main-$ENV -sudo supervisorctl restart huey-long-running-$ENV diff --git a/deploy/_legacy/restart-gateway.sh b/deploy/_legacy/restart-gateway.sh deleted file mode 100755 index 7d48547216..0000000000 --- a/deploy/_legacy/restart-gateway.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# reload the nginx config if syntax is OK -sudo nginx -t && sudo nginx -s reload diff --git a/deploy/_legacy/restart-redis.sh b/deploy/_legacy/restart-redis.sh deleted file mode 100755 index 3e853f9be5..0000000000 --- a/deploy/_legacy/restart-redis.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -cd /home/cloo/repl/$ENV/doaj/src/doaj/docker -docker-compose down || true # it's fine to fail to bring down redis if it is not started yet -# for the line below: -# --build to pick up docker/redis/Dockerfile changes -# --remove-orphans to remove all containers that are not defined in the docker/docker-compose.yml (allows us to change service names and configuration just by editing that file and deploying -# -d for detached mode rather than run in foreground -docker-compose up --build --remove-orphans -d diff --git a/deploy/_legacy/restart.sh b/deploy/_legacy/restart.sh deleted file mode 100755 index 9b793eaf02..0000000000 --- a/deploy/_legacy/restart.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Restart all DOAJ production services - -THIS_SCRIPT=`basename "$0"` -[ $# -ne 1 ] && echo "Call this script as $THIS_SCRIPT " && exit 1 - -ENV=$1 - -# Handle the exception for the harvester duplicate production site fixme: this should probably go since that testing is over -if [ "$ENV" = 'harvester' ] -then - GATE_ENV=$ENV -else - GATE_ENV=production -fi - -/home/cloo/repl/command.sh -v redis-$ENV /home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-redis.sh $ENV -/home/cloo/repl/command.sh -v $ENV /home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-apps.sh $ENV -/home/cloo/repl/command.sh -v $ENV-background /home/cloo/repl/$ENV/doaj/src/doaj/deploy/restart-background-apps.sh $ENV -/home/cloo/repl/$GATE_ENV/doaj/src/doaj/deploy/restart-gateway.sh diff --git a/deploy/_legacy/robots-production.txt b/deploy/_legacy/robots-production.txt deleted file mode 100644 index 0af56f4e5f..0000000000 --- a/deploy/_legacy/robots-production.txt +++ /dev/null @@ -1,9 +0,0 @@ -Sitemap: https://doaj.org/sitemap.xml - -User-agent: * -Disallow: /search/ -Disallow: /query/ -Disallow: /account/ -Disallow: /admin/ -Disallow: /editor/ -Disallow: /publisher/ diff --git a/deploy/_legacy/robots-test.txt b/deploy/_legacy/robots-test.txt deleted file mode 100644 index 1f53798bb4..0000000000 --- a/deploy/_legacy/robots-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * -Disallow: / diff --git a/deploy/_legacy/supervisor/harvester/doaj-harvester.conf b/deploy/_legacy/supervisor/harvester/doaj-harvester.conf deleted file mode 100644 index f01af45362..0000000000 --- a/deploy/_legacy/supervisor/harvester/doaj-harvester.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:doaj-harvester] -command= /home/cloo/repl/harvester/doaj/bin/gunicorn -c /home/cloo/repl/harvester/doaj/src/doaj/deploy/doaj_gunicorn_config.py portality.app:app -environment = DOAJENV=harvester -user=cloo -directory=/home/cloo/repl/harvester/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-access.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true -stopasgroup=true diff --git a/deploy/_legacy/supervisor/production-background/huey-long-running-production.conf b/deploy/_legacy/supervisor/production-background/huey-long-running-production.conf deleted file mode 100644 index 0049beaaef..0000000000 --- a/deploy/_legacy/supervisor/production-background/huey-long-running-production.conf +++ /dev/null @@ -1,9 +0,0 @@ -[program:huey-long-running-production] -command= /home/cloo/repl/production/doaj/bin/huey_consumer.py -v portality.tasks.consumer_long_running.long_running -environment= DOAJENV=production -user=cloo -directory=/home/cloo/repl/production/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true diff --git a/deploy/_legacy/supervisor/production-background/huey-main-production.conf b/deploy/_legacy/supervisor/production-background/huey-main-production.conf deleted file mode 100644 index 9e6ff3b111..0000000000 --- a/deploy/_legacy/supervisor/production-background/huey-main-production.conf +++ /dev/null @@ -1,9 +0,0 @@ -[program:huey-main-production] -command= /home/cloo/repl/production/doaj/bin/huey_consumer.py -v portality.tasks.consumer_main_queue.main_queue -environment= DOAJENV=production -user=cloo -directory=/home/cloo/repl/production/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true diff --git a/deploy/_legacy/supervisor/production/doaj-production.conf b/deploy/_legacy/supervisor/production/doaj-production.conf deleted file mode 100644 index 8c7c332084..0000000000 --- a/deploy/_legacy/supervisor/production/doaj-production.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:doaj-production] -command= /home/cloo/repl/production/doaj/bin/gunicorn -c /home/cloo/repl/production/doaj/src/doaj/deploy/doaj_gunicorn_config.py portality.app:app -environment = DOAJENV=production -user=cloo -directory=/home/cloo/repl/production/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-access.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true -stopasgroup=true diff --git a/deploy/_legacy/supervisor/staging/doaj-staging.conf b/deploy/_legacy/supervisor/staging/doaj-staging.conf deleted file mode 100644 index 6ed1cdfb56..0000000000 --- a/deploy/_legacy/supervisor/staging/doaj-staging.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:doaj-staging] -command=/opt/doaj/bin/gunicorn -c /opt/doaj/src/doaj/deploy/doaj_gunicorn_config.py portality.app:app -environment = DOAJENV=staging -user=cloo -directory=/opt/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-access.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true -stopasgroup=true diff --git a/deploy/_legacy/supervisor/test-background/huey-long-running-test.conf b/deploy/_legacy/supervisor/test-background/huey-long-running-test.conf deleted file mode 100644 index b3af8baf05..0000000000 --- a/deploy/_legacy/supervisor/test-background/huey-long-running-test.conf +++ /dev/null @@ -1,9 +0,0 @@ -[program:huey-long-running-test] -command=/home/cloo/repl/test/doaj/bin/huey_consumer.py -v portality.tasks.consumer_long_running.long_running -environment= DOAJENV=test -user=cloo -directory=/home/cloo/repl/test/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true diff --git a/deploy/_legacy/supervisor/test-background/huey-main-test.conf b/deploy/_legacy/supervisor/test-background/huey-main-test.conf deleted file mode 100644 index 69c12b9c29..0000000000 --- a/deploy/_legacy/supervisor/test-background/huey-main-test.conf +++ /dev/null @@ -1,9 +0,0 @@ -[program:huey-main-test] -command= /home/cloo/repl/test/doaj/bin/huey_consumer.py -v portality.tasks.consumer_main_queue.main_queue -environment= DOAJENV=test -user=cloo -directory=/home/cloo/repl/test/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true diff --git a/deploy/_legacy/supervisor/test/doaj-test.conf b/deploy/_legacy/supervisor/test/doaj-test.conf deleted file mode 100644 index 08e5a078d4..0000000000 --- a/deploy/_legacy/supervisor/test/doaj-test.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:doaj-test] -command= /home/cloo/repl/test/doaj/bin/gunicorn -c /home/cloo/repl/test/doaj/src/doaj/deploy/doaj_test_gunicorn_config.py portality.app:app -environment= DOAJENV=test -user=cloo -directory=/home/cloo/repl/test/doaj/src/doaj -stdout_logfile=/var/log/supervisor/%(program_name)s-access.log -stderr_logfile=/var/log/supervisor/%(program_name)s-error.log -autostart=true -autorestart=true -stopasgroup=true diff --git a/deploy/ci_deploy_static.sh b/deploy/ci_deploy_static.sh index f68cf84006..e1286de72f 100755 --- a/deploy/ci_deploy_static.sh +++ b/deploy/ci_deploy_static.sh @@ -2,19 +2,19 @@ # This script is to deploy the additional static installation of DOAJ on the test server. # Required: create virtualenv & clone project -# virtualenv -p python3.8 doaj-static && mkdir doaj-static/src && cd doaj-static/src -# git clone https://github.com/DOAJ/doaj.git +# git clone https://github.com/DOAJ/doaj.git doaj-static +# virtualenv -p python3.8 doaj-static/venv # # Symlink the supervisor script -# sudo ln -sf /home/cloo/doaj-static/src/doaj/deploy/supervisor/test/doaj-static.conf /etc/supervisor/conf.d/doaj-static.conf +# sudo ln -sf /home/cloo/doaj-static/deploy/supervisor/test/doaj-static.conf /etc/supervisor/conf.d/doaj-static.conf # sudo supervisorctl reread # # Ensure test server AWS credentials are in ~/.aws/ # Run from the doaj folder that's already checked out -# activate the virtualenv that we expect to be at /home/cloo/doaj-static -. /home/cloo/doaj-static/bin/activate -cd /home/cloo/doaj-static/src/doaj +# activate the virtualenv that we expect to be at /home/cloo/doaj-static/venv +. /home/cloo/doaj-static/venv/bin/activate +cd /home/cloo/doaj-static/ # Install DOAJ submodules and requirements git submodule update --init --recursive diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 70937fd127..c06f47c562 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -13,9 +13,9 @@ sudo pip install awscli # Run from the doaj folder that's already checked out -# activate the virtualenv that we expect to be at /home/cloo/doaj -. /home/cloo/doaj/bin/activate -cd /home/cloo/doaj/src/doaj || exit 1 +# activate the virtualenv that we expect to be at /home/cloo/doaj/venv +. /home/cloo/doaj/venv/bin/activate +cd /home/cloo/doaj/ || exit 1 # Get the app configuration secrets from AWS - NOTE: on a mac, base64 needs -D rather than -d if [ "$ENV" = 'production' ] diff --git a/deploy/logrotate/README.md b/deploy/logrotate/README.md index 2ef592ccad..15f091882f 100644 --- a/deploy/logrotate/README.md +++ b/deploy/logrotate/README.md @@ -8,13 +8,13 @@ and set with `sudo passwd root` if necessary. ``` -sudo cp /home/cloo/doaj/src/doaj/deploy/logrotate/doaj-analytics /etc/logrotate.d/doaj-analytics +sudo cp /home/cloo/doaj/deploy/logrotate/doaj-analytics /etc/logrotate.d/doaj-analytics ``` and ``` -sudo cp /home/cloo/doaj/src/doaj/deploy/logrotate/doaj-nginx /etc/logrotate.d/doaj-nginx +sudo cp /home/cloo/doaj/deploy/logrotate/doaj-nginx /etc/logrotate.d/doaj-nginx ``` For the AWS S3 upload to work correctly, the correct credentials must be saved in `~/.aws/`, and diff --git a/deploy/nginx/doaj b/deploy/nginx/doaj index 4e6c3b0576..2372d1248f 100644 --- a/deploy/nginx/doaj +++ b/deploy/nginx/doaj @@ -77,23 +77,23 @@ server { error_log /var/log/nginx/doaj.error.log; location =/static/doaj/doajArticle.xsd { - alias /home/cloo/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; + alias /home/cloo/doaj/portality/static/doaj/doajArticles.xsd; } location =/static/doaj/doajArticles.xsd { - alias /home/cloo/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; + alias /home/cloo/doaj/portality/static/doaj/doajArticles.xsd; } location =/static/doaj/iso_639-2b.xsd { - alias /home/cloo/doaj/src/doaj/portality/static/doaj/iso_639-2b.xsd; + alias /home/cloo/doaj/portality/static/doaj/iso_639-2b.xsd; } location /static/crossref { - alias /home/cloo/doaj/src/doaj/portality/static/crossref; + alias /home/cloo/doaj/portality/static/crossref; } location =/robots.txt { - alias /home/cloo/doaj/src/doaj/deploy/robots-production.txt; + alias /home/cloo/doaj/deploy/robots-production.txt; } location / { @@ -260,11 +260,11 @@ server { } location =/robots.txt { - alias /home/cloo/doaj/src/doaj/deploy/robots-production.txt; + alias /home/cloo/doaj/deploy/robots-production.txt; } location /static/ { - alias /home/cloo/doaj/src/doaj/portality/static/; + alias /home/cloo/doaj/portality/static/; autoindex off; expires max; add_header 'Access-Control-Allow-Origin' '*'; # CORS settings since we don't get these from the app @@ -274,7 +274,7 @@ server { } location /assets/ { - alias /home/cloo/doaj/src/doaj/cms/assets/; + alias /home/cloo/doaj/cms/assets/; autoindex off; expires max; add_header 'Access-Control-Allow-Origin' '*'; # CORS settings since we don't get these from the app diff --git a/deploy/nginx/doaj-test b/deploy/nginx/doaj-test index 3e1b0f7e4a..0f618c7580 100644 --- a/deploy/nginx/doaj-test +++ b/deploy/nginx/doaj-test @@ -4,15 +4,15 @@ server { server_name testdoaj.cottagelabs.com testdoajes.cottagelabs.com staticdoaj.cottagelabs.com; location =/static/doaj/doajArticle.xsd { - alias /home/cloo/doaj/src/doaj/portality/static/doaj/doajArticles.xsd; + alias /home/cloo/doaj/portality/static/doaj/doajArticles.xsd; } location =/static/doaj/iso_639-2b.xsd { - alias /home/cloo/doaj/src/doaj/portality/static/doaj/iso_639-2b.xsd; + alias /home/cloo/doaj/portality/static/doaj/iso_639-2b.xsd; } location /static/crossref { - alias /home/cloo/doaj/src/doaj/portality/static/crossref; + alias /home/cloo/doaj/portality/static/crossref; } location / { @@ -63,10 +63,10 @@ server { proxy_pass http://doaj_apps; } location =/robots.txt { - alias /home/cloo/doaj/src/doaj/deploy/robots-test.txt; + alias /home/cloo/doaj/deploy/robots-test.txt; } location /static/ { - alias /home/cloo/doaj/src/doaj/portality/static/; + alias /home/cloo/doaj/portality/static/; autoindex off; expires max; } @@ -97,10 +97,10 @@ server { proxy_pass http://doaj_static; } location =/robots.txt { - alias /home/cloo/doaj-static/src/doaj/deploy/robots-test.txt; + alias /home/cloo/doaj-static/deploy/robots-test.txt; } location /static/ { - alias /home/cloo/doaj-static/src/doaj/portality/static/; + alias /home/cloo/doaj-static/portality/static/; autoindex off; expires max; } diff --git a/deploy/supervisor/production-background/huey-long-running.conf b/deploy/supervisor/production-background/huey-long-running.conf index 41073e246a..4e23fdbc7d 100644 --- a/deploy/supervisor/production-background/huey-long-running.conf +++ b/deploy/supervisor/production-background/huey-long-running.conf @@ -1,8 +1,8 @@ [program:huey-long-running] -command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_long_running.long_running +command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_long_running.long_running environment= DOAJENV=production user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/production-background/huey-main.conf b/deploy/supervisor/production-background/huey-main.conf index a80925eb91..e02c498351 100644 --- a/deploy/supervisor/production-background/huey-main.conf +++ b/deploy/supervisor/production-background/huey-main.conf @@ -1,8 +1,8 @@ [program:huey-main] -command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_main_queue.main_queue +command=/home/cloo/doaj/venv/bin/python /home/cloo/doaj/venv/bin/huey_consumer.py -v portality.tasks.consumer_main_queue.main_queue environment= DOAJENV=production user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/production/doaj.conf b/deploy/supervisor/production/doaj.conf index ff5f11e20e..42819001ff 100644 --- a/deploy/supervisor/production/doaj.conf +++ b/deploy/supervisor/production/doaj.conf @@ -1,8 +1,8 @@ [program:doaj] -command=/home/cloo/doaj/bin/gunicorn -c /home/cloo/doaj/src/doaj/deploy/doaj_gunicorn_config.py portality.app:app +command=/home/cloo/doaj/bin/gunicorn -c /home/cloo/doaj/deploy/doaj_gunicorn_config.py portality.app:app environment = DOAJENV=production user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-access.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/test/doaj-static.conf b/deploy/supervisor/test/doaj-static.conf index 28da26e942..db44562bc4 100644 --- a/deploy/supervisor/test/doaj-static.conf +++ b/deploy/supervisor/test/doaj-static.conf @@ -1,8 +1,8 @@ [program:doaj-static] -command=/home/cloo/doaj-static/bin/gunicorn -c /home/cloo/doaj-static/src/doaj/deploy/doaj_static_gunicorn_config.py portality.app:app +command=/home/cloo/doaj-static/venv/bin/gunicorn -c /home/cloo/doaj-static/deploy/doaj_static_gunicorn_config.py portality.app:app environment= DOAJENV=test user=cloo -directory=/home/cloo/doaj-static/src/doaj +directory=/home/cloo/doaj-static/ stdout_logfile=/var/log/supervisor/%(program_name)s-access.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/test/doaj.conf b/deploy/supervisor/test/doaj.conf index 557bc86347..2ead79e33d 100644 --- a/deploy/supervisor/test/doaj.conf +++ b/deploy/supervisor/test/doaj.conf @@ -1,8 +1,8 @@ [program:doaj] -command=/home/cloo/doaj/bin/gunicorn -c /home/cloo/doaj/src/doaj/deploy/doaj_test_gunicorn_config.py portality.app:app +command=/home/cloo/doaj/venv/bin/gunicorn -c /home/cloo/doaj/deploy/doaj_test_gunicorn_config.py portality.app:app environment= DOAJENV=test user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-access.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/test/huey-long-running.conf b/deploy/supervisor/test/huey-long-running.conf index e7f85d46ef..3c0e8000e6 100644 --- a/deploy/supervisor/test/huey-long-running.conf +++ b/deploy/supervisor/test/huey-long-running.conf @@ -2,7 +2,7 @@ command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_long_running.long_running environment= DOAJENV=test user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true diff --git a/deploy/supervisor/test/huey-main.conf b/deploy/supervisor/test/huey-main.conf index 41fee7f811..a279d4bb86 100644 --- a/deploy/supervisor/test/huey-main.conf +++ b/deploy/supervisor/test/huey-main.conf @@ -2,7 +2,7 @@ command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_main_queue.main_queue environment= DOAJENV=test user=cloo -directory=/home/cloo/doaj/src/doaj +directory=/home/cloo/doaj stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log stderr_logfile=/var/log/supervisor/%(program_name)s-error.log autostart=true From d1554f1b6e5b58aa1a821dd8e3d21cd7244f7e1d Mon Sep 17 00:00:00 2001 From: Steven Eardley Date: Wed, 27 Nov 2024 16:17:59 +0100 Subject: [PATCH 2/2] set redis host for production to use the background-1 server --- production.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/production.cfg b/production.cfg index 93c4fad623..5c7dfa02ee 100644 --- a/production.cfg +++ b/production.cfg @@ -56,3 +56,6 @@ EVENT_SEND_FUNCTION = "portality.events.shortcircuit.send_event" # https://github.com/DOAJ/doajPM/issues/3565 2023-03-07 PRESERVATION_PAGE_UNDER_MAINTENANCE = False + +# Ensure all app machines can contact background server first with redis-cli -h e.g. redis-cli -h 10.131.191.148 +REDIS_HOST = '10.131.191.148' \ No newline at end of file