Merge pull request #3603 from WikiWatershed/tt/linux-updates

deployment/ansible/group_vars/all

@@ -20,7 +20,7 @@ postgresql_support_repository_channel: "main"
 postgresql_support_libpq_version: "13.*.pgdg20.04+1"
 postgresql_support_psycopg2_version: "2.8.*"
 postgis_version: "3"
-postgis_package_version: "3.3*pgdg20.04+1"
+postgis_package_version: "3.4*pgdg20.04+1"
 
 daemontools_version: "1:0.76-7"
 
@@ -44,9 +44,10 @@ docker_compose_version: "1.26.*"
 geop_host: "localhost"
 geop_port: 8090
 
-geop_version: "5.3.0"
+geop_version: "5.4.0"
 geop_cache_enabled: 1
-geop_timeout: 200
+geop_timeout: 200s
+geop_bucket: datahub-catalogs-us-east-1
 
 nginx_cache_dir: "/var/cache/nginx"
 

deployment/ansible/roles/model-my-watershed.base/defaults/main.yml

@@ -14,6 +14,7 @@ envdir_config:
   MMW_GEOPROCESSING_PORT: "{{ geop_port }}"
   MMW_GEOPROCESSING_VERSION: "{{ geop_version }}"
   MMW_GEOPROCESSING_TIMEOUT: "{{ geop_timeout }}"
+  MMW_GEOPROCESSING_BUCKET: "{{ geop_bucket }}"
   MMW_ITSI_CLIENT_ID: "{{ itsi_client_id }}"
   MMW_ITSI_SECRET_KEY: "{{ itsi_secret_key }}"
   MMW_ITSI_BASE_URL: "{{ itsi_base_url }}"

deployment/ansible/roles/model-my-watershed.geoprocessing/templates/systemd-geoprocessing.service.j2

@@ -4,13 +4,12 @@ After=network.target
 
 [Service]
 {% if ['development', 'test'] | some_are_in(group_names) -%}
-Environment=MMW_GEOPROCESSING_TIMEOUT={{ geop_timeout }}s AWS_PROFILE={{ aws_profile }}
-{% else %}
-Environment=MMW_GEOPROCESSING_TIMEOUT={{ geop_timeout }}s
+Environment=AWS_PROFILE={{ aws_profile }}
 {% endif %}
+Environment="JAVA_WITH_ENV=/usr/bin/envdir /etc/mmw.d/env /usr/bin/java"
 User=mmw
 WorkingDirectory={{ geop_home }}
-ExecStart=/usr/bin/java -jar mmw-geoprocessing-{{ geop_version }}.jar
+ExecStart=/bin/sh -c '${JAVA_WITH_ENV} -jar mmw-geoprocessing-{{ geop_version }}.jar'
 StandardOutput=syslog
 StandardError=syslog
 SyslogIdentifier=geoprocessing

scripts/aws/setupdb.sh

@@ -15,8 +15,9 @@ where options are one or more of: \n
     -d  load/reload DRB stream data\n
     -m  load/reload mapshed data\n
     -p  load/reload DEP data\n
-    -c  load/reload nhdplus catchment data
+    -c  load/reload nhdplus catchment data\n
     -q  load/reload water quality data\n
+    -X  purge s3 cache as well\n
     -x  purge s3 cache for given path\n
 "
 
@@ -29,6 +30,7 @@ load_hires_stream=false
 load_mapshed=false
 load_water_quality=false
 load_catchment=false
+should_purge_cache=false
 
 while getopts ":hbsSdpmqcf:x:" opt; do
     case $opt in
@@ -55,6 +57,8 @@ while getopts ":hbsSdpmqcf:x:" opt; do
             file_to_load=$OPTARG ;;
         x)
             path_to_purge=$OPTARG ;;
+        X)
+            should_purge_cache=true ;;
         \?)
             echo "invalid option: -$OPTARG"
             exit ;;
@@ -85,9 +89,11 @@ function download_and_load {
 }
 
 function purge_tile_cache {
-    for path in "${PATHS[@]}"; do
-        aws s3 rm --recursive "s3://tile-cache.${PUBLIC_HOSTED_ZONE_NAME}/${path}/"
-    done
+    if [ "$should_purge_cache" = "true" ] ; then
+        for path in "${PATHS[@]}"; do
+            aws s3 rm --recursive "s3://tile-cache.${PUBLIC_HOSTED_ZONE_NAME}/${path}/"
+        done
+    fi
 }
 
 function create_trgm_indexes {

src/mmw/mmw/settings/base.py

@@ -9,6 +9,8 @@
 https://docs.djangoproject.com/en/1.7/ref/settings/
 """
 
+import re
+
 from os import environ
 from os.path import abspath, basename, dirname, join, normpath
 from sys import path
@@ -128,7 +130,11 @@ def get_env_setting(setting):
 CELERY_TASK_DEFAULT_EXCHANGE = 'tasks'
 CELERY_TASK_DEFAULT_ROUTING_KEY = "task.%s" % STACK_COLOR
 
-CELERY_TASK_TIME_LIMIT = int(environ.get('MMW_GEOPROCESSING_TIMEOUT', 120))
+# MMW_GEOPROCESSING_TIMEOUT specified with "s" suffix for mmw-geoprocessing
+MMW_GEOPROCESSING_TIMEOUT = environ.get('MMW_GEOPROCESSING_TIMEOUT', '120s')
+
+CELERY_TASK_TIME_LIMIT = int(
+    re.search(r'\d+', MMW_GEOPROCESSING_TIMEOUT).group())
 TASK_REQUEST_TIMEOUT = CELERY_TASK_TIME_LIMIT - 10
 # END CELERY CONFIGURATION