Merge branch 'release-1.6'

.circleci/config.yml

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with ooicgsn-roundabout in the COPYING.md file at the project root.
 # If not, see <http://www.gnu.org/licenses/>.
-#  
+#
 
 workflows:
   version: 2
@@ -29,6 +29,7 @@ workflows:
             only:
               - develop
               - automated_tests
+              - release-1.6
 
 version: 2
 jobs:
@@ -77,12 +78,11 @@ jobs:
           name: Running Automated tests
           command: >-
             docker-compose -f local.yml run --rm tests \
-               sh -c "until curl -Is http://localhost:8000; do echo 'sleeping'; sleep 10; done  && 
+               sh -c "until curl -Is http://localhost:8000; do echo 'sleeping'; sleep 10; done  &&
                   google-chrome --version && chromedriver --version &&
                   nodejs -v &&
                   ./RunAllTests-Chrome-Linux.bat"
 
-
       - run:
           name: Stopping Docker application
           command: >-

.envs.example/.local/.django

@@ -11,6 +11,13 @@ REDIS_URL=redis://redis:6379/0
 # ------------------------------------------------------------------------------
 DJANGO_SECRET_KEY=9vgy4xpSMfbmesHKI5mJjZn2V6tHZhQHgSmvAoGzq4HnLpX1LagUTGiYwHJQNWCo
 
+# Celery
+# ------------------------------------------------------------------------------
+
+# Flower
+CELERY_FLOWER_USER=celeryusername
+CELERY_FLOWER_PASSWORD=celeryuserpassword
+
 # Roundabout Specific Django settings
 # ------------------------------------------------------------------------------
 

.envs.example/.production/.django

@@ -36,6 +36,13 @@ WEB_CONCURRENCY=4
 # ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/0
 
+# Celery
+# ------------------------------------------------------------------------------
+
+# Flower
+CELERY_FLOWER_USER=celeryusername
+CELERY_FLOWER_PASSWORD=celeryuserpassword
+
 # Roundabout Specific Django settings
 # ------------------------------------------------------------------------------
 

compose/local/django/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-alpine
+FROM python:3.8-alpine
 
 ENV PYTHONUNBUFFERED 1
 
@@ -28,6 +28,18 @@ COPY ./compose/local/django/start /start
 RUN sed -i 's/\r//' /start
 RUN chmod +x /start
 
+COPY ./compose/local/django/celery/worker/start /start-celeryworker
+RUN sed -i 's/\r//' /start-celeryworker
+RUN chmod +x /start-celeryworker
+
+COPY ./compose/local/django/celery/beat/start /start-celerybeat
+RUN sed -i 's/\r//' /start-celerybeat
+RUN chmod +x /start-celerybeat
+
+COPY ./compose/local/django/celery/flower/start /start-flower
+RUN sed -i 's/\r//' /start-flower
+RUN chmod +x /start-flower
+
 WORKDIR /app
 
 ENTRYPOINT ["/entrypoint"]

compose/local/django/celery/beat/start

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+rm -f './celerybeat.pid'
+celery -A config.celery_app beat -l INFO

compose/local/django/celery/flower/start

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+celery flower \
+    --app=config.celery_app \
+    --broker="${CELERY_BROKER_URL}" \
+    --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

compose/local/django/celery/worker/start

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+celery -A config.celery_app worker -l INFO

compose/local/tests/Dockerfile

@@ -28,8 +28,14 @@ RUN wget https://chromedriver.storage.googleapis.com/85.0.4183.83/chromedriver_l
 RUN chmod +x /tests/RunAllTests-Chrome-Linux.bat
 RUN sed -i 's/\r//' /tests/RunAllTests-Chrome-Linux.bat
 
-RUN chmod +x /tests/RunAllTests-Firefox-Linux.bat
-RUN sed -i 's/\r//' /tests/RunAllTests-Firefox-Linux.bat
+RUN chmod +x /tests/Cleanup-Docker.bat
+RUN sed -i 's/\r//' /tests/Cleanup-Docker.bat
+
+RUN chmod +x /tests/Buildup-Docker.bat
+RUN sed -i 's/\r//' /tests/Buildup-Docker.bat
+
+#RUN chmod +x /tests/RunAllTests-Firefox-Linux.bat
+#RUN sed -i 's/\r//' /tests/RunAllTests-Firefox-Linux.bat
 
 WORKDIR /tests
 

compose/production/django/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-alpine
+FROM python:3.8-alpine
 
 ENV PYTHONUNBUFFERED 1
 
@@ -9,7 +9,11 @@ RUN apk update \
   # Pillow dependencies
   && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \
   # CFFI dependencies
-  && apk add libffi-dev py-cffi
+  && apk add libffi-dev py-cffi \
+  # Translations dependencies
+  && apk add gettext \
+  # https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell
+  && apk add postgresql-client
 
 RUN addgroup -S django \
     && adduser -S -G django django
@@ -29,6 +33,21 @@ RUN sed -i 's/\r//' /start
 RUN chmod +x /start
 RUN chown django /start
 
+COPY ./compose/production/django/celery/worker/start /start-celeryworker
+RUN sed -i 's/\r//' /start-celeryworker
+RUN chmod +x /start-celeryworker
+RUN chown django /start-celeryworker
+
+COPY ./compose/production/django/celery/beat/start /start-celerybeat
+RUN sed -i 's/\r//' /start-celerybeat
+RUN chmod +x /start-celerybeat
+RUN chown django /start-celerybeat
+
+COPY ./compose/production/django/celery/flower/start /start-flower
+RUN sed -i 's/\r//' /start-flower
+RUN chmod +x /start-flower
+RUN chown django /start-flower
+
 COPY . /app
 
 RUN mkdir /app/staticfiles && mkdir /app/media

compose/production/django/celery/beat/start

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+rm -f './celerybeat.pid'
+celery -A config.celery_app beat -l INFO

compose/production/django/celery/flower/start

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+celery flower \
+    --app=config.celery_app \
+    --broker="${CELERY_BROKER_URL}" \
+    --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

compose/production/django/celery/worker/start

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+
+celery -A config.celery_app worker -l INFO

compose/production/nginx/rdb-testing.conf

@@ -1,4 +1,4 @@
-# declare rdb-testing.whoi.edu for demo purposes
+# declare rdb-testing.whoi.edu for testing purposes
 
 # Redirect all non-encrypted to encrypted
 server {
@@ -20,14 +20,40 @@ server {
     location / {
         # use variable and resolver to keep nginx from not starting if no upstream server
         resolver 127.0.0.11;
-        set $demo_server django_testing:8000;
+        set $testing_server django_testing:8000;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto https;  # <-
         proxy_set_header Host $http_host;
         proxy_redirect off;
 
         if (!-f $request_filename) {
-            proxy_pass http://$demo_server;
+            proxy_pass http://$testing_server;
+            break;
+        }
+    }
+}
+
+server {
+    server_name rdb-testing.whoi.edu;
+    listen 5555;  # <-
+
+    ssl on;  # <-
+    ssl_certificate /etc/ssl/certs/whoi-inCommon-certificate.cer;  # <-
+    ssl_certificate_key /etc/ssl/certs/whoi-inCommon-private.key;  # <-
+
+    client_max_body_size 0M;
+
+    location / {
+        # use variable and resolver to keep nginx from not starting if no upstream server
+        resolver 127.0.0.11;
+        set $flower_server flower:5555;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;  # <-
+        proxy_set_header Host $http_host;
+        proxy_redirect off;
+
+        if (!-f $request_filename) {
+            proxy_pass http://$flower_server;
             break;
         }
     }

config/celery_app.py

@@ -0,0 +1,37 @@
+"""
+# Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
+#
+# This file is part of the Roundabout Database project ("RDB" or
+# "ooicgsn-roundabout").
+#
+# ooicgsn-roundabout is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# ooicgsn-roundabout is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ooicgsn-roundabout in the COPYING.md file at the project root.
+# If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+
+app = Celery("roundabout")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()