[CPCN-326] improve(api): Use separate process for push celery worker (#…

…132)

* [CPCN-326] improve(api): Move push api/worker to separate processes

* fix mgmt_api/products (due to changes in superdesk/newsroom-core#578)

* fix(behave): Add cache_type to config

* config(fireq): Enable separate upstream for push endpoints

* Revert "config(fireq): Enable separate upstream for push endpoints"

This reverts commit 719faff.

* Remove `push` endpoints process

* improve(web wsgi): Use gunicorn_config similar to Superdesks

server/Procfile

@@ -1,6 +1,7 @@
-web: gunicorn -b 0.0.0.0:$PORT -w 3 newsroom.web.app:app
+web: gunicorn -c gunicorn_config.py newsroom.web.app:app
 websocket: python -m newsroom.web.ws
-worker: celery -A newsroom.web.worker.celery worker
+worker: celery -A newsroom.web.worker.celery worker -X "${SUPERDESK_CELERY_PREFIX}newsroom.push"
+push_worker: celery -A newsroom.web.worker.celery worker -Q "${SUPERDESK_CELERY_PREFIX}newsroom.push"
 beat: celery -A newsroom.web.worker.celery beat
 newsapi: gunicorn -b "0.0.0.0:${APIPORT:-$PORT}" -w 3 newsroom.news_api.app:app
 mgmtapi: gunicorn -b "0.0.0.0:${MGMTPORT:-$PORT}" -w 3 cp.mgmt_api.app:app

server/cp/mgmt_api/products.py

@@ -1,4 +1,4 @@
-from newsroom.products import ProductsResource, ProductsService
+from newsroom.products.products import ProductsResource, ProductsService
 import superdesk
 
 

server/features/environment.py

@@ -21,6 +21,7 @@ def before_all(context):
         'INSTALLED_APPS': [],
         'ELASTICSEARCH_FORCE_REFRESH': True,
         'MGMT_API_ENABLED': True,
+        'CACHE_TYPE': "null",
     }
     setup_before_all(context, config, app_factory=get_app)
 
@@ -33,6 +34,7 @@ def before_scenario(context, scenario):
         'ELASTICSEARCH_FORCE_REFRESH': True,
         'MGMT_API_ENABLED': True,
         'AUTH_SERVER_SHARED_SECRET': 'test-secret',
+        'CACHE_TYPE': "null",
     }
 
     setup_before_scenario(context, scenario, config, app_factory=get_app)

server/gunicorn_config.py

@@ -0,0 +1,12 @@
+import os
+import multiprocessing
+
+port = os.environ.get("PORT", "5000")
+bind = f"0.0.0.0:{port}"
+workers = int(os.environ.get("WEB_CONCURRENCY", multiprocessing.cpu_count() + 1))
+
+accesslog = "-"
+access_log_format = "%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb"
+
+reload = 'SUPERDESK_RELOAD' in os.environ
+timeout = int(os.environ.get('WEB_TIMEOUT', 30))