Merge pull request #1174 from BlackOrder/logging-to-stdout-for-HTTP-s…

…erver-and-REST-API

Add logging to stdout for HTTP server and REST API

docker/Dockerfile.restapi

@@ -6,16 +6,18 @@ MAINTAINER Jookies LTD <[email protected]>
 RUN groupadd -r jasmin && useradd -r -g jasmin jasmin
 
 # Install requirements
-RUN apt-get update && apt-get install -y \
-    libffi-dev \
-    libssl-dev \
-    # Run python with jemalloc
-    # More on this:
-    # - https://zapier.com/engineering/celery-python-jemalloc/
-    # - https://paste.pics/581cc286226407ab0be400b94951a7d9
-    libjemalloc2
-
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update; \
+    apt-get install -y --no-install-recommends \
+        libffi-dev \
+        libssl-dev \
+        # Run python with jemalloc
+        # More on this:
+        # - https://zapier.com/engineering/celery-python-jemalloc/
+        # - https://paste.pics/581cc286226407ab0be400b94951a7d9
+        libjemalloc2 \
+    ; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*
 
 # Run python with jemalloc
 ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -26,15 +28,20 @@ ENV RESOURCE_PATH ${CONFIG_PATH}/resource
 ENV STORE_PATH ${CONFIG_PATH}/store
 ENV LOG_PATH /var/log/jasmin
 
-RUN mkdir -p ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
+RUN mkdir -p ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 RUN chown jasmin:jasmin ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 
 WORKDIR /build
 
+RUN pip install --upgrade pip
+
 COPY . .
 
 RUN pip install .
 
+# Clean build dir
+RUN rm -rf /build
+
 # For RestAPI MODE
 RUN pip install gunicorn
 
@@ -45,26 +52,17 @@ COPY misc/config/resource ${RESOURCE_PATH}
 
 WORKDIR /etc/jasmin
 
-# Default Redis and RabbitMQ connections
+# Default Jasmin, Redis, and RabbitMQ connections
+ENV JCLI_BIND '0.0.0.0'
 ENV AMQP_BROKER_HOST 'rabbitmq'
 ENV AMQP_BROKER_PORT 5672
 ENV REDIS_CLIENT_HOST 'redis'
 ENV REDIS_CLIENT_PORT 6379
 
-# For RestAPI MODE
-ENV RESTAPI_MODE 0
-ENV RESTAPI_OLD_HTTP_HOST '127.0.0.1'
-
-ENV ENABLE_PUBLISH_SUBMIT_SM_RESP 0
-
-# Change binding host for jcli
-RUN sed -i '/\[jcli\]/a bind=0.0.0.0' ${CONFIG_PATH}/jasmin.cfg
-# Change binding port for redis, and amqp
-RUN sed -i "/\[redis-client\]/a port=$REDIS_CLIENT_PORT" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a port=$AMQP_BROKER_PORT" ${CONFIG_PATH}/jasmin.cfg
-# Change binding host for redis, and amqp
-RUN sed -i "/\[redis-client\]/a host=$REDIS_CLIENT_HOST" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a host=$AMQP_BROKER_HOST" ${CONFIG_PATH}/jasmin.cfg
+# For RestAPI http mode
+ENV RESTAPI_HTTP_MODE 0
+# For RestAPI Celery worker mode
+ENV RESTAPI_WORKER_MODE 0
 
 EXPOSE 2775 8990 1401 8080
 VOLUME ["/var/log/jasmin", "/etc/jasmin", "/etc/jasmin/store"]

docker/Dockerfile.restapi.alpine

@@ -6,60 +6,54 @@ MAINTAINER Jookies LTD <[email protected]>
 RUN addgroup -S jasmin && adduser -S -g jasmin jasmin
 
 # Install requirements
-RUN apk --update add \
-    gcc \
-    musl-dev \
-    libffi-dev \
-    openssl-dev \
-    python3-dev \
-    py3-pip \
-    git \
-    bash 
-
-# For RestAPI MODE
-ENV RESTAPI_MODE 0
-ENV RESTAPI_OLD_HTTP_HOST '127.0.0.1'
-
-ENV ENABLE_PUBLISH_SUBMIT_SM_RESP 0
+RUN apk --no-cache add \
+        libffi-dev \
+        openssl-dev \
+        bash \
+    ; \
+    rm -vrf /var/cache/apk/*
 
 ENV ROOT_PATH /
 ENV CONFIG_PATH /etc/jasmin
-ENV RESOURCE_PATH /etc/jasmin/resource
-ENV STORE_PATH /etc/jasmin/store
+ENV RESOURCE_PATH ${CONFIG_PATH}/resource
+ENV STORE_PATH ${CONFIG_PATH}/store
 ENV LOG_PATH /var/log/jasmin
 
 RUN mkdir -p ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 RUN chown jasmin:jasmin ${CONFIG_PATH} ${RESOURCE_PATH} ${STORE_PATH} ${LOG_PATH}
 
 WORKDIR /build
 
-RUN pip install -e git+https://github.com/jookies/txamqp.git@master#egg=txamqp3
-RUN pip install -e git+https://github.com/jookies/python-messaging.git@master#egg=python-messaging
-RUN pip install -e git+https://github.com/jookies/smpp.pdu.git@master#egg=smpp.pdu3
-RUN pip install -e git+https://github.com/jookies/smpp.twisted.git@master#egg=smpp.twisted3
+RUN pip install --upgrade pip
 
 COPY . .
 
 RUN pip install .
 
+# Clean build dir
+RUN rm -rf /build
+
 # For RestAPI MODE
 RUN pip install gunicorn
 
 ENV UNICODEMAP_JP unicode-ascii
 
 COPY misc/config/*.cfg ${CONFIG_PATH}/
-COPY misc/config/resource/*.xml ${RESOURCE_PATH}/
+COPY misc/config/resource ${RESOURCE_PATH}
 
 WORKDIR /etc/jasmin
 
-# Change binding host for jcli
-RUN sed -i '/\[jcli\]/a bind=0.0.0.0' ${CONFIG_PATH}/jasmin.cfg
-# Change binding port for redis, and amqp
-RUN sed -i "/\[redis-client\]/a port=$REDIS_CLIENT_PORT" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a port=$AMQP_BROKER_PORT" ${CONFIG_PATH}/jasmin.cfg
-# Change binding host for redis, and amqp
-RUN sed -i "/\[redis-client\]/a host=$REDIS_CLIENT_HOST" ${CONFIG_PATH}/jasmin.cfg
-RUN sed -i "/\[amqp-broker\]/a host=$AMQP_BROKER_HOST" ${CONFIG_PATH}/jasmin.cfg
+# Default Jasmin, Redis, and RabbitMQ connections
+ENV JCLI_BIND '0.0.0.0'
+ENV AMQP_BROKER_HOST 'rabbitmq'
+ENV AMQP_BROKER_PORT 5672
+ENV REDIS_CLIENT_HOST 'redis'
+ENV REDIS_CLIENT_PORT 6379
+
+# For RestAPI http mode
+ENV RESTAPI_HTTP_MODE 0
+# For RestAPI Celery worker mode
+ENV RESTAPI_WORKER_MODE 0
 
 EXPOSE 2775 8990 1401 8080
 VOLUME ["/var/log/jasmin", "/etc/jasmin", "/etc/jasmin/store"]

docker/restapi-docker-entrypoint.sh

@@ -1,40 +1,26 @@
 #!/bin/bash
 set -e
 
-# Change binding host:port for redis, and amqp
-sed -i "/\[redis-client\]/,/host=/  s/host=.*/host=$REDIS_CLIENT_HOST/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[redis-client\]/,/port=/  s/port=.*/port=$REDIS_CLIENT_PORT/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[amqp-broker\]/,/host=/  s/host=.*/host=$AMQP_BROKER_HOST/" ${CONFIG_PATH}/jasmin.cfg
-sed -i "/\[amqp-broker\]/,/port=/  s/port=.*/port=$AMQP_BROKER_PORT/" ${CONFIG_PATH}/jasmin.cfg
-
+# Clean lock files
 echo 'Cleaning lock files'
 rm -f /tmp/*.lock
 
-# RestAPI
-if [ "$RESTAPI_MODE" = 1 ]; then
-  # find jasmin installation directory
-  jasminRoot=$(python -c "import jasmin as _; print(_.__path__[0])")
-  # update jasmin-restAPI config
-  sed -i "/# RESTAPI/,/old_api_uri/  s/old_api_uri.*/old_api_uri = 'http:\/\/$RESTAPI_OLD_HTTP_HOST:1401'/" ${jasminRoot}/protocols/rest/config.py 
-  sed -i "/# CELERY/,/broker_url/  s/broker_url.*/broker_url = 'amqp:\/\/guest:guest@$AMQP_BROKER_HOST:$AMQP_BROKER_PORT\/\/'/" ${jasminRoot}/protocols/rest/config.py 
-  sed -i "/# CELERY/,/result_backend/  s/result_backend.*/result_backend = 'redis:\/\/:@$REDIS_CLIENT_HOST:$REDIS_CLIENT_PORT\/1'/" ${jasminRoot}/protocols/rest/config.py 
+# If RestAPI http Mode, start Guicorn
+if [ "$RESTAPI_HTTP_MODE" = 1 ]; then
+  # start restapi
   exec gunicorn -b 0.0.0.0:8080 jasmin.protocols.rest:api --access-logfile /var/log/jasmin/rest-api.access.log --disable-redirect-access-to-syslog
-else
-  if [ "$ENABLE_PUBLISH_SUBMIT_SM_RESP" = 1 ]; then
-    # Enable publish_submit_sm_resp
-    echo 'Enabling publish_submit_sm_resp'
-    sed -i "s/.*publish_submit_sm_resp\s*=.*/publish_submit_sm_resp=True/g" ${CONFIG_PATH}/jasmin.cfg
-  else
-    # Disable publish_submit_sm_resp
-    echo 'Disabling publish_submit_sm_resp'
-    sed -i "s/.*publish_submit_sm_resp\s*=.*/publish_submit_sm_resp=False/g" ${CONFIG_PATH}/jasmin.cfg
-  fi
 
+# If Celery Worker is enabled, start Celery worker
+elif [ "$RESTAPI_WORKER_MODE" = 1 ]; then
+  echo 'Starting Celery worker'
+  exec celery -A jasmin.protocols.rest.tasks worker -l INFO -c 4 --autoscale=10,3
+
+# Else start jasmind
+else
   if [ "$2" = "--enable-interceptor-client" ]; then
     echo 'Starting interceptord'
     interceptord.py &
   fi
-
   echo 'Starting jasmind'
   exec "$@"
 fi

jasmin/protocols/http/server.py

@@ -1,6 +1,7 @@
 import logging
 from datetime import datetime
 from logging.handlers import TimedRotatingFileHandler
+import sys
 
 from twisted.internet import reactor
 from twisted.web.resource import Resource
@@ -32,7 +33,10 @@ def __init__(self, RouterPB, SMPPClientManagerPB, config, interceptor=None):
         log = logging.getLogger(LOG_CATEGORY)
         if len(log.handlers) != 1:
             log.setLevel(config.log_level)
-            handler = TimedRotatingFileHandler(filename=config.log_file, when=config.log_rotate)
+            if 'stdout' in config.log_file:
+                handler = logging.StreamHandler(sys.stdout)
+            else:
+                handler = TimedRotatingFileHandler(filename=config.log_file, when=config.log_rotate)
             formatter = logging.Formatter(config.log_format, config.log_date_format)
             handler.setFormatter(formatter)
             log.addHandler(handler)

jasmin/protocols/rest/__init__.py

@@ -5,19 +5,13 @@
 import sys
 
 from .api import PingResource, BalanceResource, RateResource, SendResource, SendBatchResource
-from .config import *
+from .config import RestAPIForJasminConfig
 
 sys.path.append("%s/vendor" % os.path.dirname(os.path.abspath(jasmin.__file__)))
 from falcon import HTTPUnauthorized, HTTPUnsupportedMediaType, API
 
-# @TODO: make configuration loadable from /etc/jasmin/restapi.conf
-logger = logging.getLogger('jasmin-restapi')
-if len(logger.handlers) == 0:
-    logger.setLevel(log_level)
-    handler = logging.handlers.TimedRotatingFileHandler(filename=log_file, when=log_rotate)
-    handler.setFormatter(logging.Formatter(log_format, log_date_format))
-    logger.addHandler(handler)
-
+RestAPIForJasminConfigInstance = RestAPIForJasminConfig()
+logger = RestAPIForJasminConfigInstance.logger
 
 class TokenNotFound(Exception):
     """Raised when authentication token is not found"""
@@ -36,7 +30,7 @@ def process_response(self, request, response, resource, req_succeeded):
             response.body = json.dumps(response.body)
 
         # Add Jasmin signature
-        if show_jasmin_version:
+        if RestAPIForJasminConfigInstance.show_jasmin_version:
             response.set_header('Powered-By', 'Jasmin %s' % jasmin.get_release())
 
 

jasmin/protocols/rest/api.py

@@ -6,24 +6,25 @@
 import requests
 
 import jasmin
-from .config import *
+from .config import RestAPIForJasminConfig
 from .tasks import httpapi_send
 from datetime import datetime
 from falcon import HTTPInternalServerError, HTTPPreconditionFailed
 import falcon
 
 sys.path.append("%s/vendor" % os.path.dirname(os.path.abspath(jasmin.__file__)))
 
+RestAPIForJasminConfigInstance = RestAPIForJasminConfig()
 
 class JasminHttpApiProxy:
     """Provides a WS caller for old Jasmin http api"""
 
     def call_jasmin(self, url, params=None):
         try:
-            r = requests.get('%s/%s' % (old_api_uri, url), params=params)
+            r = requests.get('%s/%s' % (RestAPIForJasminConfigInstance.http_api_uri, url), params=params)
         except requests.exceptions.ConnectionError as e:
             raise HTTPInternalServerError('Jasmin httpapi connection error',
-                                          'Could not connect to Jasmin http api (%s): %s' % (old_api_uri, e))
+                                          'Could not connect to Jasmin http api (%s): %s' % (RestAPIForJasminConfigInstance.http_api_uri, e))
         except Exception as e:
             raise HTTPInternalServerError('Jasmin httpapi unknown error', str(e))
         else:
@@ -202,7 +203,7 @@ def on_post(self, request, response):
 
         batch_id = uuid.uuid4()
         params = self.decode_request_data(request)
-        config = {'throughput': http_throughput_per_worker, 'smart_qos': smart_qos}
+        config = {'throughput': RestAPIForJasminConfigInstance.http_throughput_per_worker, 'smart_qos': RestAPIForJasminConfigInstance.smart_qos}
 
         # Batch scheduling
         countdown = self.parse_schedule_at(params.get('batch_config', {}).get('schedule_at', None))