From 899dca6493617edd623f6e449c10bdfafc826748 Mon Sep 17 00:00:00 2001 From: Jaroslav Libak Date: Mon, 14 Sep 2015 13:46:54 +0200 Subject: [PATCH] * support for empty RabbitMQ password * server locations now defined by AMQ_SERVER and MGS_SERVER env variables to allow them to be set to skydns hostnames * return 0 exit code in case of termination --- README.md | 7 ++++--- circle.yml | 10 +++++----- server.js | 2 +- service/metrics-gateway.js | 7 ++++--- service/notification.js | 18 +++++++++++++----- tests/system/metrics-gateway-tests.js | 12 ++++-------- tests/system/notification-tests.js | 22 ++++++++++++++-------- 7 files changed, 45 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 31e406f..7e31a47 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,11 @@ Same storage path used by result upload service must be accessible to result pro ## Configuration Supported environment parameters: -- AMQ_USER, AMQ_PASSWORD for authenticating to RabbitMQ. This is temporary until a better way to retrieve credentials by services is available. +- AMQ_USER, AMQ_PASSWORD for authenticating to RabbitMQ. This is temporary until a better way to retrieve credentials by services is available. AMQ_PASSWORD is optional. +- AMQ_SERVER - location of RabbitMQ server in the form hostname:port +- MGS_SERVER - location of the metrics gateway in the form hostname:port - PROCESSORS_PATH - path where result processors can be found. If not specified, "processors" directory is used - PROCESSORS_PARALLELISM - number of data processors that can be executed in parallel. If not present, number of CPU cores is used. -- MSGW_PORT - port for metrics gateway. If not specified, 8080 is used - METRICS_BATCH_SIZE - batch size to use when sending data to metrics gateway ## Building @@ -84,7 +85,7 @@ If data processor in other language is to be implemented, a new base image needs Note that normally you will be running concrete processor Docker image, not this image. Execute: -- docker run -d -e AMQ_USER="admin" -e AMQ_PASSWORD="mypass" -v "/tmp:/upload" --link rabbitmq:amqserver --link mgs:metricsgw --name result-processing gaiaadm/result-processing:0.1 +- docker run -d -e AMQ_USER="admin" -e AMQ_SERVER="rabbitmq:5672" -e MGS_SERVER="metricsgw:8080" -v "/tmp:/upload" --link rabbitmq:rabbitmq --link mgs:metricsgw --name result-processing gaiaadm/result-processing Unless at least one processor is available the process will exit immediately. Note that the mount point for uploads must be the same in both result upload service and result processing service docker image. For development purposes usage of /tmp is sufficient. For production it needs to be NFSv4 volume. Linking requires knowledge of container name/id we are linking to (i.e "mgs", "rabbitmq" in example). diff --git a/circle.yml b/circle.yml index 7adb29b..58f9708 100644 --- a/circle.yml +++ b/circle.yml @@ -7,23 +7,23 @@ dependencies: - docker build -f distribution/release/python/Dockerfile -t gaiaadm/result-processing:$BUILD_TAG-python . test: override: - - docker run -d -e RABBITMQ_PASS="mypass" --name rabbitmq gaiaadm/rabbitmq; sleep 10 + - docker run -d --name rabbitmq gaiaadm/rabbitmq; sleep 10 # run jshint - docker run --name jshint-runner gaiaadm/result-processing:$BUILD_TAG grunt jshint # run unit tests on pure Node.js image - - docker run -e AMQ_USER="admin" -e AMQ_PASSWORD="mypass" --link rabbitmq:amqserver --name unit-test-runner gaiaadm/result-processing:$BUILD_TAG grunt unit + - docker run --name unit-test-runner gaiaadm/result-processing:$BUILD_TAG grunt unit - docker cp unit-test-runner:/src/unit-test-results.xml $CIRCLE_TEST_REPORTS/nodejs - docker cp unit-test-runner:/src/unit-test-results.txt $CIRCLE_ARTIFACTS/nodejs # run system tests on pure Node.js image - - docker run -e AMQ_USER="admin" -e AMQ_PASSWORD="mypass" --link rabbitmq:amqserver --name system-test-runner gaiaadm/result-processing:$BUILD_TAG grunt system + - docker run -e AMQ_USER="admin" -e AMQ_SERVER="rabbitmq:5672" --link rabbitmq:rabbitmq --name system-test-runner gaiaadm/result-processing:$BUILD_TAG grunt system - docker cp system-test-runner:/src/system-test-results.xml $CIRCLE_TEST_REPORTS/nodejs - docker cp system-test-runner:/src/system-test-results.txt $CIRCLE_ARTIFACTS/nodejs # run unit tests on Python image - - docker run -e AMQ_USER="admin" -e AMQ_PASSWORD="mypass" --link rabbitmq:amqserver --name unit-test-runner-python gaiaadm/result-processing:$BUILD_TAG-python grunt unit + - docker run --name unit-test-runner-python gaiaadm/result-processing:$BUILD_TAG-python grunt unit - docker cp unit-test-runner-python:/src/unit-test-results.xml $CIRCLE_TEST_REPORTS/python - docker cp unit-test-runner-python:/src/unit-test-results.txt $CIRCLE_ARTIFACTS/python # run system tests on Python image - - docker run -e AMQ_USER="admin" -e AMQ_PASSWORD="mypass" --link rabbitmq:amqserver --name system-test-runner-python gaiaadm/result-processing:$BUILD_TAG-python grunt system + - docker run -e AMQ_USER="admin" -e AMQ_SERVER="rabbitmq:5672" --link rabbitmq:rabbitmq --name system-test-runner-python gaiaadm/result-processing:$BUILD_TAG-python grunt system - docker cp system-test-runner-python:/src/system-test-results.xml $CIRCLE_TEST_REPORTS/python - docker cp system-test-runner-python:/src/system-test-results.txt $CIRCLE_ARTIFACTS/python deployment: diff --git a/server.js b/server.js index efd6845..e21e613 100644 --- a/server.js +++ b/server.js @@ -24,7 +24,7 @@ exitOnSignal('SIGTERM'); function exitOnSignal(signal) { process.on(signal, function() { logger.debug('Caught ' + signal + ', exiting'); - graceApp.shutdown(1); + graceApp.shutdown(0); }); } diff --git a/service/metrics-gateway.js b/service/metrics-gateway.js index 60bdc97..a7f1ea7 100644 --- a/service/metrics-gateway.js +++ b/service/metrics-gateway.js @@ -17,9 +17,10 @@ var logger = log4js.getLogger('metrics_gateway.js'); * @returns {string} */ function getSendUri() { - var host = process.env.MSGW_HOST || 'metricsgw'; - var port = process.env.MSGW_PORT || 8080; - return 'http://' + host + ':' + port + '/mgs/rest/v1/gateway/event'; + if (!process.env.MGS_SERVER) { + throw new Error('MGS_SERVER environment variable is not specified'); + } + return 'http://' + process.env.MGS_SERVER + '/mgs/rest/v1/gateway/event'; } /** diff --git a/service/notification.js b/service/notification.js index 975b7ee..a88ea95 100644 --- a/service/notification.js +++ b/service/notification.js @@ -27,14 +27,22 @@ function getAmqCredentials() { if (!process.env.AMQ_USER) { throw new Error('AMQ_USER environment variable is not specified'); } - if (!process.env.AMQ_PASSWORD) { - throw new Error('AMQ_PASSWORD environment variable is not specified'); - } + var pwd = process.env.AMQ_PASSWORD ? process.env.AMQ_PASSWORD : ''; return { - username: process.env.AMQ_USER, password: process.env.AMQ_PASSWORD + username: process.env.AMQ_USER, password: pwd }; } +/** + * Returns hostname:port of RabbitMQ server. + */ +function getAmqServer() { + if (!process.env.AMQ_SERVER) { + throw new Error('AMQ_SERVER environment variable is not specified'); + } + return process.env.AMQ_SERVER; +} + /** * Returns number of processors that may execute in parallel. */ @@ -160,7 +168,7 @@ function initChannel(conn, processorDescs, msgConsumer) { function initAmq(processorDescs, msgConsumer) { var credentials = getAmqCredentials(); var url = 'amqp://' + credentials.username + ':' + credentials.password + - '@amqserver:5672?frameMax=0x1000&heartbeat=30'; + '@' + getAmqServer() + '?frameMax=0x1000&heartbeat=30'; return amqp.connect(url).then(function(conn) { // TODO: handle reconnect in case close caused by certain errors (not invalid credentials) diff --git a/tests/system/metrics-gateway-tests.js b/tests/system/metrics-gateway-tests.js index 559471e..1ef9e74 100644 --- a/tests/system/metrics-gateway-tests.js +++ b/tests/system/metrics-gateway-tests.js @@ -12,13 +12,11 @@ describe('metrics-gateway client tests', function() { describe('invalid metrics-gateway-service', function() { afterEach(function() { - process.env.MSGW_HOST = null; - process.env.MSGW_PORT = null; + process.env.MGS_SERVER = null; }); it('send with invalid hostname', function(done) { - process.env.MSGW_HOST = 'localhost'; - process.env.MSGW_PORT = 32000; + process.env.MGS_SERVER = 'localhost:32000'; metricsGateway.send({accessToken: 'accessToken'}, {some:'data'}, function(err) { assert(err instanceof Error, 'Expected the callback be called with an error'); done(); @@ -32,7 +30,6 @@ describe('metrics-gateway client tests', function() { var server; before(function(done) { - process.env.MSGW_HOST = 'localhost'; // start internal express server on random port app = express(); var router = express.Router(); @@ -48,7 +45,7 @@ describe('metrics-gateway client tests', function() { app.use(defaultErrorHandler); server = http.createServer(app); server.listen(0, function () { - process.env.MSGW_PORT = server.address().port; + process.env.MGS_SERVER = 'localhost:' + server.address().port; done(); }); }); @@ -103,8 +100,7 @@ describe('metrics-gateway client tests', function() { }); after(function(done) { - process.env.MSGW_HOST = null; - process.env.MSGW_PORT = null; + process.env.MGS_SERVER = null; server.close(done); }); diff --git a/tests/system/notification-tests.js b/tests/system/notification-tests.js index d9692c1..dafef05 100644 --- a/tests/system/notification-tests.js +++ b/tests/system/notification-tests.js @@ -132,16 +132,22 @@ describe('notification tests', function() { function getAmqUrl() { if (!process.env.AMQ_USER) { - throw new Error('AMQ_USER environment variable is not specified'); + throw new Error('AMQ_USER environment variable is not specified'); } - if (!process.env.AMQ_PASSWORD) { - throw new Error('AMQ_PASSWORD environment variable is not specified'); - } - var amq_hostname = process.env.AMQ_HOSTNAME || 'amqserver'; - var amq_port = process.env.AMQ_PORT || '5672'; + var pwd = process.env.AMQ_PASSWORD ? process.env.AMQ_PASSWORD : ''; + + return 'amqp://' + process.env.AMQ_USER + ':' + pwd + + '@' + getAmqServer() + '?frameMax=0x1000&heartbeat=30'; + } - return 'amqp://' + process.env.AMQ_USER + ':' + process.env.AMQ_PASSWORD + - '@' + amq_hostname + ':' + amq_port + '?frameMax=0x1000&heartbeat=30'; + /** + * Returns hostname:port of RabbitMQ server. + */ + function getAmqServer() { + if (!process.env.AMQ_SERVER) { + throw new Error('AMQ_SERVER environment variable is not specified'); + } + return process.env.AMQ_SERVER; } function send(contentMetadata, callback) {