From 50be6318a4aeb197587ae37cc00511988e70b86a Mon Sep 17 00:00:00 2001 From: Jaroslav Libak Date: Wed, 12 Aug 2015 16:25:21 +0200 Subject: [PATCH] changed parameter prefix to uppercase and all parameter keys are converted to uppercase as well due to difference between Windows and Linux in env parameter key case sensitivity --- README.md | 4 ++-- service/processors.js | 5 +++-- tests/system/dev-processors/dummy/processor.js | 2 +- tests/system/processors-tests.js | 6 +++--- tests/system/test-processors/processing-error/processor.js | 2 +- .../test-processors/produce-invalid-objects/processor.js | 2 +- .../test-processors/produce-many-objects/processor.js | 2 +- .../system/test-processors/produce-two-objects/processor.js | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 97130f1..9c73aab 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ Result processing component consists of two parts: - are executed as independent processes (thus eliminating problem with memory leaks) - can be implemented in any programing language - come with file named "processor-descriptor.json" which defines processor name, program to execute and declaration of what data it is capable of processing -- receive parameters as environment variables prefixed with "p_". The following environment variables can be expected: "p_dataType", "p_contentType". "p_contentType" represents the HTTP Content-Type header value. Custom metadata from data providers will be accessible with prefix "p_c_". +- receive parameters as environment variables prefixed with "P_". The following environment variables can be expected: "P_DATATYPE", "P_CONTENTTYPE". "P_CONTENTTYPE" represents the HTTP Content-Type header value. Custom metadata from data providers will be accessible with prefix "P_C_". Note that received parameter keys will always be uppercase regardless of the case used during data collection. This is to ensure compatibility between Windows (dev) and Linux (production) environments. Parameter values are case sensitive. - receive uploaded file on STDIN. The file can be binary or textual (i.e XML, JSON) and in theory can be quite big. It is not recommended to parse it at once. Processing ends when EOF is received from STDIN. - processed results are written to STDOUT in the form of JSON array containing JSON objects. JSON objects must have format expected by metrics-gateway-service ("/mgs/rest/v1/gateway/event"). It is recommended to write JSON objects to STDOUT while processing STDIN. - log can be written to STDERR (of all log levels, not just errors). It ends up in result upload service log under processor name. - must exit with 0 if there was no error, 1 if there was a general error - should support SIGTERM to terminate processing. After SIGTERM is sent, any output produced will be ignored by result processing service. When SIGTERM is received, STDIN will be closed as well (which may lead to parsing error due to incomplete input). SIGTERM is then a hint to application that this state is desired. -- must support execution when STDIN is closed immediately, no "p_" parameters are present and exit with 0. This is used by result processing service to verify that the processor can be executed successfully. +- must support execution when STDIN is closed immediately, no "P_" parameters are present and exit with 0. This is used by result processing service to verify that the processor can be executed successfully. Result processors receive parameters as environment variables instead of process arguments since in general its easier to process environment variables than command line arguments where CLI parsing libraries are necessary. diff --git a/service/processors.js b/service/processors.js index 99fba3b..78e9216 100644 --- a/service/processors.js +++ b/service/processors.js @@ -161,7 +161,7 @@ function onLogFromChild(processorDesc, str) { * Converts content metadata properties to environment variables object that can be used for process execution. We use * environment variables instead of process arguments since for program it may not be easy to read the arguments - one * has to use specialized argument parsing libraries which may be buggy or work differently. Its much easier to access - * process environment variables. Environment variables will be prefixed with p_. String case is preserved. + * process environment variables. Environment variables will be prefixed with P_. String case is preserved. * * @param contentMetadata */ @@ -170,7 +170,8 @@ function getEnvParams(contentMetadata) { var keys = Object.keys(contentMetadata); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - envParams['p_' + key] = contentMetadata[key]; + // we convert key to uppercase since in Windows env variable keys are not case sensitive, while in Linux they are + envParams['P_' + key.toUpperCase()] = contentMetadata[key]; } return envParams; } diff --git a/tests/system/dev-processors/dummy/processor.js b/tests/system/dev-processors/dummy/processor.js index e9ded1c..219ac8f 100644 --- a/tests/system/dev-processors/dummy/processor.js +++ b/tests/system/dev-processors/dummy/processor.js @@ -7,7 +7,7 @@ console.error('Process parameters:'); var keys = Object.keys(process.env); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - if (key.lastIndexOf('p_', 0) === 0) { + if (key.lastIndexOf('P_', 0) === 0) { var value = process.env[key]; paramCount++; console.error(key.substr(2) + ' = ' + value); diff --git a/tests/system/processors-tests.js b/tests/system/processors-tests.js index d97b46b..f7b0311 100644 --- a/tests/system/processors-tests.js +++ b/tests/system/processors-tests.js @@ -168,7 +168,7 @@ describe('processors tests', function() { // data is JSON object notifier.on('data', function(data) { dataSeen++; - assert.strictEqual(data.dataType, 'produce-two-objects-metric/produce-two-objects-category' + dataSeen, 'Expected correct dataType value'); + assert.strictEqual(data.DATATYPE, 'produce-two-objects-metric/produce-two-objects-category' + dataSeen, 'Expected correct dataType value'); }); }); @@ -190,7 +190,7 @@ describe('processors tests', function() { // data is JSON object notifier.on('data', function(data) { dataSeen++; - assert.strictEqual(data.dataType, 'produce-many-objects-metric/produce-many-objects-category' + dataSeen, 'Expected correct dataType value'); + assert.strictEqual(data.DATATYPE, 'produce-many-objects-metric/produce-many-objects-category' + dataSeen, 'Expected correct dataType value'); }); }); @@ -243,7 +243,7 @@ describe('processors tests', function() { notifier.on('data', function(data) { // should be invoked only 1x dataSeen++; - assert.strictEqual(data.dataType, 'produce-invalid-objects-metric/produce-invalid-objects-category' + dataSeen, 'Expected correct dataType value'); + assert.strictEqual(data.DATATYPE, 'produce-invalid-objects-metric/produce-invalid-objects-category' + dataSeen, 'Expected correct dataType value'); }); notifier.on('error', function(err) { errorSeen = true; diff --git a/tests/system/test-processors/processing-error/processor.js b/tests/system/test-processors/processing-error/processor.js index eee163d..c563061 100644 --- a/tests/system/test-processors/processing-error/processor.js +++ b/tests/system/test-processors/processing-error/processor.js @@ -4,7 +4,7 @@ var paramCount = 0; var keys = Object.keys(process.env); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - if (key.lastIndexOf('p_', 0) === 0) { + if (key.lastIndexOf('P_', 0) === 0) { var value = process.env[key]; paramCount++; } diff --git a/tests/system/test-processors/produce-invalid-objects/processor.js b/tests/system/test-processors/produce-invalid-objects/processor.js index 224d17a..e23908c 100644 --- a/tests/system/test-processors/produce-invalid-objects/processor.js +++ b/tests/system/test-processors/produce-invalid-objects/processor.js @@ -6,7 +6,7 @@ var paramCount = 0; var keys = Object.keys(process.env); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - if (key.lastIndexOf('p_', 0) === 0) { + if (key.lastIndexOf('P_', 0) === 0) { var value = process.env[key]; params[key.substr(2)] = value; paramCount++; diff --git a/tests/system/test-processors/produce-many-objects/processor.js b/tests/system/test-processors/produce-many-objects/processor.js index 880c912..33a50df 100644 --- a/tests/system/test-processors/produce-many-objects/processor.js +++ b/tests/system/test-processors/produce-many-objects/processor.js @@ -6,7 +6,7 @@ function getParams() { var keys = Object.keys(process.env); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - if (key.lastIndexOf('p_', 0) === 0) { + if (key.lastIndexOf('P_', 0) === 0) { var value = process.env[key]; params[key.substr(2)] = value; } diff --git a/tests/system/test-processors/produce-two-objects/processor.js b/tests/system/test-processors/produce-two-objects/processor.js index f235ec6..3e14a6d 100644 --- a/tests/system/test-processors/produce-two-objects/processor.js +++ b/tests/system/test-processors/produce-two-objects/processor.js @@ -6,7 +6,7 @@ function getParams() { var keys = Object.keys(process.env); for (var i = 0; i < keys.length; i++) { var key = keys[i]; - if (key.lastIndexOf('p_', 0) === 0) { + if (key.lastIndexOf('P_', 0) === 0) { var value = process.env[key]; params[key.substr(2)] = value; }