From 0801e1fc22a5e194c3fdf430a8a10ce1a01b9e65 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 09:50:24 +0200 Subject: [PATCH 01/23] allow to send to CB batch update for multimeasures --- lib/services/ngsi/entities-NGSI-v2.js | 563 ++++++++++++++------------ 1 file changed, 293 insertions(+), 270 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index dc5e3b733..d8514b6ac 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -260,7 +260,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal * @param {Object} typeInformation Configuration information for the device. * @param {String} token User token to identify against the PEP Proxies (optional). */ -function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, callback) { +function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token, callback) { //aux function used to builf JEXL context. //it returns a flat object from an Attr array function reduceAttrToPlainObject(attrs, initObj = {}) { @@ -274,335 +274,354 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call } } - let entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture - let jexlctxt = {}; //will store the whole context (not just for JEXL) - let payload = {}; //will store the final payload - let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); //Make a clone and overwrite typeInformation = JSON.parse(JSON.stringify(typeInformation)); - //Check mandatory information: type if (!typeInformation || !typeInformation.type) { callback(new errors.TypeNotFound(null, entityName, typeInformation)); return; } - //Rename all measures with matches with id and type to measure_id and measure_type - for (let measure of measures) { - if (measure.name === 'id' || measure.name === 'type') { - measure.name = constants.MEASURE + measure.name; - } - } - - //Make a copy of measures in an plain object: plainMeasures - plainMeasures = reduceAttrToPlainObject(measures); - //Build the initital JEXL Context - //All the measures (avoid references make another copy instead) - jexlctxt = reduceAttrToPlainObject(measures); - //All the static - jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); - //id type Service and Subservice - jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); + let payload = {}; //will store the final payload + let entities = {}; + payload.actionType = 'append'; + payload.entities = []; + const currentIsoDate = new Date().toISOString(); + const currentMoment = moment(currentIsoDate); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; - logger.debug( - context, - 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', - entityName, - plainMeasures, - typeInformation, - jexlctxt, - mustInsertTimeInstant - ); + // Check if measures is a single measure or a array of measures (a multimeasure) + if (originMeasures[0] && !originMeasures[0][0]) { + originMeasures = [originMeasures]; + } + for (let measures of originMeasures) { + entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture + let jexlctxt = {}; //will store the whole context (not just for JEXL) - //Now we can calculate the EntityName of primary entity - let entityNameCalc = null; - if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { - try { - logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); - entityNameCalc = expressionPlugin.applyExpression(typeInformation.entityNameExp, jexlctxt, typeInformation); - } catch (e) { - logger.debug( - context, - 'Error evaluating expression for entityName: %j with context: %j', - typeInformation.entityNameExp, - jexlctxt - ); + let plainMeasures = null; //will contain measures POJO + + //Rename all measures with matches with id and type to measure_id and measure_type + for (let measure of measures) { + if (measure.name === 'id' || measure.name === 'type') { + measure.name = constants.MEASURE + measure.name; + } } - } - entityName = entityNameCalc ? entityNameCalc : entityName; - //enrich JEXL context - jexlctxt['entity_name'] = entityName; + //Make a copy of measures in an plain object: plainMeasures + plainMeasures = reduceAttrToPlainObject(measures); + //Build the initital JEXL Context + //All the measures (avoid references make another copy instead) + jexlctxt = reduceAttrToPlainObject(measures); + //All the static + jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); + //id type Service and Subservice + jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); - let preprocessedAttr = []; - //Add Raw Static, Lazy, Command and Actives attr attributes - if (typeInformation && typeInformation.staticAttributes) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); - } - if (typeInformation && typeInformation.lazy) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); - } - if (typeInformation && typeInformation.active) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.active); - } + logger.debug( + context, + 'sendUpdateValueNgsi2 loop with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', + entityName, + plainMeasures, + typeInformation, + jexlctxt, + mustInsertTimeInstant + ); - //Proccess every proto Attribute to populate entities data steuture - entities[entityName] = {}; - entities[entityName][typeInformation.type] = []; - - for (let currentAttr of preprocessedAttr) { - let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) - let attrEntityName = entityName; - let attrEntityType = typeInformation.type; - let valueExpression = null; - //manage active attr without object__id (name by default) - currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; - //Enrich the attr (skip, hit, value, meta-timeInstant) - currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; - - //determine AttrEntityName for multientity - if ( - currentAttr.entity_name !== null && - currentAttr.entity_name !== undefined && - currentAttr.entity_name !== '' && - typeof currentAttr.entity_name == 'string' - ) { + //Now we can calculate the EntityName of primary entity + let entityNameCalc = null; + if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { try { - logger.debug( - context, - 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', - currentAttr.name, - currentAttr.entity_name, - jexlctxt + logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); + entityNameCalc = expressionPlugin.applyExpression( + typeInformation.entityNameExp, + jexlctxt, + typeInformation ); - attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); - if (!attrEntityName) { - attrEntityName = currentAttr.entity_name; - } } catch (e) { logger.debug( context, - 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', - currentAttr.entity_name, + 'Error evaluating expression for entityName: %j with context: %j', + typeInformation.entityNameExp, jexlctxt ); - attrEntityName = currentAttr.entity_name; } } - //determine AttrEntityType for multientity - if ( - currentAttr.entity_type !== null && - currentAttr.entity_type !== undefined && - currentAttr.entity_type !== '' && - typeof currentAttr.entity_type === 'string' - ) { - attrEntityType = currentAttr.entity_type; - } + entityName = entityNameCalc ? entityNameCalc : entityName; + //enrich JEXL context + jexlctxt['entity_name'] = entityName; - //PRE POPULATE CONTEXT - jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; - - //determine Value - if (currentAttr.value !== undefined) { - //static attributes already have a value - hitted = true; - valueExpression = currentAttr.value; - } else if (plainMeasures[currentAttr.object_id] !== undefined) { - //we have got a meaure for that Attr - //actives ¿lazis? - hitted = true; - valueExpression = plainMeasures[currentAttr.object_id]; + let preprocessedAttr = []; + //Add Raw Static, Lazy, Command and Actives attr attributes + if (typeInformation && typeInformation.staticAttributes) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); } - //remove measures that has been shadowed by an alias (some may be left and managed later) - //Maybe we must filter object_id if there is name == object_id - measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); - - if ( - currentAttr.expression !== undefined && - currentAttr.expression !== '' && - typeof currentAttr.expression == 'string' - ) { - try { + if (typeInformation && typeInformation.lazy) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); + } + if (typeInformation && typeInformation.active) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.active); + } + + //Proccess every proto Attribute to populate entities data steuture + entities[entityName] = {}; + entities[entityName][typeInformation.type] = []; + + for (let currentAttr of preprocessedAttr) { + let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) + let attrEntityName = entityName; + let attrEntityType = typeInformation.type; + let valueExpression = null; + //manage active attr without object__id (name by default) + currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; + //Enrich the attr (skip, hit, value, meta-timeInstant) + currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; + + //determine AttrEntityName for multientity + if ( + currentAttr.entity_name !== null && + currentAttr.entity_name !== undefined && + currentAttr.entity_name !== '' && + typeof currentAttr.entity_name == 'string' + ) { + try { + logger.debug( + context, + 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', + currentAttr.name, + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); + if (!attrEntityName) { + attrEntityName = currentAttr.entity_name; + } + } catch (e) { + logger.debug( + context, + 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = currentAttr.entity_name; + } + } + + //determine AttrEntityType for multientity + if ( + currentAttr.entity_type !== null && + currentAttr.entity_type !== undefined && + currentAttr.entity_type !== '' && + typeof currentAttr.entity_type === 'string' + ) { + attrEntityType = currentAttr.entity_type; + } + + //PRE POPULATE CONTEXT + jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; + + //determine Value + if (currentAttr.value !== undefined) { + //static attributes already have a value + hitted = true; + valueExpression = currentAttr.value; + } else if (plainMeasures[currentAttr.object_id] !== undefined) { + //we have got a meaure for that Attr + //actives ¿lazis? hitted = true; - valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); - //we fallback to null if anything unexpecte happend - if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = plainMeasures[currentAttr.object_id]; + } + //remove measures that has been shadowed by an alias (some may be left and managed later) + //Maybe we must filter object_id if there is name == object_id + measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); + + if ( + currentAttr.expression !== undefined && + currentAttr.expression !== '' && + typeof currentAttr.expression == 'string' + ) { + try { + hitted = true; + valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); + //we fallback to null if anything unexpecte happend + if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = null; + } + } catch (e) { valueExpression = null; } - } catch (e) { - valueExpression = null; + logger.debug( + context, + 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', + currentAttr.name, + currentAttr.expression, + jexlctxt, + valueExpression + ); } - logger.debug( - context, - 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', - currentAttr.name, - currentAttr.expression, - jexlctxt, - valueExpression - ); - } - currentAttr.hitted = hitted; - currentAttr.value = valueExpression; + currentAttr.hitted = hitted; + currentAttr.value = valueExpression; - //store de New Attributte in entity data structure - if (hitted === true) { - if (entities[attrEntityName] === undefined) { - entities[attrEntityName] = {}; - } - if (entities[attrEntityName][attrEntityType] === undefined) { - entities[attrEntityName][attrEntityType] = []; + //store de New Attributte in entity data structure + if (hitted === true) { + if (entities[attrEntityName] === undefined) { + entities[attrEntityName] = {}; + } + if (entities[attrEntityName][attrEntityType] === undefined) { + entities[attrEntityName][attrEntityType] = []; + } + //store de New Attributte + entities[attrEntityName][attrEntityType].push(currentAttr); } - //store de New Attributte - entities[attrEntityName][attrEntityType].push(currentAttr); - } - //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) - jexlctxt[currentAttr.name] = valueExpression; + //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) + jexlctxt[currentAttr.name] = valueExpression; - // Expand metadata value expression - if (currentAttr.metadata) { - for (var metaKey in currentAttr.metadata) { - if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { - let newAttrMeta = {}; - if (currentAttr.metadata[metaKey].type) { - newAttrMeta['type'] = currentAttr.metadata[metaKey].type; - } - let metaValueExpression; - try { - metaValueExpression = jexlParser.applyExpression( - currentAttr.metadata[metaKey].expression, - jexlctxt, - typeInformation - ); - //we fallback to null if anything unexpecte happend - if ( - metaValueExpression === null || - metaValueExpression === undefined || - Number.isNaN(metaValueExpression) - ) { + // Expand metadata value expression + if (currentAttr.metadata) { + for (var metaKey in currentAttr.metadata) { + if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { + let newAttrMeta = {}; + if (currentAttr.metadata[metaKey].type) { + newAttrMeta['type'] = currentAttr.metadata[metaKey].type; + } + let metaValueExpression; + try { + metaValueExpression = jexlParser.applyExpression( + currentAttr.metadata[metaKey].expression, + jexlctxt, + typeInformation + ); + //we fallback to null if anything unexpecte happend + if ( + metaValueExpression === null || + metaValueExpression === undefined || + Number.isNaN(metaValueExpression) + ) { + metaValueExpression = null; + } + } catch (e) { metaValueExpression = null; } - } catch (e) { - metaValueExpression = null; + newAttrMeta['value'] = metaValueExpression; + currentAttr.metadata[metaKey] = newAttrMeta; } - newAttrMeta['value'] = metaValueExpression; - currentAttr.metadata[metaKey] = newAttrMeta; } } } - } - //now we can compute explicit (Bool or Array) with the complete JexlContext - let explicit = false; - if (typeof typeInformation.explicitAttrs === 'string') { - try { - explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); - if (explicit instanceof Array && mustInsertTimeInstant) { - explicit.push(constants.TIMESTAMP_ATTRIBUTE); + //now we can compute explicit (Bool or Array) with the complete JexlContext + let explicit = false; + if (typeof typeInformation.explicitAttrs === 'string') { + try { + explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); + if (explicit instanceof Array && mustInsertTimeInstant) { + explicit.push(constants.TIMESTAMP_ATTRIBUTE); + } + logger.debug( + context, + 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', + typeInformation.explicitAttrs, + jexlctxt, + explicit + ); + } catch (e) { + // nothing to do: exception is already logged at info level } - logger.debug( - context, - 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', - typeInformation.explicitAttrs, - jexlctxt, - explicit - ); - } catch (e) { - // nothing to do: exception is already logged at info level + } else if (typeof typeInformation.explicitAttrs == 'boolean') { + explicit = typeInformation.explicitAttrs; } - } else if (typeof typeInformation.explicitAttrs == 'boolean') { - explicit = typeInformation.explicitAttrs; - } - - //more mesures may be added to the attribute list (unnhandled/left mesaures) l - if (explicit === false && Object.keys(measures).length > 0) { - entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); - } - - //PRE-PROCESSING FINISHED - //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - //Get ready to build and send NGSI payload (entities-->payload) - payload.actionType = 'append'; + //more mesures may be added to the attribute list (unnhandled/left mesaures) l + if (explicit === false && Object.keys(measures).length > 0) { + entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); + } - payload.entities = []; - const currentIsoDate = new Date().toISOString(); - const currentMoment = moment(currentIsoDate); - for (let ename in entities) { - for (let etype in entities[ename]) { - let e = {}; - e.id = String(ename); - e.type = String(etype); - let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. - let timestampAttrs = null; - if (mustInsertTimeInstant) { - // get timestamp for current entity - - timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); - if (timestampAttrs && timestampAttrs.length > 0) { - timestamp.value = timestampAttrs[0]['value']; - } + //PRE-PROCESSING FINISHED + //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload + + //Get ready to build and send NGSI payload (entities-->payload) + //payload.actionType = 'append'; + + //payload.entities = []; + //const currentIsoDate = new Date().toISOString(); + //const currentMoment = moment(currentIsoDate); + for (let ename in entities) { + for (let etype in entities[ename]) { + let e = {}; + e.id = String(ename); + e.type = String(etype); + let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. + let timestampAttrs = null; + if (mustInsertTimeInstant) { + // get timestamp for current entity - if (timestamp.value) { - if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { - callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); - return; + timestampAttrs = entities[ename][etype].filter( + (item) => item.name === constants.TIMESTAMP_ATTRIBUTE + ); + if (timestampAttrs && timestampAttrs.length > 0) { + timestamp.value = timestampAttrs[0]['value']; } - } else { - if (!typeInformation.timezone) { - timestamp.value = currentIsoDate; - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + + if (timestamp.value) { + if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { + callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); + return; + } } else { - timestamp.value = currentMoment - .tz(typeInformation.timezone) - .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + if (!typeInformation.timezone) { + timestamp.value = currentIsoDate; + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } else { + timestamp.value = currentMoment + .tz(typeInformation.timezone) + .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } } } - } - //extract attributes - let isEmpty = true; - for (let attr of entities[ename][etype]) { - if ( - attr.name !== 'id' && - attr.name !== 'type' && - (attr.value !== attr.skipValue || attr.skipValue === undefined) && - (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures - (typeof explicit === 'boolean' || //true and false already handled - (explicit instanceof Array && //check the array version - (explicit.includes(attr.name) || - explicit.some( - (item) => attr.object_id !== undefined && item.object_id === attr.object_id - )))) - ) { - isEmpty = false; - if (mustInsertTimeInstant) { - // Add TimeInstant to all attribute metadata of all entities - if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { - if (!attr.metadata) { - attr.metadata = {}; + //extract attributes + let isEmpty = true; + for (let attr of entities[ename][etype]) { + if ( + attr.name !== 'id' && + attr.name !== 'type' && + (attr.value !== attr.skipValue || attr.skipValue === undefined) && + (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures + (typeof explicit === 'boolean' || //true and false already handled + (explicit instanceof Array && //check the array version + (explicit.includes(attr.name) || + explicit.some( + (item) => attr.object_id !== undefined && item.object_id === attr.object_id + )))) + ) { + isEmpty = false; + if (mustInsertTimeInstant) { + // Add TimeInstant to all attribute metadata of all entities + if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { + if (!attr.metadata) { + attr.metadata = {}; + } + attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } - attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } + e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - } - if (!isEmpty) { - if (mustInsertTimeInstant) { - e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + if (!isEmpty) { + if (mustInsertTimeInstant) { + e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + } + payload.entities.push(e); } - payload.entities.push(e); } } - } + } // end for (let measures of originMeasures) let url = '/v2/op/update'; let options = NGSIUtils.createRequestObject(url, typeInformation, token); @@ -620,10 +639,14 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call // Note that the options object is prepared for the second case (multi entity), so we "patch" it // only in the first case - //Multientity more than one name o more than one type at primary entity - let multientity = Object.keys(entities).length > 1 || Object.keys(entities[entityName]).length > 1; + //Multi: multientity (more than one name o more than one type at primary entity) + // of multimeasure (originMeasures is an array of more than one element) + let multi = + Object.keys(entities).length > 1 || + Object.keys(entities[entityName]).length > 1 || + originMeasures.length > 1; - if (!multientity) { + if (!multi) { // recreate options object to use single entity update url = '/v2/entities?options=upsert'; options = NGSIUtils.createRequestObject(url, typeInformation, token); From ad5f514fd72e7bca15d0cba008da1a32031b181b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 13:07:51 +0200 Subject: [PATCH 02/23] add test --- test/functional/testCases.js | 66 ++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 9f88f75d1..3d9a887b9 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -513,6 +513,72 @@ const testCases = [ } ] }, + { + describeName: '0022 Simple group with active attributes and multimeasures', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'attr_a', + type: 'Number' + } + ], + static_attributes: [] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + loglevel: 'debug', + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', + type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + [ + { + a: 0 + } + ], + [ + { + a: 6 + } + ] + ] + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + attr_a: { + value: 6, + type: 'Number' + } + } + } + ] + }, // 0100 - JEXL TESTS { describeName: '0100 - Simple group with active attribute + JEXL expression boolean (!)', From ad1460bff65c47aa1217676edd6f074e365e6f57 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 16:00:20 +0200 Subject: [PATCH 03/23] update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb..0d4ce7bcd 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix: allow send multiple measures to CB in a batch (v2/op/update) instead of using multiples single request (iotagent-json#825) From 751c4e51a295d30046d093ce277a01b1287c2aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:09:38 +0200 Subject: [PATCH 04/23] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 059f36ea1..018d07f7a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (v2/op/update) instead of using multiples single request (iotagent-json#825) +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) \ No newline at end of file From ecf1e04743ef29258e40fef0603d24175fe02c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:10:10 +0200 Subject: [PATCH 05/23] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 018d07f7a..5d84eee8d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) \ No newline at end of file +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) \ No newline at end of file From ae17e28b0c5574c3c103e9ee01e7b045fc8ba4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:10:54 +0200 Subject: [PATCH 06/23] Update lib/services/ngsi/entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index d8514b6ac..5497f5a0e 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -299,7 +299,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token originMeasures = [originMeasures]; } for (let measures of originMeasures) { - entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture + entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entities data structure let jexlctxt = {}; //will store the whole context (not just for JEXL) let plainMeasures = null; //will contain measures POJO From 8c7ebcdacee9dcbb6b83847201b9d04899120fd0 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 7 May 2024 14:22:56 +0200 Subject: [PATCH 07/23] Update entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 5497f5a0e..3cb3b666a 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -545,12 +545,6 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - //Get ready to build and send NGSI payload (entities-->payload) - //payload.actionType = 'append'; - - //payload.entities = []; - //const currentIsoDate = new Date().toISOString(); - //const currentMoment = moment(currentIsoDate); for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From cfbe12fb428a494b37c4f1f5ceaeeac227cdafd0 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 7 May 2024 15:28:51 +0200 Subject: [PATCH 08/23] Update entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 3cb3b666a..8adc42b16 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -544,7 +544,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - + //Get ready to build and send NGSI payload (entities-->payload) for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From 3705bd68e31907d4a1c1822c5d3263c1b2772e12 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 11:36:28 +0200 Subject: [PATCH 09/23] fix and clone typeInformation --- lib/services/ngsi/entities-NGSI-v2.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 8adc42b16..e91e6e37e 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -260,7 +260,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal * @param {Object} typeInformation Configuration information for the device. * @param {String} token User token to identify against the PEP Proxies (optional). */ -function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token, callback) { +function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, token, callback) { //aux function used to builf JEXL context. //it returns a flat object from an Attr array function reduceAttrToPlainObject(attrs, initObj = {}) { @@ -273,11 +273,10 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token return initObj; } } - + //Make a clone and overwrite + let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); - //Make a clone and overwrite - typeInformation = JSON.parse(JSON.stringify(typeInformation)); //Check mandatory information: type if (!typeInformation || !typeInformation.type) { callback(new errors.TypeNotFound(null, entityName, typeInformation)); @@ -303,6 +302,8 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token let jexlctxt = {}; //will store the whole context (not just for JEXL) let plainMeasures = null; //will contain measures POJO + //Make a clone and overwrite + typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); //Rename all measures with matches with id and type to measure_id and measure_type for (let measure of measures) { @@ -545,6 +546,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload //Get ready to build and send NGSI payload (entities-->payload) + for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From 94db695ed0dc7248cb4e4e1572f8069a07dad88c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 11:45:20 +0200 Subject: [PATCH 10/23] Update multimeasure functional teste --- test/functional/testCases.js | 43 ++++++++++++--------- test/functional/testUtils.js | 73 ++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 37 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 3d9a887b9..a8dcd2f71 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -544,7 +544,7 @@ const testCases = [ }, should: [ { - loglevel: 'debug', + //loglevel: 'debug', shouldName: 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update @@ -556,25 +556,34 @@ const testCases = [ k: globalEnv.apikey }, json: [ - [ - { - a: 0 - } - ], - [ - { - a: 6 - } - ] + { + a: 0 + }, + { + a: 6 + } ] }, expectation: { - id: globalEnv.entity_name, - type: globalEnv.entity_type, - attr_a: { - value: 6, - type: 'Number' - } + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] } } ] diff --git a/test/functional/testUtils.js b/test/functional/testUtils.js index f41e8433a..495e8d470 100644 --- a/test/functional/testUtils.js +++ b/test/functional/testUtils.js @@ -98,28 +98,61 @@ function sendMeasureIotaLib(measure, provision) { * @param {Object} json * @returns {Array} measures */ -function jsonToIotaMeasures(json) { - let measures = []; - for (let key in json) { - /* eslint-disable-next-line no-prototype-builtins */ - if (json.hasOwnProperty(key)) { - let measure = { - name: key, - value: json[key] - }; - // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. - // When sending the data through iot - if (key === 'TimeInstant') { - measure.type = 'DateTime'; - } else { - // Although the type is not meaningfull and we could have picked any string for this, - // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories - measure.type = 'Text'; +function jsonToIotaMeasures(originJson) { + // FIXME: maybe this could be refactored to use less code + if (originJson && originJson[0]) { + // multimeasure case + let finalMeasures = []; + + for (let json of originJson) { + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); + } + } + finalMeasures.push(measures); + } + return finalMeasures; + } else { + let json = originJson; + + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); } - measures.push(measure); } + return measures; } - return measures; } /** @@ -170,7 +203,7 @@ async function testCase(measure, expectation, provision, env, config, type, tran let receivedContext = []; let cbMockRoute = ''; // Set the correct route depending if the test is multientity or not - if (type === 'multientity') { + if (type === 'multientity' || type === 'multimeasure') { cbMockRoute = '/v2/op/update'; } else { cbMockRoute = '/v2/entities?options=upsert'; From 328ee2ed65e671e55dac560021676134f0347603 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 12:00:04 +0200 Subject: [PATCH 11/23] update test --- test/functional/testCases.js | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index a8dcd2f71..a0a040b2a 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -559,6 +559,21 @@ const testCases = [ { a: 0 }, + { + a: 1 + }, + { + a: 2 + }, + { + a: 3 + }, + { + a: 4 + }, + { + a: 5 + }, { a: 6 } @@ -575,6 +590,46 @@ const testCases = [ id: globalEnv.entity_name, type: globalEnv.entity_type }, + { + attr_a: { + type: 'Number', + value: 1 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, { attr_a: { type: 'Number', From 9034c0254970e75bd0ec39d673a5d0c4f510c33b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 13 May 2024 13:16:54 +0200 Subject: [PATCH 12/23] clean old comment --- test/functional/testCases.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index a0a040b2a..6cf4d064e 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -544,10 +544,9 @@ const testCases = [ }, should: [ { - //loglevel: 'debug', shouldName: 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', - type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update + type: 'multimeasure', measure: { url: 'http://localhost:' + config.http.port + '/iot/json', method: 'POST', From 1bbd16897e22000fe7bed05af461452b2a7c2fbf Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 14 May 2024 11:23:37 +0200 Subject: [PATCH 13/23] avoid duplicate origintypeInformation --- lib/services/ngsi/entities-NGSI-v2.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index e91e6e37e..1870a997f 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -274,12 +274,11 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, } } //Make a clone and overwrite - let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); - let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); + let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(originTypeInformation); //Check mandatory information: type - if (!typeInformation || !typeInformation.type) { - callback(new errors.TypeNotFound(null, entityName, typeInformation)); + if (!originTypeInformation || !originTypeInformation.type) { + callback(new errors.TypeNotFound(null, entityName, originTypeInformation)); return; } @@ -291,7 +290,8 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, const currentIsoDate = new Date().toISOString(); const currentMoment = moment(currentIsoDate); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) - const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; + const mustInsertTimeInstant = + originTypeInformation.timestamp !== undefined ? originTypeInformation.timestamp : false; // Check if measures is a single measure or a array of measures (a multimeasure) if (originMeasures[0] && !originMeasures[0][0]) { @@ -620,7 +620,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, } // end for (let measures of originMeasures) let url = '/v2/op/update'; - let options = NGSIUtils.createRequestObject(url, typeInformation, token); + let options = NGSIUtils.createRequestObject(url, originTypeInformation, token); options.json = payload; // Prevent to update an entity with an empty payload: more than id and type @@ -645,7 +645,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, if (!multi) { // recreate options object to use single entity update url = '/v2/entities?options=upsert'; - options = NGSIUtils.createRequestObject(url, typeInformation, token); + options = NGSIUtils.createRequestObject(url, originTypeInformation, token); delete payload.actionType; let entityAttrs = payload.entities[0]; @@ -670,7 +670,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, request( options, - generateNGSI2OperationHandler('update', entityName, typeInformation, token, options, callback) + generateNGSI2OperationHandler('update', entityName, originTypeInformation, token, options, callback) ); } else { logger.debug( From 8d7fea1e2a003b2b7d648b7b1a794722edc3fb8f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 14 May 2024 11:34:52 +0200 Subject: [PATCH 14/23] declare typeInformation into loop --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 1870a997f..5330ca9d8 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -303,7 +303,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, let plainMeasures = null; //will contain measures POJO //Make a clone and overwrite - typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); + let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); //Rename all measures with matches with id and type to measure_id and measure_type for (let measure of measures) { From 53bc93a8b22507318122f1ce939ef1a67fcc7379 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 09:35:47 +0200 Subject: [PATCH 15/23] add tests about sorted multimeasures by TimeInstant --- test/functional/testCases.js | 132 +++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 6cf4d064e..cc7169c35 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -639,6 +639,138 @@ const testCases = [ } ] } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:25:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:30:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } } ] }, From b6a7d234fb7dc585744376c9828840548fc47acf Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:17:24 +0200 Subject: [PATCH 16/23] sort entities by TimeInstant --- lib/services/ngsi/entities-NGSI-v2.js | 17 ++- test/functional/testCases.js | 162 +++++++++++++++++++++++--- 2 files changed, 163 insertions(+), 16 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 5330ca9d8..bb98bf864 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -662,7 +662,22 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, transformedObject.type = entityAttrs.type; options.json = transformedObject; options.method = 'POST'; - } // else: keep current options object created for a batch update + } else { + // keep current options object created for a batch update + // but try sort entities by TimeInstant + if (payload.entities.every((entity) => 'TimeInstant' in entity)) { + payload.entities.sort( + (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() + ); + options.json = payload; + } else { + logger.debug( + context, + "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", + payload.entities + ); + } + } //Send the NGSI request logger.debug(context, 'Updating device value in the Context Broker at: %j', options.url); diff --git a/test/functional/testCases.js b/test/functional/testCases.js index cc7169c35..4efc88c6e 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -446,7 +446,7 @@ const testCases = [ ] }, { - describeName: '0021 Simple group with active attributes with metadata', + describeName: '0021 - Simple group with active attributes with metadata', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -514,7 +514,7 @@ const testCases = [ ] }, { - describeName: '0022 Simple group with active attributes and multimeasures', + describeName: '0022 - Simple group with active attributes and multimeasures', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -642,7 +642,7 @@ const testCases = [ }, { shouldName: - 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings', + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and order', type: 'multimeasure', measure: { url: 'http://localhost:' + config.http.port + '/iot/json', @@ -771,6 +771,138 @@ const testCases = [ } ] } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and sorted by TimeInstant', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:30:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:25:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } } ] }, @@ -2879,56 +3011,56 @@ const testCases = [ actionType: 'append', entities: [ { - id: globalEnv.entity_name, + id: 'TestType:TestDevice1', type: globalEnv.entity_type, - a: { + a1: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } } }, TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice1', + id: 'TestType:TestDevice2', type: globalEnv.entity_type, - a1: { + a2: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } } }, TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice2', + id: globalEnv.entity_name, type: globalEnv.entity_type, - a2: { + a: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } }, TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } From 8ec17ceb4fe5e548404f257df8340c5173d5c64d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:27:06 +0200 Subject: [PATCH 17/23] fix linter --- lib/services/ngsi/entities-NGSI-v2.js | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index bb98bf864..56c0a9ede 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -662,21 +662,19 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, transformedObject.type = entityAttrs.type; options.json = transformedObject; options.method = 'POST'; + } else if (payload.entities.every((entity) => 'TimeInstant' in entity)) { + // Try sort entities by TimeInstant + payload.entities.sort( + (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() + ); + options.json = payload; } else { - // keep current options object created for a batch update - // but try sort entities by TimeInstant - if (payload.entities.every((entity) => 'TimeInstant' in entity)) { - payload.entities.sort( - (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() - ); - options.json = payload; - } else { - logger.debug( - context, - "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", - payload.entities - ); - } + // keep current options object created for a batch update + logger.debug( + context, + "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", + payload.entities + ); } //Send the NGSI request From b5bae06f3d97f420d72cd30984dc6ba66fbb9fca Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:36:13 +0200 Subject: [PATCH 18/23] update CNR --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 5d84eee8d..f5501faee 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) \ No newline at end of file +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825) +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From d9d0c26fe2caed2e08136780e30204380273f2ad Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 17 May 2024 10:49:38 +0200 Subject: [PATCH 19/23] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f5501faee..05feb0873 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825) +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From a6f7daf74a00d88a8c4a189fbe608897e1377da5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 20 May 2024 11:02:46 +0200 Subject: [PATCH 20/23] update doc --- doc/api.md | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/doc/api.md b/doc/api.md index d661861c0..f3e285f75 100644 --- a/doc/api.md +++ b/doc/api.md @@ -32,6 +32,7 @@ - [Measurement transformation order](#measurement-transformation-order) - [Multientity measurement transformation support (`object_id`)](#multientity-measurement-transformation-support-object_id) - [Timestamp Processing](#timestamp-processing) + - [Multimeasure support](#multimeasure-support) - [Overriding global Context Broker host](#overriding-global-context-broker-host) - [Multitenancy, FIWARE Service and FIWARE ServicePath](#multitenancy-fiware-service-and-fiware-servicepath) - [Secured access to the Context Broker](#secured-access-to-the-context-broker) @@ -1044,6 +1045,127 @@ Some additional considerations to take into account: measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place). +## Multimeasure support + +A device could receive several measures at the same time. + +For example: + +```json +[ + { + "vol": 0 + }, + { + "vol": 1 + }, + { + "vol": 2 + } +] +``` + +In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + } + ] +} +``` + +Moreover if a multimeasure contains TimeInstant attribute, then CB update is sorted by attribute TimeInstant: + +For example: + +```json +[ + { + "vol": 0, + "TimeInstant": "2024-04-10T10:15:00Z" + }, + { + "vol": 1, + "TimeInstant": "2024-04-10T10:10:00Z" + }, + { + "vol": 2, + "TimeInstant": "2024-04-10T10:05:00Z" + } +] +``` + +In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 2 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:05:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:10:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:15:00Z" + } + } + ] +} +``` + ## Overriding global Context Broker host **cbHost**: Context Broker host URL. This option can be used to override the global CB configuration for specific types From 9e4ea596cd25ca90e0793b967e1aa34919eb6fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 May 2024 17:16:35 +0200 Subject: [PATCH 21/23] Apply suggestions from code review --- doc/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index f3e285f75..105654150 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1065,7 +1065,7 @@ For example: ] ``` -In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: ```json { From 15a82ebbd7996d6e30d213e41524c1754dfd7e11 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Mon, 20 May 2024 17:53:35 +0200 Subject: [PATCH 22/23] Modify multimeasures doc --- test/functional/README.md | 97 ++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/test/functional/README.md b/test/functional/README.md index 47d477be0..6020a6952 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -39,7 +39,7 @@ test cases are automatically generated. Each test case is defined as an object w or if the `transport` element is not defined. See the "Advanced features" section for more information. - `shouldName`: The name of the `IT` test case. This will be used to generate the test case name in the mocha test suite. - - `type`: The type of the test case. This can be `single` or `multientity`. See the "Advanced features" section + - `type`: The type of the test case. This can be `single`, `multimeasure` or `multientity`. See the "Advanced features" section for more information. - `measure`: The JSON object that will be sent to the IoTA JSON measure API. This will be used to send the measure. It contains the following elements: @@ -199,41 +199,11 @@ as a batch operation (see the following example). #### Multimeasures -It is also supported to test cases in which is sent more than one measure. To do so, you need to define the test case -`expectation` as an array, with one object for each measurement. Then, the suite will recognize the array length and will -expect the same number of NGSI requests. I.E: +It is also supported to test cases in which is sent more than one measure. To do so, you need to set add to the test case +the parameter `should.type` to the value `'multimeasure'`. -```js -[ - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 10, - type: 'Number' - }, - status: { - value: false, - type: 'Boolean' - } - }, - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 20, - type: 'Number' - }, - status: { - value: true, - type: 'Boolean' - } - } -]; -``` - -You also should define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of -objects. Each object will be a measure that will be sent to the Context Broker in a different request. I.E: +You must define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of +objects. I.E: ```javascript measure: { @@ -246,16 +216,69 @@ measure: { json: [ { s: false, - t: 10 + t: 21 }, { s: true, - t: 20 + t: 22 + }, + { + s: false, + t: 23 + } + ] +} +``` + +And you should define the test case `expectation` as an object, following a Context Broker batch operation. I.E: + +```js +expectation: { + actionType: 'append', + entities: [ + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 21 + }, + status: { + type: 'Boolean', + value: false + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 22 + }, + status: { + type: 'Boolean', + value: true + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 23 + }, + status: { + type: 'Boolean', + value: false + } } ] } ``` +Then, a batch request would be sent to the Context Broker containing the different measures. More information about +how the IoT Agent send multimeasures to the Context Broker [here](/doc/api.md#multimeasure-support). + #### Transport The test suite supports using the internal node lib function `iotAgentLib.update`, `HTTP` or `MQTT` for measure sending. From 6d8642bb0a3cd8104ab4a9d738ad3ef085bf8b7c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 21 May 2024 08:00:53 +0200 Subject: [PATCH 23/23] Update doc/api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- doc/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index 105654150..17500ba85 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1120,7 +1120,7 @@ For example: ] ``` -In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: ```json {