diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4c03a591..1fc89e083 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,11 @@ jobs: image: mongo:4.2 ports: - 27017:27017 + redis: + image: redis:6 + ports: + - 6379:6379 + strategy: matrix: node-version: @@ -76,6 +81,10 @@ jobs: image: mongo:4.2 ports: - 27017:27017 + redis: + image: redis:6 + ports: + - 6379:6379 steps: - name: Git checkout uses: actions/checkout@v2 diff --git a/README.md b/README.md index c4c3c0f7c..38578ae04 100644 --- a/README.md +++ b/README.md @@ -364,8 +364,8 @@ related with the fact that different people assign different interpretations on used in these licenses. Due to this, some people believe that there is a risk in just _using_ software under GPL or AGPL licenses (even without _modifying_ it). -For the avoidance of doubt, the owners of this software licensed under an AGPL-3.0 license -wish to make a clarifying public statement as follows: +For the avoidance of doubt, the owners of this software licensed under an AGPL-3.0 license wish to make a clarifying +public statement as follows: > Please note that software derived as a result of modifying the source code of this software in order to fix a bug or > incorporate enhancements is considered a derivative work of the product. Software that merely uses or aggregates (i.e. diff --git a/doc/advanced-topics.md b/doc/advanced-topics.md index 61869a13b..81d865648 100644 --- a/doc/advanced-topics.md +++ b/doc/advanced-topics.md @@ -1,37 +1,36 @@ ## Advanced Topics -* [Secured access to the Context Broker](#secured-access-to-the-context-broker) -* [GeoJSON support NGSI-LD only](#geojson-support-ngsi-ld-only) -* [Metadata support](#metadata-support) - * [NGSI LD data and metadata considerations](#ngsi-ld-data-and-metadata-considerations) -* [NGSI-LD Linked Data support](#ngsi-ld-linked-data-support) -* [Autoprovision configuration (autoprovision)](#autoprovision-configuration-autoprovision) -* [Explicitly defined attributes (explicitAttrs)](#explicitly-defined-attributes-explicitattrs) -* [Configuring operation to persist the data in Context Broker (appendMode)](#configuring-operation-to-persist-the-data-in-context-broker-appendmode) -* [Data mapping plugins](#data-mapping-plugins) - * [Development](#development) - * [Provided plugins](#provided-plugins) - * [Timestamp Compression plugin (compressTimestamp)](#timestamp-compression-plugin-compresstimestamp) - * [Attribute Alias plugin (attributeAlias)](#attribute-alias-plugin-attributealias) - * [Event plugin (addEvents)](#event-plugin-addevents) - * [Timestamp Processing Plugin (timestampProcess)](#timestamp-processing-plugin-timestampprocess) - * [Expression Translation plugin (expressionTransformation)](#expression-translation-plugin-expressiontransformation) - * [Multientity plugin (multiEntity)](#multientity-plugin-multientity) - * [Bidirectionality plugin (bidirectional)](#bidirectionality-plugin-bidirectional) - * [Autoprovision configuration (autoprovision)](#autoprovision-configuration-autoprovision) - * [Explicitly defined attributes (explicitAttrs)](#explicitly-defined-attributes-explicitattrs) - * [Configuring operation to persist the data in Context Broker (appendMode)](#configuring-operation-to-persist-the-data-in-context-broker-appendmode) -* [Old IoTAgent data migration](#old-iotagent-data-migration) - +- [Secured access to the Context Broker](#secured-access-to-the-context-broker) +- [GeoJSON support NGSI-LD only](#geojson-support-ngsi-ld-only) +- [Metadata support](#metadata-support) + - [NGSI LD data and metadata considerations](#ngsi-ld-data-and-metadata-considerations) +- [NGSI-LD Linked Data support](#ngsi-ld-linked-data-support) +- [Autoprovision configuration (autoprovision)](#autoprovision-configuration-autoprovision) +- [Explicitly defined attributes (explicitAttrs)](#explicitly-defined-attributes-explicitattrs) +- [Configuring operation to persist the data in Context Broker (appendMode)](#configuring-operation-to-persist-the-data-in-context-broker-appendmode) +- [Data mapping plugins](#data-mapping-plugins) + - [Development](#development) + - [Provided plugins](#provided-plugins) + - [Timestamp Compression plugin (compressTimestamp)](#timestamp-compression-plugin-compresstimestamp) + - [Attribute Alias plugin (attributeAlias)](#attribute-alias-plugin-attributealias) + - [Event plugin (addEvents)](#event-plugin-addevents) + - [Timestamp Processing Plugin (timestampProcess)](#timestamp-processing-plugin-timestampprocess) + - [Expression Translation plugin (expressionTransformation)](#expression-translation-plugin-expressiontransformation) + - [Multientity plugin (multiEntity)](#multientity-plugin-multientity) + - [Bidirectionality plugin (bidirectional)](#bidirectionality-plugin-bidirectional) + - [Autoprovision configuration (autoprovision)](#autoprovision-configuration-autoprovision) + - [Explicitly defined attributes (explicitAttrs)](#explicitly-defined-attributes-explicitattrs) + - [Configuring operation to persist the data in Context Broker (appendMode)](#configuring-operation-to-persist-the-data-in-context-broker-appendmode) +- [Old IoTAgent data migration](#old-iotagent-data-migration) ### Secured access to the Context Broker For access to instances of the Context Broker secured with a [PEP Proxy](https://github.com/telefonicaid/fiware-orion-pep), an authentication mechanism based in Keystone Trust -tokens is provided. A trust token is a way of Keystone to allow an user delegates a role to another user for a subservice. -It is a long-term token that can be issued by any user to give another user permissions -to impersonate him with a given role in a given project (subservice). Such impersonation itself is in turn based on a short-term -access token. +tokens is provided. A trust token is a way of Keystone to allow an user delegates a role to another user for a +subservice. It is a long-term token that can be issued by any user to give another user permissions to impersonate him +with a given role in a given project (subservice). Such impersonation itself is in turn based on a short-term access +token. For the authentication mechanisms to work, the `authentication` attribute in the configuration has to be fully configured, and the `authentication.enabled` subattribute should have the value `true`. @@ -73,8 +72,8 @@ Agent. Complete info on Keystone trust tokens could be found at: -- [Trusts concept](https://docs.openstack.org/keystone/stein/user/trusts) -- [Trusts API](https://docs.openstack.org/keystone/stein/api_curl_examples.html#post-v3-os-trust-trusts) +- [Trusts concept](https://docs.openstack.org/keystone/stein/user/trusts) +- [Trusts API](https://docs.openstack.org/keystone/stein/api_curl_examples.html#post-v3-os-trust-trusts) ### GeoJSON support NGSI-LD only @@ -269,26 +268,28 @@ updated as shown: ### Autoprovision configuration (autoprovision) -By default, when a measure arrives to the IoTAgent, if the `device_id` does not match with an existing one, then, the IoTA -creates a new device and a new entity according to the group config. Defining the field `autoprovision` to `false` -when provisioning the device group, the IoTA to reject the measure at the southbound, allowing only to persist the -data to devices that are already provisioned. It makes no sense to use this field in device provisioning since it is -intended to avoid provisioning devices (and for it to be effective, it would have to be provisional). +By default, when a measure arrives to the IoTAgent, if the `device_id` does not match with an existing one, then, the +IoTA creates a new device and a new entity according to the group config. Defining the field `autoprovision` to `false` +when provisioning the device group, the IoTA to reject the measure at the southbound, allowing only to persist the data +to devices that are already provisioned. It makes no sense to use this field in device provisioning since it is intended +to avoid provisioning devices (and for it to be effective, it would have to be provisional). ### Explicitly defined attributes (explicitAttrs) -If a given measure element (object_id) is not defined in the mappings of the device or group provision, the measure is stored -in the Context Broker by adding a new attribute to the entity with the same name of the undefined measure element. By adding the -field `explicitAttrs` with `true` value to device or group provision, the IoTAgent rejects the measure elements that are not defined -in the mappings of device or group provision, persisting only the one defined in the mappings of the provision. If `explicitAttrs` -is provided both at device and group level, the device level takes precedence. +If a given measure element (object_id) is not defined in the mappings of the device or group provision, the measure is +stored in the Context Broker by adding a new attribute to the entity with the same name of the undefined measure +element. By adding the field `explicitAttrs` with `true` value to device or group provision, the IoTAgent rejects the +measure elements that are not defined in the mappings of device or group provision, persisting only the one defined in +the mappings of the provision. If `explicitAttrs` is provided both at device and group level, the device level takes +precedence. ### Configuring operation to persist the data in Context Broker (appendMode) -This is a flag that can be enabled by activating the parameter `appendMode` in the configuration file or by using the `IOTA_APPEND_MODE` -environment variable (more info [here](https://github.com/telefonicaid/iotagent-node-lib/blob/master/doc/installationguide.md)). -If this flag is activated, the update requests to the Context Broker will be performed always with APPEND type, instead of the -default UPDATE. This have implications in the use of attributes with Context Providers, so this flag should be used with care. +This is a flag that can be enabled by activating the parameter `appendMode` in the configuration file or by using the +`IOTA_APPEND_MODE` environment variable (more info +[here](https://github.com/telefonicaid/iotagent-node-lib/blob/master/doc/installationguide.md)). If this flag is +activated, the update requests to the Context Broker will be performed always with APPEND type, instead of the default +UPDATE. This have implications in the use of attributes with Context Providers, so this flag should be used with care. ### Data mapping plugins @@ -373,8 +374,8 @@ events in the IoT Agent with the configured type name will be marked as events. ##### Timestamp Processing Plugin (timestampProcess) -This plugin processes the entity attributes looking for a `TimeInstant` attribute. If one is found, for NGSIv2, -the plugin adds a `TimeInstant` attribute as metadata for every other attribute in the same request. With NGSI-LD, the +This plugin processes the entity attributes looking for a `TimeInstant` attribute. If one is found, for NGSIv2, the +plugin adds a `TimeInstant` attribute as metadata for every other attribute in the same request. With NGSI-LD, the Standard `observedAt` property-of-a-property is used instead. ##### Expression Translation plugin (expressionTransformation) @@ -438,7 +439,8 @@ When a device is provisioned with bidirectional attributes, the IoTAgent subscri change notification for that attribute arrives to the IoTA, it applies the transformation defined in the device provisioning payload to the notification, and calls the underlying notification handler with the transformed entity. -The following `attributes` section shows an example of the plugin configuration (using IOTA_AUTOCAST=false to avoid translation from geo:point to geo:json) +The following `attributes` section shows an example of the plugin configuration (using IOTA_AUTOCAST=false to avoid +translation from geo:point to geo:json) ```json "attributes": [ diff --git a/doc/api.md b/doc/api.md index 2060487f5..5afb673f7 100644 --- a/doc/api.md +++ b/doc/api.md @@ -75,26 +75,26 @@ information configured: The table below shows the information held in the service group provisioning resource. The table also contains the correspondence between the API resource fields and the same fields in the database model. -| Payload Field | DB Field | Definition | -| ------------------------------ | ------------------------------ | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `service` | `service` | Service of the devices of this type | -| `subservice` | `subservice` | Subservice of the devices of this type. | -| `resource` | `resource` | string representing the Southbound resource that will be used to assign a type to a device (e.g.: pathname in the southbound port). | -| `apikey` | `apikey` | API Key string. | -| `timestamp` | `timestamp` | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true | -| `entity_type` | `entity_type` | name of the Entity `type` to assign to the group. | -| `trust` | `trust` | trust token to use for secured access to the Context Broker for this type of devices (optional; only needed for secured scenarios). | -| `cbHost` | `cbHost` | Context Broker connection information. This options can be used to override the global ones for specific types of devices. | -| `lazy` | `lazy` | list of common lazy attributes of the device. For each attribute, its `name` and `type` must be provided. | -| `commands` | `commands` | list of common commands attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional. | -| `attributes` | `attributes` | list of common active attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional. | -| `static_attributes` | `staticAttributes` | this attributes will be added to all the entities of this group 'as is', additional `metadata` is optional. | -| `internal_attributes` | `internalAttributes` | optional section with free format, to allow specific IoT Agents to store information along with the devices in the Device Registry. | -| `expressionLanguage` | `expresionLanguage` | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, `legacy` is used as default value. | -| `explicitAttrs` | `explicitAttrs` | optional boolean value, to support selective ignore of measures so that IOTA doesn’t progress. If not specified default is `false`. | -| `ngsiVersion` | `ngsiVersion` | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. Possible values are: `v2` or `ld`. The default is `v2`. When not running in mixed mode, this field is ignored. | -| `defaultEntityNameConjunction` | `defaultEntityNameConjunction` | optional string value to set default conjunction string used to compose a default `entity_name` when is not provided at device provisioning time. | -| `autoprovision` | `autoprovision` | optional boolean: If `false`, autoprovisioned devices (i.e. devices that are not created with an explicit provision operation but when the first measure arrives) are not allowed in this group. Default (in the case of omitting the field) is `true`. | +| Payload Field | DB Field | Definition | +| ------------------------------ | ------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `service` | `service` | Service of the devices of this type | +| `subservice` | `subservice` | Subservice of the devices of this type. | +| `resource` | `resource` | string representing the Southbound resource that will be used to assign a type to a device (e.g.: pathname in the southbound port). | +| `apikey` | `apikey` | API Key string. | +| `timestamp` | `timestamp` | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true | +| `entity_type` | `entity_type` | name of the Entity `type` to assign to the group. | +| `trust` | `trust` | trust token to use for secured access to the Context Broker for this type of devices (optional; only needed for secured scenarios). | +| `cbHost` | `cbHost` | Context Broker connection information. This options can be used to override the global ones for specific types of devices. | +| `lazy` | `lazy` | list of common lazy attributes of the device. For each attribute, its `name` and `type` must be provided. | +| `commands` | `commands` | list of common commands attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional. | +| `attributes` | `attributes` | list of common active attributes of the device. For each attribute, its `name` and `type` must be provided, additional `metadata` is optional. | +| `static_attributes` | `staticAttributes` | this attributes will be added to all the entities of this group 'as is', additional `metadata` is optional. | +| `internal_attributes` | `internalAttributes` | optional section with free format, to allow specific IoT Agents to store information along with the devices in the Device Registry. | +| `expressionLanguage` | `expresionLanguage` | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, `legacy` is used as default value. | +| `explicitAttrs` | `explicitAttrs` | optional boolean value, to support selective ignore of measures so that IOTA doesn’t progress. If not specified default is `false`. | +| `ngsiVersion` | `ngsiVersion` | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. Possible values are: `v2` or `ld`. The default is `v2`. When not running in mixed mode, this field is ignored. | +| `defaultEntityNameConjunction` | `defaultEntityNameConjunction` | optional string value to set default conjunction string used to compose a default `entity_name` when is not provided at device provisioning time. | +| `autoprovision` | `autoprovision` | optional boolean: If `false`, autoprovisioned devices (i.e. devices that are not created with an explicit provision operation but when the first measure arrives) are not allowed in this group. Default (in the case of omitting the field) is `true`. | ### Service Group Endpoint @@ -205,45 +205,45 @@ Note that there is a 1:1 correspondence between payload fields and DB fields (bu ### Relationship between service groups and devices -Devices may be associated to exisiting service groups (or not) based in `apiKey` matching or `type` matching (in the case `apiKey` -matching fails). For instance, let's consider a situation in which a service group has been provisioned with `type=X`/`apiKey=111` -and no other service group has been provisioned. +Devices may be associated to exisiting service groups (or not) based in `apiKey` matching or `type` matching (in the +case `apiKey` matching fails). For instance, let's consider a situation in which a service group has been provisioned +with `type=X`/`apiKey=111` and no other service group has been provisioned. -* IoT Agent receives an anonymous measure with `apiKey=111`. The matching `apiKey` means the entity inherits from service group. - Device entity has `type=X` and `apiKey=111` -* IoT Agent receives a provisioning request for an explicit device of `type=Y`/`apiKey=111`. The matching `apiKey` means the entity - inherits from service group but type is overridden. Device entity has `type=Y` and `apiKey=111` -* IoT Agent receives a provisioning request for an explicit device of `type=X`/`apiKey=222`. The matching `type` means the entity - inherits from service group but `apiKey` is overridden. Device entity has `type=X` and `apiKey=222`. -* IoT Agent receives a provisioning request for an explicit device of `type=Y`/`apiKey=222`. No matching. Device entity has `type=Y` - and `apiKey=222` and no service group. +- IoT Agent receives an anonymous measure with `apiKey=111`. The matching `apiKey` means the entity inherits from + service group. Device entity has `type=X` and `apiKey=111` +- IoT Agent receives a provisioning request for an explicit device of `type=Y`/`apiKey=111`. The matching `apiKey` + means the entity inherits from service group but type is overridden. Device entity has `type=Y` and `apiKey=111` +- IoT Agent receives a provisioning request for an explicit device of `type=X`/`apiKey=222`. The matching `type` means + the entity inherits from service group but `apiKey` is overridden. Device entity has `type=X` and `apiKey=222`. +- IoT Agent receives a provisioning request for an explicit device of `type=Y`/`apiKey=222`. No matching. Device + entity has `type=Y` and `apiKey=222` and no service group. ### Device model The table below shows the information held in the Device resource. The table also contains the correspondence between the API resource fields and the same fields in the database model. -| Payload Field | DB Field | Definition | Example of value | -| --------------------- | -------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------- | -| `device_id` | `id` | Device ID that will be used to identify the device. | UO834IO | -| `service` | `service` | Name of the service the device belongs to (will be used in the fiware-service header). | smartGondor | -| `service_path` | `subservice` | Name of the subservice the device belongs to (used in the fiware-servicepath header). | /gardens | -| `entity_name` | `name` | Name of the entity representing the device in the Context Broker | ParkLamplight12 | -| `entity_type` | `type` | Type of the entity in the Context Broker | Lamplights | -| `timezone` | `timezone` | Time zone of the sensor if it has any | America/Santiago | -| `timestamp` | `timestamp` | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true | -| `apikey` | `apikey` | Optional Apikey key string to use instead of group apikey | 9n4hb1vpwbjozzmw9f0flf9c2 | -| `endpoint` | `endpoint` | Endpoint where the device is going to receive commands, if any. | http://theDeviceUrl:1234/commands | -| `protocol` | `protocol` | Name of the device protocol, for its use with an IoT Manager. | IoTA-UL | -| `transport` | `transport` | Name of the device transport protocol, for the IoT Agents with multiple transport protocols. | MQTT | -| `attributes` | `active` | List of active attributes of the device | `[ { "name": "attr_name", "type": "Text" } ]` | -| `lazy` | `lazy` | List of lazy attributes of the device | `[ { "name": "attr_name", "type": "Text" } ]` | -| `commands` | `commands` | List of commands of the device | `[ { "name": "attr_name", "type": "Text" } ]` | -| `internal_attributes` | `internalAttributes` | List of internal attributes with free format for specific IoT Agent configuration | LWM2M mappings from object URIs to attributes | -| `static_attributes` | `staticAttributes` | List of static attributes to append to the entity. All the updateContext requests to the CB will have this set of attributes appended. | `[ { "name": "attr_name", "type": "Text" } ]` | -| `expressionLanguage` | `expresionLanguage` | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, legacy is used as default value. | -| `explicitAttrs` | `explicitAttrs` | Boolean value to support selective ignore of measures for device so that IOTA doesn’t progress. If not specified default is `false`. | `true/false` | -| `ngsiVersion` | `ngsiVersion` | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored. | `v2/ld` | +| Payload Field | DB Field | Definition | Example of value | +| --------------------- | -------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :-------------------------------------------- | +| `device_id` | `id` | Device ID that will be used to identify the device. | UO834IO | +| `service` | `service` | Name of the service the device belongs to (will be used in the fiware-service header). | smartGondor | +| `service_path` | `subservice` | Name of the subservice the device belongs to (used in the fiware-servicepath header). | /gardens | +| `entity_name` | `name` | Name of the entity representing the device in the Context Broker | ParkLamplight12 | +| `entity_type` | `type` | Type of the entity in the Context Broker | Lamplights | +| `timezone` | `timezone` | Time zone of the sensor if it has any | America/Santiago | +| `timestamp` | `timestamp` | Optional flag about whether or not to add the `TimeInstant` attribute to the device entity created, as well as a `TimeInstant` metadata to each attribute, with the current timestamp. With NGSI-LD, the Standard `observedAt` property-of-a-property is created instead. | true | +| `apikey` | `apikey` | Optional Apikey key string to use instead of group apikey | 9n4hb1vpwbjozzmw9f0flf9c2 | +| `endpoint` | `endpoint` | Endpoint where the device is going to receive commands, if any. | http://theDeviceUrl:1234/commands | +| `protocol` | `protocol` | Name of the device protocol, for its use with an IoT Manager. | IoTA-UL | +| `transport` | `transport` | Name of the device transport protocol, for the IoT Agents with multiple transport protocols. | MQTT | +| `attributes` | `active` | List of active attributes of the device | `[ { "name": "attr_name", "type": "Text" } ]` | +| `lazy` | `lazy` | List of lazy attributes of the device | `[ { "name": "attr_name", "type": "Text" } ]` | +| `commands` | `commands` | List of commands of the device | `[ { "name": "attr_name", "type": "Text" } ]` | +| `internal_attributes` | `internalAttributes` | List of internal attributes with free format for specific IoT Agent configuration | LWM2M mappings from object URIs to attributes | +| `static_attributes` | `staticAttributes` | List of static attributes to append to the entity. All the updateContext requests to the CB will have this set of attributes appended. | `[ { "name": "attr_name", "type": "Text" } ]` | +| `expressionLanguage` | `expresionLanguage` | optional boolean value, to set expression language used to compute expressions, possible values are: legacy or jexl. When not set or wrongly set, legacy is used as default value. | +| `explicitAttrs` | `explicitAttrs` | Boolean value to support selective ignore of measures for device so that IOTA doesn’t progress. If not specified default is `false`. | `true/false` | +| `ngsiVersion` | `ngsiVersion` | optional string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored. | `v2/ld` | #### Attribute lists diff --git a/doc/architecture.md b/doc/architecture.md index ae259e12e..ff68a2bd1 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -32,7 +32,7 @@ basis preprovisioning the devices). Device measures can have three different beh The following sequence diagram shows the different NGSI interactions an IoT Agent makes with the Context Broker, explained in the following subsections (using the example of a OMA Lightweight M2M device). -![General ](./img/ngsiInteractions.png "NGSI Interactions") +![General ](./img/ngsiInteractions.png 'NGSI Interactions') Be aware that the IoT Agents are only required to support NGSI10 operations `updateContext` and `queryContext` in their standard formats (currently in JSON format; XML deprecated) but will not answer to NGSI9 operations (or NGSI convenience @@ -207,16 +207,16 @@ the concrete IoT Agent implementations will be to map between the native device The following figure offers a graphical example of how a COAP IoT Agent work, ordered from the registration of the device to a command update to the device. -![General ](./img/iotAgentLib.png "Architecture Overview") +![General ](./img/iotAgentLib.png 'Architecture Overview') ### The `TimeInstant` element As part of the device to entity mapping process the IoT Agent creates and updates automatically a special timestamp. This timestamp is represented as two different properties of the mapped entity:: -- With NGSI-v2, an attribute metadata named `TimeInstant` (per dynamic attribute mapped, which captures as an - ISO8601 timestamp when the associated measurement (represented as attribute value) was observed. With NGSI-LD, the - Standard `observedAt` property-of-a-property is used instead. +- With NGSI-v2, an attribute metadata named `TimeInstant` (per dynamic attribute mapped, which captures as an ISO8601 + timestamp when the associated measurement (represented as attribute value) was observed. With NGSI-LD, the Standard + `observedAt` property-of-a-property is used instead. - For NGSI-v2 only, an additional attribute `TimeInstant` is added to the entity which captures as an ISO8601 timestamp when the last measurement received from the device was observed. diff --git a/doc/caching-strategy.md b/doc/caching-strategy.md new file mode 100644 index 000000000..41e6b93df --- /dev/null +++ b/doc/caching-strategy.md @@ -0,0 +1,78 @@ +# Configuration Data Caching Strategies + +The IoTAgent Library options to enforce several different data caching strategies to increase throughput. The actual +data caching strategy required in each use case will differ, and will always be a compromise between speed and data +latency. Several options are discussed below. + +## Memory Only + +if `deviceRegistry.type = memory` within the config, a transient memory-based device registry will be used to register +all the devices. This registry will be emptied whenever the process is restarted. Since all data is lost on exit there +is no concept of disaster recovery. **Memory only** should only be used for testing and in scenarios where all +provisioning can be added by a script on start-up. + +## MongoDB + +if `deviceRegistry.type = mongodb` within the config. a MongoDB database will be used to store all the device +information, so it will be persistent from one execution to the other. This offers a disaster recovery mechanism so that +when any IoT Agent goes down, data is not lost. Furthermore since the data is no longer held locally and is always +received from the database this allows multiple IoT Agent instances to access the same data in common. + +### MongoDB without Cache + +This is the default operation mode with Mongo-DB. Whenever a measure is received, provisioning data is requested from +the database. This may become a bottleneck in high availability systems. + +### MongoDB with Redis + +A [Redis](https://redis.io/) cache can be used to reduce database reads. if `redis.enabled = true` within the config, +the queries for the last few seconds of mongo db requests will be cached. The redis store is simpler than MongoDB +queries so access is swifter. Separate strategies can be defined for group and devices. These can read from separate +databases or even separate Redis instances. The `TTL` - time to live defines the latency of the system. + +```javascript +redis = { + deviceHost: "http://my-devices-redis", + devicePort: 6379, + deviceDB: 1, + deviceTTL: 600, + groupHost: "http://my-groups-redis", + groupPort: 6379, + groupDB: 0, + groupTTL: 600, +}; +``` + +When a single IoT Agent is used, the data is always consistent, however if multiple IoT Agent instances are used then it +is possible that a **IoT Agent 1** may update the provisioning data but **IoT Agent 2** may not read it until values +from its cache have expired. + +### MongoDB with memCache + +if `memCache.enabled = true` within the config this provides a transient memory-based cache in front of the mongo-DB +instance. It effectively combines the advantages of fast in-memory access with the reliability of a Mongo-DB database. + +```javascript +memCache = { + enabled: true, + deviceMax: 200, + deviceTTL: 60, + groupMax: 50, + groupTTL: 60, +}; +``` + +The memCache data is not shared across instances and therefore should be reserved to short term data storage. Multiple +IoT Agents would potential hold inconsistent provisioning data until the cache has expired. + +### MongoDB with multiple caches + +if both `memCache.enabled = true` and `redis.enabled = true` within the config, a strategy of multiple caches will be +used. In memory access should be the fastest and most volatile. The Redis cache should be used to relieve pressure on +MongoDB at a second caching level. + +## Bypassing cache + +In some cases consistent provisioning data is more vital than throughput. When creating or updating a provisioned device +or service group adding a `cache` attribute with the value `true` will ensure that the data can be cached, otherwise it is never placed into a +cache and therefore always consistently received from the Mongo-DB instance. diff --git a/doc/development.md b/doc/development.md index 794fc4379..252ecdb36 100644 --- a/doc/development.md +++ b/doc/development.md @@ -48,6 +48,9 @@ Module mocking during testing can be done with [proxyquire](https://github.com/t To run tests, type ```bash +docker run --name redis --publish 6379:6379 --detach redis:6 +docker run --name mongodb --publish 27017:27017 --detach mongo:4.2 + npm test ``` diff --git a/doc/getting-started.md b/doc/getting-started.md index 692073f72..20fe615be 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -10,22 +10,22 @@ custom settings may also be required dependent upon the actual IoT Agent used. ```javascript config = { - logLevel: "DEBUG", + logLevel: 'DEBUG', contextBroker: { - host: "orion", - port: "1026", + host: 'orion', + port: '1026' }, server: { port: 4041, - host: "0.0.0.0", + host: '0.0.0.0' }, deviceRegistry: { - type: "memory", + type: 'memory' }, - service: "openiot", - subservice: "/", - providerUrl: "http://iot-agent:4041", - defaultType: "Thing", + service: 'openiot', + subservice: '/', + providerUrl: 'http://iot-agent:4041', + defaultType: 'Thing' }; ``` diff --git a/doc/howto.md b/doc/howto.md index 13df1ccca..25e345f0e 100644 --- a/doc/howto.md +++ b/doc/howto.md @@ -75,12 +75,12 @@ folder of your project. Remember to change the Context Broker IP to your local C Now we can begin with the code of our IoT Agent. The very minimum code we need to start an IoT Agent is the following: ```javascript -var iotAgentLib = require("iotagent-node-lib"), - config = require("./config"); +var iotAgentLib = require('iotagent-node-lib'), + config = require('./config'); iotAgentLib.activate(config, function (error) { if (error) { - console.log("There was an error activating the IOTA"); + console.log('There was an error activating the IOTA'); process.exit(1); } }); @@ -112,10 +112,10 @@ In order to add the Express dependency to your project, add the following line t The require section would end up like this (the standard `http` module is also needed): ```javascript -var iotAgentLib = require("iotagent-node-lib"), - http = require("http"), - express = require("express"), - config = require("./config"); +var iotAgentLib = require('iotagent-node-lib'), + http = require('http'), + express = require('express'), + config = require('./config'); ``` And install the dependencies as usual with `npm install`. You will have to require both `express` and `http` in your @@ -129,16 +129,16 @@ function initSouthbound(callback) { southboundServer = { server: null, app: express(), - router: express.Router(), + router: express.Router() }; - southboundServer.app.set("port", 8080); - southboundServer.app.set("host", "0.0.0.0"); + southboundServer.app.set('port', 8080); + southboundServer.app.set('host', '0.0.0.0'); - southboundServer.router.get("/iot/d", manageULRequest); + southboundServer.router.get('/iot/d', manageULRequest); southboundServer.server = http.createServer(southboundServer.app); - southboundServer.app.use("/", southboundServer.router); - southboundServer.server.listen(southboundServer.app.get("port"), southboundServer.app.get("host"), callback); + southboundServer.app.use('/', southboundServer.router); + southboundServer.server.listen(southboundServer.app.get('port'), southboundServer.app.get('host'), callback); } ``` @@ -154,18 +154,18 @@ function manageULRequest(req, res, next) { iotAgentLib.retrieveDevice(req.query.i, req.query.k, function (error, device) { if (error) { res.status(404).send({ - message: "Couldn't find the device: " + JSON.stringify(error), + message: "Couldn't find the device: " + JSON.stringify(error) }); } else { values = parseUl(req.query.d, device); - iotAgentLib.update(device.name, device.type, "", values, device, function (error) { + iotAgentLib.update(device.name, device.type, '', values, device, function (error) { if (error) { res.status(500).send({ - message: "Error updating the device", + message: 'Error updating the device' }); } else { res.status(200).send({ - message: "Device successfully updated", + message: 'Device successfully updated' }); } }); @@ -190,17 +190,17 @@ function parseUl(data, device) { } function createAttribute(element) { - var pair = element.split("|"), + var pair = element.split('|'), attribute = { name: pair[0], value: pair[1], - type: findType(pair[0]), + type: findType(pair[0]) }; return attribute; } - return data.split(",").map(createAttribute); + return data.split(',').map(createAttribute); } ``` @@ -227,14 +227,14 @@ show the modifications in the `activate()` function: ```javascript iotAgentLib.activate(config, function (error) { if (error) { - console.log("There was an error activating the IOTA"); + console.log('There was an error activating the IOTA'); process.exit(1); } else { initSouthbound(function (error) { if (error) { - console.log("Could not initialize South bound API due to the following error: %s", error); + console.log('Could not initialize South bound API due to the following error: %s', error); } else { - console.log("Both APIs started successfully"); + console.log('Both APIs started successfully'); } }); } @@ -286,7 +286,7 @@ A HTTP request library will be needed in order to make those calls. To this exte used. In order to do so, add the following require statement to the initialization code: ```javascript -request = require("request"); +request = require('request'); ``` and add the `request` dependency to the `package.json` file: @@ -303,11 +303,11 @@ and add the `request` dependency to the `package.json` file: The require section should now look like this: ```javascript -var iotAgentLib = require("iotagent-node-lib"), - http = require("http"), - express = require("express"), - request = require("request"), - config = require("./config"); +var iotAgentLib = require('iotagent-node-lib'), + http = require('http'), + express = require('express'), + request = require('request'), + config = require('./config'); ``` ### Implementation @@ -321,11 +321,11 @@ for the context provisioning requests. At this point, we should provide two hand ```javascript function queryContextHandler(id, type, service, subservice, attributes, callback) { var options = { - url: "http://127.0.0.1:9999/iot/d", - method: "GET", + url: 'http://127.0.0.1:9999/iot/d', + method: 'GET', qs: { - q: attributes.join(), - }, + q: attributes.join() + } }; request(options, function (error, response, body) { @@ -350,21 +350,21 @@ attributes). Here is the code for the `createResponse()` function: ```javascript function createResponse(id, type, attributes, body) { - var values = body.split(","), + var values = body.split(','), responses = []; for (var i = 0; i < attributes.length; i++) { responses.push({ name: attributes[i], - type: "string", - value: values[i], + type: 'string', + value: values[i] }); } return { id: id, type: type, - attributes: responses, + attributes: responses }; } ``` @@ -374,11 +374,11 @@ function createResponse(id, type, attributes, body) { ```javascript function updateContextHandler(id, type, service, subservice, attributes, callback) { var options = { - url: "http://127.0.0.1:9999/iot/d", - method: "GET", + url: 'http://127.0.0.1:9999/iot/d', + method: 'GET', qs: { - d: createQueryFromAttributes(attributes), - }, + d: createQueryFromAttributes(attributes) + } }; request(options, function (error, response, body) { @@ -388,31 +388,31 @@ function updateContextHandler(id, type, service, subservice, attributes, callbac callback(null, { id: id, type: type, - attributes: attributes, + attributes: attributes }); } }); } ``` -The updateContext handler deals with the modification requests that arrive at the North Port of the IoT Agent via `/v2/op/update`. It is -invoked once for each entity requested (note that a single request can contain multiple entity updates), with the same -parameters used in the queryContext handler. The only difference is the value of the attributes array, now containing a -list of attribute objects, each containing name, type and value. The handler must also make use of the callback to -return a list of updated attributes. +The updateContext handler deals with the modification requests that arrive at the North Port of the IoT Agent via +`/v2/op/update`. It is invoked once for each entity requested (note that a single request can contain multiple entity +updates), with the same parameters used in the queryContext handler. The only difference is the value of the attributes +array, now containing a list of attribute objects, each containing name, type and value. The handler must also make use +of the callback to return a list of updated attributes. For this handler we have used a helper function called `createQueryFromAttributes()`, that transforms the NGSI representation of the attributes to the UL type expected by the device: ```javascript function createQueryFromAttributes(attributes) { - var query = ""; + var query = ''; for (var i in attributes) { - query += attributes[i].name + "|" + attributes[i].value; + query += attributes[i].name + '|' + attributes[i].value; if (i != attributes.length - 1) { - query += ","; + query += ','; } } @@ -555,9 +555,9 @@ variable and afterward the value of the multiCore in the `config.js` file. The r (the standard `http` module is also needed): ```javascript -var iotAgent = require("../lib/iotagent-implementation"), - iotAgentLib = require("iotagent-node-lib"), - config = require("./config"); +var iotAgent = require('../lib/iotagent-implementation'), + iotAgentLib = require('iotagent-node-lib'), + config = require('./config'); ``` It is important to mention the purpose of the `iotAgent` variable. It is the proper implementation of the IoT Agent @@ -573,9 +573,9 @@ about starting the IoTAgent: ```javascript iotAgentLib.startServer(config, iotAgent, function (error) { if (error) { - console.log(context, "Error starting IoT Agent: [%s] Exiting process", error); + console.log(context, 'Error starting IoT Agent: [%s] Exiting process', error); } else { - console.log(context, "IoT Agent started"); + console.log(context, 'IoT Agent started'); } }); ``` @@ -602,7 +602,7 @@ handlers themselves. Here we can see the definition of the configuration handler ```javascript function configurationHandler(configuration, callback) { - console.log("\n\n* REGISTERING A NEW CONFIGURATION:\n%s\n\n", JSON.stringify(configuration, null, 4)); + console.log('\n\n* REGISTERING A NEW CONFIGURATION:\n%s\n\n', JSON.stringify(configuration, null, 4)); callback(null, configuration); } ``` @@ -619,8 +619,8 @@ feature, let's use the provisioning handler to change the value of the type of t ```javascript function provisioningHandler(device, callback) { - console.log("\n\n* REGISTERING A NEW DEVICE:\n%s\n\n", JSON.stringify(device, null, 4)); - device.type = "CertifiedType"; + console.log('\n\n* REGISTERING A NEW DEVICE:\n%s\n\n', JSON.stringify(device, null, 4)); + device.type = 'CertifiedType'; callback(null, device); } ``` diff --git a/doc/installationguide.md b/doc/installationguide.md index 931def7ae..69fbe486e 100644 --- a/doc/installationguide.md +++ b/doc/installationguide.md @@ -56,9 +56,10 @@ allowing the computer to interpret the rest of the data with more clarity and de } ``` -Under mixed mode, **NGSI v2** payloads are used for context broker communications by default, but this payload may also be switched -to **NGSI LD** at service group or device provisioning time using the `ngsiVersion` field in the provisioning API. -The `ngsiVersion` field switch may be added at either group or device level, with the device level overriding the group setting. +Under mixed mode, **NGSI v2** payloads are used for context broker communications by default, but this payload may also +be switched to **NGSI LD** at service group or device provisioning time using the `ngsiVersion` field in the +provisioning API. The `ngsiVersion` field switch may be added at either group or device level, with the device level +overriding the group setting. - **server**: configuration used to create the Context Server (port where the IoT Agent will be listening as a Context Provider and base root to prefix all the paths). The `port` attribute is required. If no `baseRoot` attribute is @@ -159,7 +160,7 @@ used for the same purpose. For instance: ```javascript { - type: "mongodb"; + type: 'mongodb'; } ``` @@ -203,6 +204,40 @@ used for the same purpose. For instance: } ``` +- **memCache**: Whether to use a memory cache in front of Mongo-DB when using the `mongodb` **deviceRegistry** option + to reduce I/O. This memory cache will hold and serve a set of recently requested groups and devices (up to a given + maximum time-to-live) and return the cached response so long as the value is still within `TTL`. When enabled the + default values are to hold up to 200 devices and 160 groups in memory and retain values for up to 60 seconds. + +```javascript +{ + enabled: true, + deviceMax: 200, + deviceTTL: 60, + groupMax: 50, + groupTTL: 60 +} +``` + +- **redis**: Whether to use a redis cache in front of Mongo-DB when using the `mongodb` **deviceRegistry** option to + reduce I/O. This redis cache will hold and serve a set of recently requested groups and devices (up to a given + maximum time-to-live) and return the cached response so long as the value is still within `TTL`. When enabled the + default values are to retain values for up to 600 seconds. Separate settings are available for devices and service + groups. + +```javascript +{ + deviceHost: "localhost", + devicePort: 6379, + deviceDB: 1, + deviceTTL: 600, + groupHost: "localhost", + groupPort: 6379, + groupDB: 0, + groupTTL: 600 +} +``` + - **iotManager**: configures all the information needed to register the IoT Agent in the IoTManager. If this section is present, the IoTA will try to register to a IoTAM in the `host`, `port` and `path` indicated, with the information configured in the object. The IoTAgent URL that will be reported will be the `providedUrl` (described @@ -241,8 +276,8 @@ used for the same purpose. For instance: - **singleConfigurationMode**: enables the Single Configuration mode for backwards compatibility (see description in the Overview). Default to false. - **timestamp**: if this flag is activated: - - For NGSI-v2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from - device information. This flag is overwritten by `timestamp` flag in group or device + - For NGSI-v2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from device + information. This flag is overwritten by `timestamp` flag in group or device - With NGSI-LD, the standard `observedAt` property-of-a-property is created instead. - **defaultResource**: default string to use as resource for the registration of new Configurations (if no resource is provided). @@ -275,8 +310,8 @@ used for the same purpose. For instance: - **explicitAttrs**: if this flag is activated, only provisioned attributes will be processed to Context Broker. This flag is overwritten by `explicitAttrs` flag in group or device provision. - **defaultEntityNameConjunction**: the default conjunction string used to compose a default `entity_name` when is not - provided at device provisioning time; in that case `entity_name` is composed by `type` + `:` + `device_id`. - Default value is `:`. This value is overwritten by `defaultEntityNameConjunction` in group provision. + provided at device provisioning time; in that case `entity_name` is composed by `type` + `:` + `device_id`. Default + value is `:`. This value is overwritten by `defaultEntityNameConjunction` in group provision. - **relaxTemplateValidation**: if this flag is activated, `objectId` attributes for incoming devices are not validated, and may exceptionally include characters (such as semi-colons) which are [forbidden](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) according to the @@ -291,61 +326,75 @@ with container-based technologies, like Docker, Heroku, etc... The following table shows the accepted environment variables, as well as the configuration parameter the variable overrides. -| Environment variable | Configuration attribute | -| :------------------------------- | :------------------------------ | -| IOTA_CB_URL | `contextBroker.url` | -| IOTA_CB_HOST | `contextBroker.host` | -| IOTA_CB_PORT | `contextBroker.port` | -| IOTA_CB_NGSI_VERSION | `contextBroker.ngsiVersion` | -| IOTA_NORTH_HOST | `server.host` | -| IOTA_NORTH_PORT | `server.port` | -| IOTA_PROVIDER_URL | `providerUrl` | -| IOTA_AUTH_ENABLED | `authentication.enabled` | -| IOTA_AUTH_TYPE | `authentication.type` | -| IOTA_AUTH_HEADER | `authentication.header` | -| IOTA_AUTH_URL | `authentication.url` | -| IOTA_AUTH_HOST | `authentication.host` | -| IOTA_AUTH_PORT | `authentication.port` | -| IOTA_AUTH_USER | `authentication.user` | -| IOTA_AUTH_PASSWORD | `authentication.password` | -| IOTA_AUTH_CLIENT_ID | `authentication.clientId` | -| IOTA_AUTH_CLIENT_SECRET | `authentication.clientSecret` | -| IOTA_AUTH_TOKEN_PATH | `authentication.tokenPath` | -| IOTA_AUTH_PERMANENT_TOKEN | `authentication.permanentToken` | -| IOTA_REGISTRY_TYPE | `deviceRegistry.type` | -| IOTA_LOG_LEVEL | `logLevel` | -| IOTA_TIMESTAMP | `timestamp` | -| IOTA_IOTAM_URL | `iotManager.url` | -| IOTA_IOTAM_HOST | `iotManager.host` | -| IOTA_IOTAM_PORT | `iotManager.port` | -| IOTA_IOTAM_PATH | `iotManager.path` | -| IOTA_IOTAM_AGENTPATH | `iotManager.agentPath` | -| IOTA_IOTAM_PROTOCOL | `iotManager.protocol` | -| IOTA_IOTAM_DESCRIPTION | `iotManager.description` | -| IOTA_MONGO_HOST | `mongodb.host` | -| IOTA_MONGO_PORT | `mongodb.port` | -| IOTA_MONGO_DB | `mongodb.db` | -| IOTA_MONGO_REPLICASET | `mongodb.replicaSet` | -| IOTA_MONGO_USER | `mongodb.user` | -| IOTA_MONGO_PASSWORD | `mongodb.password` | -| IOTA_MONGO_AUTH_SOURCE | `mongodb.authSource` | -| IOTA_MONGO_RETRIES | `mongodb.retries` | -| IOTA_MONGO_RETRY_TIME | `mongodb.retryTime` | -| IOTA_MONGO_SSL | `mongodb.ssl` | -| IOTA_MONGO_EXTRAARGS | `mongodb.extraArgs` | -| IOTA_SINGLE_MODE | `singleConfigurationMode` | -| IOTA_APPEND_MODE | `appendMode` | -| IOTA_POLLING_EXPIRATION | `pollingExpiration` | -| IOTA_POLLING_DAEMON_FREQ | `pollingDaemonFrequency` | -| IOTA_AUTOCAST | `autocast` | -| IOTA_MULTI_CORE | `multiCore` | -| IOTA_JSON_LD_CONTEXT | `jsonLdContext` | -| IOTA_FALLBACK_TENANT | `fallbackTenant` | -| IOTA_FALLBACK_PATH | `fallbackPath` | -| IOTA_DEFAULT_EXPRESSION_LANGUAGE | `defaultExpressionLanguage` | -| IOTA_EXPLICIT_ATTRS | `explicitAttrs` | -| IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction` | -| IOTA_RELAX_TEMPLATE_VALIDATION | `relaxTemplateValidation` | +| Environment variable | Configuration attribute | +| :----------------------------------- | :------------------------------ | +| IOTA_CB_URL | `contextBroker.url` | +| IOTA_CB_HOST | `contextBroker.host` | +| IOTA_CB_PORT | `contextBroker.port` | +| IOTA_CB_NGSI_VERSION | `contextBroker.ngsiVersion` | +| IOTA_NORTH_HOST | `server.host` | +| IOTA_NORTH_PORT | `server.port` | +| IOTA_PROVIDER_URL | `providerUrl` | +| IOTA_AUTH_ENABLED | `authentication.enabled` | +| IOTA_AUTH_TYPE | `authentication.type` | +| IOTA_AUTH_HEADER | `authentication.header` | +| IOTA_AUTH_URL | `authentication.url` | +| IOTA_AUTH_HOST | `authentication.host` | +| IOTA_AUTH_PORT | `authentication.port` | +| IOTA_AUTH_USER | `authentication.user` | +| IOTA_AUTH_PASSWORD | `authentication.password` | +| IOTA_AUTH_CLIENT_ID | `authentication.clientId` | +| IOTA_AUTH_CLIENT_SECRET | `authentication.clientSecret` | +| IOTA_AUTH_TOKEN_PATH | `authentication.tokenPath` | +| IOTA_AUTH_PERMANENT_TOKEN | `authentication.permanentToken` | +| IOTA_REGISTRY_TYPE | `deviceRegistry.type` | +| IOTA_MEMCACHE_ENABLED | `memCache.enabled` | +| IOTA_MEMCACHE_DEVICE_MAX | `memCache.deviceMax` | +| IOTA_MEMCACHE_DEVICE_TTL | `memCache.deviceTTL` | +| IOTA_MEMCACHE_GROUP_MAX | `memCache.groupMax` | +| IOTA_MEMCACHE_GROUP_TTL | `memCache.groupTTL` | +| IOTA_REDIS_ENABLED | `redis.enabled` | +| IOTA_REDIS_DEVICE_HOST | `redis.deviceHost` | +| IOTA_REDIS_DEVICE_PORT | `redis.devicePort` | +| IOTA_REDIS_DEVICE_DB | `redis.deviceDB` | +| IOTA_REDIS_DEVICE_TTL | `redis.deviceTTL` | +| IOTA_REDIS_GROUP_HOST | `redis.groupHost` | +| IOTA_REDIS_GROUP_PORT | `redis.groupPort` | +| IOTA_REDIS_GROUP_DB | `redis.groupDB` | +| IOTA_REDIS_GROUP_TTL | `redis.groupTTL` | +| IOTA_LOG_LEVEL | `logLevel` | +| IOTA_TIMESTAMP | `timestamp` | +| IOTA_IOTAM_URL | `iotManager.url` | +| IOTA_IOTAM_HOST | `iotManager.host` | +| IOTA_IOTAM_PORT | `iotManager.port` | +| IOTA_IOTAM_PATH | `iotManager.path` | +| IOTA_IOTAM_AGENTPATH | `iotManager.agentPath` | +| IOTA_IOTAM_PROTOCOL | `iotManager.protocol` | +| IOTA_IOTAM_DESCRIPTION | `iotManager.description` | +| IOTA_MONGO_HOST | `mongodb.host` | +| IOTA_MONGO_PORT | `mongodb.port` | +| IOTA_MONGO_DB | `mongodb.db` | +| IOTA_MONGO_REPLICASET | `mongodb.replicaSet` | +| IOTA_MONGO_USER | `mongodb.user` | +| IOTA_MONGO_PASSWORD | `mongodb.password` | +| IOTA_MONGO_AUTH_SOURCE | `mongodb.authSource` | +| IOTA_MONGO_RETRIES | `mongodb.retries` | +| IOTA_MONGO_RETRY_TIME | `mongodb.retryTime` | +| IOTA_MONGO_SSL | `mongodb.ssl` | +| IOTA_MONGO_EXTRAARGS | `mongodb.extraArgs` | +| IOTA_SINGLE_MODE | `singleConfigurationMode` | +| IOTA_APPEND_MODE | `appendMode` | +| IOTA_POLLING_EXPIRATION | `pollingExpiration` | +| IOTA_POLLING_DAEMON_FREQ | `pollingDaemonFrequency` | +| IOTA_AUTOCAST | `autocast` | +| IOTA_MULTI_CORE | `multiCore` | +| IOTA_JSON_LD_CONTEXT | `jsonLdContext` | +| IOTA_FALLBACK_TENANT | `fallbackTenant` | +| IOTA_FALLBACK_PATH | `fallbackPath` | +| IOTA_DEFAULT_EXPRESSION_LANGUAGE | `defaultExpressionLanguage` | +| IOTA_EXPLICIT_ATTRS | `explicitAttrs` | +| IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction` | +| IOTA_RELAX_TEMPLATE_VALIDATION | `relaxTemplateValidation` | Note: diff --git a/lib/commonConfig.js b/lib/commonConfig.js index 5dc23c76f..2dc42955e 100644 --- a/lib/commonConfig.js +++ b/lib/commonConfig.js @@ -129,6 +129,22 @@ function processEnvironmentVariables() { 'IOTA_AUTH_TOKEN_PATH', 'IOTA_AUTH_PERMANENT_TOKEN', 'IOTA_REGISTRY_TYPE', + 'IOTA_MEMCACHE_ENABLED', + 'IOTA_MEMCACHE_DEVICE_MAX', + 'IOTA_MEMCACHE_DEVICE_TTL', + 'IOTA_MEMCACHE_GROUP_MAX', + 'IOTA_MEMCACHE_GROUP_TTL', + 'IOTA_REDIS_ENABLED', + 'IOTA_REDIS_DEVICE_HOST', + 'IOTA_REDIS_DEVICE_PORT', + 'IOTA_REDIS_DEVICE_PASSWORD', + 'IOTA_REDIS_DEVICE_DB', + 'IOTA_REDIS_DEVICE_TTL', + 'IOTA_REDIS_GROUP_HOST', + 'IOTA_REDIS_GROUP_PORT', + 'IOTA_REDIS_GROUP_PASSWORD', + 'IOTA_REDIS_GROUP_DB', + 'IOTA_REDIS_GROUP_TTL', 'IOTA_LOG_LEVEL', 'IOTA_TIMESTAMP', 'IOTA_IOTAM_HOST', @@ -188,7 +204,9 @@ function processEnvironmentVariables() { 'IOTA_AUTH_CLIENT_ID', 'IOTA_AUTH_CLIENT_SECRET', 'IOTA_MONGO_USER', - 'IOTA_MONGO_PASSWORD' + 'IOTA_MONGO_PASSWORD', + 'IOTA_REDIS_DEVICE_PASSWORD', + 'IOTA_REDIS_GROUP_PASSWORD' ]; // Substitute Docker Secret Variables where set. @@ -243,9 +261,9 @@ function processEnvironmentVariables() { config.contextBroker.jsonLdContext = process.env.IOTA_JSON_LD_CONTEXT.split(',').map((ctx) => ctx.trim()); } - if (Array.isArray(config.contextBroker.jsonLdContext) && config.contextBroker.jsonLdContext.length === 1){ + if (Array.isArray(config.contextBroker.jsonLdContext) && config.contextBroker.jsonLdContext.length === 1) { config.contextBroker.jsonLdContext = config.contextBroker.jsonLdContext[0]; - } + } config.contextBroker.fallbackTenant = process.env.IOTA_FALLBACK_TENANT || config.contextBroker.service || 'iotagent'; @@ -323,6 +341,81 @@ function processEnvironmentVariables() { config.deviceRegistry.type = process.env.IOTA_REGISTRY_TYPE; } + // In Memory Caching policy + config.memCache = config.memCache || {}; + if (process.env.IOTA_MEMCACHE_ENABLED) { + config.memCache.enabled = process.env.IOTA_MEMCACHE_ENABLED === 'true'; + } + const memCache = config.memCache; + if (memCache.enabled) { + memCache.deviceMax = memCache.deviceMax || 200; + memCache.deviceTTL = memCache.deviceTTL || 60; + memCache.groupMax = memCache.groupMax || 50; + memCache.groupTTL = memCache.groupTTL || 60; + + if (process.env.IOTA_MEMCACHE_DEVICE_MAX) { + memCache.deviceMax = process.env.IOTA_MEMCACHE_DEVICE_MAX; + } + if (process.env.IOTA_MEMCACHE_DEVICE_TTL) { + memCache.deviceTTL = process.env.IOTA_MEMCACHE_DEVICE_TTL; + } + if (process.env.IOTA_MEMCACHE_GROUP_MAX) { + memCache.groupMax = process.env.IOTA_MEMCACHE_GROUP_MAX; + } + if (process.env.IOTA_MEMCACHE_GROUP_TTL) { + memCache.groupTTL = process.env.IOTA_MEMCACHE_GROUP_TTL; + } + } + + // Redis Caching policy + config.redis = config.redis || {}; + if (process.env.IOTA_REDIS_ENABLED) { + config.redis.enabled = process.env.IOTA_REDIS_ENABLED === 'true'; + } + const redis = config.redis; + if (redis.enabled) { + redis.deviceHost = redis.deviceHost || 'localhost'; + redis.devicePort = redis.devicePort || 6379; + redis.deviceDB = redis.deviceDB || 1; + redis.deviceTTL = redis.deviceTTL || 600; + redis.groupHost = redis.groupHost || 'localhost'; + redis.groupPort = redis.groupPort || 6379; + redis.groupDB = redis.groupDB || 0; + redis.groupTTL = redis.groupTTL || 600; + + if (process.env.IOTA_REDIS_DEVICE_HOST) { + config.redis.deviceHost = process.env.IOTA_REDIS_DEVICE_HOST; + } + if (process.env.IOTA_REDIS_DEVICE_PORT) { + config.redis.devicePort = process.env.IOTA_REDIS_DEVICE_PORT; + } + if (process.env.IOTA_REDIS_DEVICE_PASSWORD) { + config.redis.devicePassword = process.env.IOTA_REDIS_DEVICE_PASSWORD; + } + if (process.env.IOTA_REDIS_DEVICE_DB) { + config.redis.deviceDB = process.env.IOTA_REDIS_DEVICE_DB; + } + if (process.env.IOTA_REDIS_DEVICE_TTL) { + config.redis.deviceTTL = process.env.IOTA_REDIS_DEVICE_TTL; + } + + if (process.env.IOTA_REDIS_GROUP_HOST) { + config.redis.groupHost = process.env.IOTA_REDIS_GROUP_HOST; + } + if (process.env.IOTA_REDIS_GROUP_PORT) { + config.redis.groupPort = process.env.IOTA_REDIS_GROUP_PORT; + } + if (process.env.IOTA_REDIS_GROUP_PASSWORD) { + config.redis.groupPassword = process.env.IOTA_REDIS_GROUP_PASSWORD; + } + if (process.env.IOTA_REDIS_GROUP_DB) { + config.redis.groupDB = process.env.IOTA_REDIS_GROUP_DB; + } + if (process.env.IOTA_REDIS_GROUP_TTL) { + config.redis.groupTTL = process.env.IOTA_REDIS_GROUP_TTL; + } + } + // Log Level configuration if (process.env.IOTA_LOG_LEVEL) { config.logLevel = process.env.IOTA_LOG_LEVEL; @@ -470,7 +563,9 @@ function processEnvironmentVariables() { if (process.env.IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION) { config.defaultEntityNameConjunction = process.env.IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION; } else { - config.defaultEntityNameConjunction = config.defaultEntityNameConjunction ? config.defaultEntityNameConjunction : ':'; + config.defaultEntityNameConjunction = config.defaultEntityNameConjunction + ? config.defaultEntityNameConjunction + : ':'; } } @@ -525,7 +620,6 @@ function ngsiVersion() { return 'unknown'; } - /** * It checks if a combination of typeInformation or common Config is LD * diff --git a/lib/fiware-iotagent-lib.js b/lib/fiware-iotagent-lib.js index 9567ee59e..f46002bcd 100644 --- a/lib/fiware-iotagent-lib.js +++ b/lib/fiware-iotagent-lib.js @@ -156,6 +156,9 @@ function doActivate(newConfig, callback) { registry = require('./services/devices/deviceRegistryMongoDB'); groupRegistry = require('./services/groups/groupRegistryMongoDB'); commandRegistry = require('./services/commands/commandRegistryMongoDB'); + + registry.initialiseCaches(newConfig); + groupRegistry.initialiseCaches(newConfig); } else { logger.info(context, 'Falling back to Transient Memory registry for NGSI Library'); diff --git a/lib/model/Device.js b/lib/model/Device.js index 48aa6fc35..c8f6db84a 100644 --- a/lib/model/Device.js +++ b/lib/model/Device.js @@ -50,7 +50,8 @@ const Device = new Schema({ autoprovision: Boolean, expressionLanguage: String, explicitAttrs: Boolean, - ngsiVersion: String + ngsiVersion: String, + cache: Boolean }); function load(db) { diff --git a/lib/model/Group.js b/lib/model/Group.js index 2e081f83a..10b48c0bd 100644 --- a/lib/model/Group.js +++ b/lib/model/Group.js @@ -45,7 +45,8 @@ const Group = new Schema({ expressionLanguage: String, explicitAttrs: Boolean, defaultEntityNameConjunction: String, - ngsiVersion: String + ngsiVersion: String, + cache: Boolean }); function load(db) { diff --git a/lib/services/common/iotManagerService.js b/lib/services/common/iotManagerService.js index 7d8e9ef24..618ccd193 100644 --- a/lib/services/common/iotManagerService.js +++ b/lib/services/common/iotManagerService.js @@ -60,7 +60,8 @@ function register(callback) { explicitAttrs: service.explicitAttrs, expressionLanguage: service.expressionLanguage, defaultEntityNameConjunction: service.defaultEntityNameConjunction, - ngsiVersion: service.ngsiVersion + ngsiVersion: service.ngsiVersion, + cache: service.cache }; } diff --git a/lib/services/devices/deviceRegistryMongoDB.js b/lib/services/devices/deviceRegistryMongoDB.js index 76fe1e08e..374cc5f29 100644 --- a/lib/services/devices/deviceRegistryMongoDB.js +++ b/lib/services/devices/deviceRegistryMongoDB.js @@ -30,6 +30,14 @@ const errors = require('../../errors'); const constants = require('../../constants'); const Device = require('../../model/Device'); const async = require('async'); +const cacheManager = require('cache-manager'); +const redisStore = require('cache-manager-ioredis'); + +let redisCache; +let redisClient; +let memoryCache; +let cache; + let context = { op: 'IoTAgentNGSI.MongoDBDeviceRegister' }; @@ -56,8 +64,69 @@ const attributeList = [ 'timestamp', 'explicitAttrs', 'expressionLanguage', - 'ngsiVersion' + 'ngsiVersion', + 'cache' ]; +/** + * Sets up the in-memory and Redis caches for devices, should one be required. + */ +function initialiseCaches(config) { + function isCacheableValue(value) { + if (value !== null && value !== false && value !== undefined) { + return value.cache; + } + return false; + } + function initialiseRedis(redis, isCacheableValue) { + const redisCache = cacheManager.caching({ + store: redisStore, + host: redis.deviceRedisHost, + port: redis.deviceRedisPort, + password: redis.deviceRedisPassword, + db: redis.deviceRedisDB, + ttl: redis.deviceRedisTTL, + isCacheableValue + }); + // listen for redis connection error event + redisClient = redisCache.store.getClient(); + redisClient.on('error', (error) => { + logger.error(context, 'Redis for Devices: ' + error); + }); + redisClient.on('connect', () => { + logger.info(context, 'Successfully connected to Redis for Devices'); + }); + return redisCache; + } + + function initialiseMemCache(memCache, isCacheableValue) { + return cacheManager.caching({ + store: 'memory', + max: memCache.deviceMax, + ttl: memCache.deviceTTL, + isCacheableValue + }); + } + + redisCache = config.redis.enabled ? initialiseRedis(config.redis, isCacheableValue) : undefined; + memoryCache = config.memCache.enabled ? initialiseMemCache(config.memCache, isCacheableValue) : undefined; + + if (redisCache) { + cache = memoryCache ? cacheManager.multiCaching([memoryCache, redisCache], { isCacheableValue }) : redisCache; + } else if (memoryCache) { + cache = memoryCache; + } else { + cache = undefined; + } +} + +/** + * Empties the memory cache + */ +function clearCache() { + if (cache) { + cache.reset(); + } +} /** * Generates a handler for the save device operations. The handler will take the customary error and the saved device @@ -136,6 +205,7 @@ function removeDevice(id, service, subservice, callback) { callback(new errors.InternalDbError(error)); } else { logger.debug(context, 'Device [%s] successfully removed.', id); + clearCache(); callback(null); } @@ -223,7 +293,20 @@ function getDeviceById(id, service, subservice, callback) { }; context = fillService(context, queryParams); logger.debug(context, 'Looking for device with id [%s].', id); - findOneInMongoDB(queryParams, id, callback); + + if (cache) { + cache.wrap( + JSON.stringify(queryParams), + (cacheCallback) => { + findOneInMongoDB(queryParams, id, cacheCallback); + }, + (error, data) => { + callback(error, data); + } + ); + } else { + findOneInMongoDB(queryParams, id, callback); + } } /** @@ -282,6 +365,7 @@ function update(device, callback) { if (error) { callback(error); } else { + clearCache(); data.lazy = device.lazy; data.active = device.active; data.internalId = device.internalId; @@ -295,6 +379,7 @@ function update(device, callback) { data.registrationId = device.registrationId; data.explicitAttrs = device.explicitAttrs; data.ngsiVersion = device.ngsiVersion; + data.cache = device.cache; /* eslint-disable-next-line new-cap */ const deviceObj = new Device.model(data); @@ -361,3 +446,4 @@ exports.get = alarmsInt(constants.MONGO_ALARM, getDevice); exports.getSilently = getDevice; exports.getByName = alarmsInt(constants.MONGO_ALARM, getByName); exports.clear = alarmsInt(constants.MONGO_ALARM, clear); +exports.initialiseCaches = initialiseCaches; diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js index 78a1ba758..5f03c7ed9 100644 --- a/lib/services/devices/deviceService.js +++ b/lib/services/devices/deviceService.js @@ -214,7 +214,13 @@ function findConfigurationGroup(deviceObj, callback) { } else { config .getGroupRegistry() - .findTypeSilently(deviceObj.service, deviceObj.subservice, deviceObj.type, deviceObj.apikey, handlerGroupFindByType); + .findTypeSilently( + deviceObj.service, + deviceObj.subservice, + deviceObj.type, + deviceObj.apikey, + handlerGroupFindByType + ); } } @@ -581,7 +587,12 @@ function findOrCreate(deviceId, group, callback) { callback(error, device, group); }); } else { - logger.info(context, 'Device %j not provisioned due autoprovision is disabled by its conf %j', newDevice, group); + logger.info( + context, + 'Device %j not provisioned due autoprovision is disabled by its conf %j', + newDevice, + group + ); callback(new errors.DeviceNotFound(deviceId)); } } else { diff --git a/lib/services/groups/groupRegistryMongoDB.js b/lib/services/groups/groupRegistryMongoDB.js index 8ac17decb..747cddce2 100644 --- a/lib/services/groups/groupRegistryMongoDB.js +++ b/lib/services/groups/groupRegistryMongoDB.js @@ -32,6 +32,13 @@ const constants = require('../../constants'); const errors = require('../../errors'); const Group = require('../../model/Group'); const async = require('async'); +const cacheManager = require('cache-manager'); +const redisStore = require('cache-manager-ioredis'); + +let redisCache; +let redisClient; +let memoryCache; +let cache; let context = { op: 'IoTAgentNGSI.MongoDBGroupRegister' }; @@ -57,8 +64,68 @@ const attributeList = [ 'explicitAttrs', 'expressionLanguage', 'defaultEntityNameConjunction', - 'ngsiVersion' + 'ngsiVersion', + 'cache' ]; +/** + * Sets up the in-memory and Redis caches for service groups, should one be required. + */ +function initialiseCaches(config) { + function isCacheableValue(value) { + if (value !== null && value !== false && value !== undefined) { + return value.cache; + } + return false; + } + function initialiseRedis(redis, isCacheableValue) { + const redisCache = cacheManager.caching({ + store: redisStore, + host: redis.groupHost, + port: redis.groupPort, + password: redis.groupPassword, + db: redis.groupDB, + ttl: redis.groupTTL, + isCacheableValue + }); + redisClient = redisCache.store.getClient(); + redisClient.on('error', (error) => { + logger.error(context, 'Redis for Service Groups : ' + error); + }); + redisClient.on('connect', () => { + logger.info(context, 'Successfully connected to Redis for Service Groups'); + }); + return redisCache; + } + + function initialiseMemCache(memCache, isCacheableValue) { + return cacheManager.caching({ + store: 'memory', + max: memCache.groupMax, + ttl: memCache.groupTTL, + isCacheableValue + }); + } + + redisCache = config.redis.enabled ? initialiseRedis(config.redis, isCacheableValue) : undefined; + memoryCache = config.memCache.enabled ? initialiseMemCache(config.memCache, isCacheableValue) : undefined; + + if (redisCache) { + cache = memoryCache ? cacheManager.multiCaching([memoryCache, redisCache], { isCacheableValue }) : redisCache; + } else if (memoryCache) { + cache = memoryCache; + } else { + cache = undefined; + } +} + +/** + * Empties the memory cache + */ +function clearCache() { + if (cache) { + cache.reset(); + } +} /** * Generates a handler for the save device group operations. The handler will take the customary error and the saved @@ -247,7 +314,19 @@ function findBy(fields) { context = fillService(context, { service: 'n/a', subservice: 'n/a' }); logger.debug(context, 'Looking for group params %j with queryObj %j', fields, queryObj); - findOneInMongoDB(queryObj, fields, callback); + if (cache) { + cache.wrap( + JSON.stringify(queryObj), + (cacheCallback) => { + findOneInMongoDB(queryObj, fields, cacheCallback); + }, + (error, data) => { + callback(error, data); + } + ); + } else { + findOneInMongoDB(queryObj, fields, callback); + } }; } @@ -262,6 +341,7 @@ function update(id, body, callback) { group[key] = body[key]; } }); + clearCache(); /* eslint-disable-next-line new-cap */ const groupObj = new Group.model(group); groupObj.isNew = false; @@ -284,6 +364,7 @@ function remove(id, callback) { callback(new errors.InternalDbError(error)); } else { logger.debug(context, 'Device [%s] successfully removed.', id); + clearCache(); callback(null, deviceGroup); } }); @@ -317,3 +398,4 @@ exports.getTypeSilently = intoTrans(context, findBy(['type'])); exports.update = alarmsInt(constants.MONGO_ALARM, intoTrans(context, update)); exports.remove = alarmsInt(constants.MONGO_ALARM, intoTrans(context, remove)); exports.clear = alarmsInt(constants.MONGO_ALARM, intoTrans(context, clear)); +exports.initialiseCaches = initialiseCaches; diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js index 651897d8f..eee1f4cb4 100644 --- a/lib/services/northBound/deviceProvisioningServer.js +++ b/lib/services/northBound/deviceProvisioningServer.js @@ -61,7 +61,8 @@ const provisioningAPITranslation = { autoprovision: 'autoprovision', explicitAttrs: 'explicitAttrs', expressionLanguage: 'expressionLanguage', - ngsiVersion: 'ngsiVersion' + ngsiVersion: 'ngsiVersion', + cache: 'cache' }; /** @@ -140,7 +141,8 @@ function handleProvision(req, res, next) { autoprovision: body.autoprovision, explicitAttrs: body.explicitAttrs, expressionLanguage: body.expressionLanguage, - ngsiVersion: body.ngsiVersion + ngsiVersion: body.ngsiVersion, + cache: body.cache }); } @@ -213,7 +215,8 @@ function toProvisioningAPIFormat(device) { autoprovision: device.autoprovision, expressionLanguage: device.expressionLanguage, explicitAttrs: device.explicitAttrs, - ngsiVersion: device.ngsiVersion + ngsiVersion: device.ngsiVersion, + cache: device.cache }; } diff --git a/lib/templates/createDevice.json b/lib/templates/createDevice.json index 26fe72e72..b822bf968 100644 --- a/lib/templates/createDevice.json +++ b/lib/templates/createDevice.json @@ -48,6 +48,10 @@ "description": "NGSI Interface for this device", "type": "string" }, + "cache": { + "description": "Flag to whether to cache the details of this device in memory", + "type": "boolean" + }, "lazy": { "description": "list of lazy attributes of the devices", "type": "array", diff --git a/lib/templates/createDeviceLax.json b/lib/templates/createDeviceLax.json index 64d25efda..2158a9144 100644 --- a/lib/templates/createDeviceLax.json +++ b/lib/templates/createDeviceLax.json @@ -48,6 +48,10 @@ "description": "NGSI Interface for this device", "type": "string" }, + "cache": { + "description": "Flag to whether to cache the details of this device in memory", + "type": "boolean" + }, "lazy": { "description": "list of lazy attributes of the devices", "type": "array", @@ -121,7 +125,7 @@ "properties": { "object_id": { "description": "ID of the attribute in the device", - "type": "string" + "type": "string" }, "type": { "description": "Type of the attribute in the target entity", @@ -181,4 +185,4 @@ } } } -} \ No newline at end of file +} diff --git a/lib/templates/deviceGroup.json b/lib/templates/deviceGroup.json index fcf63dc11..73209d4f6 100644 --- a/lib/templates/deviceGroup.json +++ b/lib/templates/deviceGroup.json @@ -45,6 +45,10 @@ "description": "NGSI Interface for this group of devices", "type": "string" }, + "cache": { + "description": "Flag to whether to cache the details of this group in memory", + "type": "boolean" + }, "attributes": { "description": "list of active attributes of the devices", "type": "array" @@ -81,4 +85,4 @@ } } } -} \ No newline at end of file +} diff --git a/lib/templates/updateDevice.json b/lib/templates/updateDevice.json index a8d360a00..1dc1ac45a 100644 --- a/lib/templates/updateDevice.json +++ b/lib/templates/updateDevice.json @@ -188,6 +188,10 @@ "ngsiVersion": { "description": "NGSI Interface for this device", "type": "string" + }, + "cache": { + "description": "Flag to whether to cache the detals of this device in memory", + "type": "boolean" } } } diff --git a/mkdocs.yml b/mkdocs.yml index 94795c5a5..13e033dcf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,10 +18,11 @@ pages: - 'Advanced Topics' : 'advanced-topics.md' - 'Library Functions': 'usermanual.md' - 'Measurement Transformation Expression Language': 'expressionLanguage.md' + - 'Caching Strategies': 'caching-strategy.md' - 'How to develop a new IoT Agent': 'howto.md' - 'North Port - NGSI Interactions': 'northboundinteractions.md' - 'Development Documentation': development.md - - 'Installation & Administration Manual': + - 'Installation & Administration Manual': - 'Installation Guide': 'installationguide.md' - 'Operations (logs & alarms)': 'operations.md' - + diff --git a/package.json b/package.json index badaafed5..fcfc3efd1 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ "dependencies": { "async": "2.6.2", "body-parser": "~1.19.0", + "cache-manager": "~3.4.0", + "cache-manager-ioredis": "~2.1.0", "command-shell-lib": "1.0.0", "express": "~4.16.4", "jexl": "2.1.1", @@ -68,6 +70,7 @@ "eslint-config-tamia": "~7.2.5", "eslint-plugin-prettier": "~3.1.4", "husky": "~4.2.5", + "ioredis": "~4.19.2", "lint-staged": "~10.2.11", "prettier": "~2.0.5", "mocha": "8.0.1", diff --git a/test/unit/mongodb/mongoDBUtils.js b/test/tools/mongoDBUtils.js similarity index 100% rename from test/unit/mongodb/mongoDBUtils.js rename to test/tools/mongoDBUtils.js diff --git a/test/tools/redisUtils.js b/test/tools/redisUtils.js new file mode 100644 index 000000000..545f5fc52 --- /dev/null +++ b/test/tools/redisUtils.js @@ -0,0 +1,53 @@ +/* + * Copyright 2014 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-iotagent-lib + * + * fiware-iotagent-lib is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * fiware-iotagent-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with fiware-iotagent-lib. + * If not, see http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License + * please contact with::[contacto@tid.es] + */ + +/* eslint-disable no-unused-vars */ + +const Redis = require('ioredis'); +const async = require('async'); + +function cleanDb(host, port, db, callback) { + const redis = new Redis({ + port, + host, + db + }); + + redis.flushdb(function (err, succeeded) { + callback(err); + }); +} + +function cleanDbs(host, port, callback) { + const redis = new Redis({ + port, + host + }); + + redis.flushall(function (err, succeeded) { + callback(err); + }); +} + +exports.cleanDb = cleanDb; +exports.cleanDbs = cleanDbs; diff --git a/test/unit/general/deviceService-test.js b/test/unit/general/deviceService-test.js index fa72b423c..503983d6a 100644 --- a/test/unit/general/deviceService-test.js +++ b/test/unit/general/deviceService-test.js @@ -187,14 +187,12 @@ describe('NGSI-v2 - Device Service: utils', function () { .post('/v2/registrations') .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' }); - contextBrokerMock .matchHeader('fiware-service', 'testservice') .matchHeader('fiware-servicepath', '/testingPath') .post('/v2/entities?options=upsert') .reply(204); - async.series([request.bind(request, groupCreation), request.bind(request, deviceCreation)], function ( error, results @@ -222,11 +220,10 @@ describe('NGSI-v2 - Device Service: utils', function () { .post('/v2/registrations') .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' }); - contextBrokerMock .matchHeader('fiware-service', 'testservice') .matchHeader('fiware-servicepath', '/testingPath') - .post('/v2/entities?options=upsert') + .post('/v2/entities?options=upsert') .reply(204); async.series([request.bind(request, groupCreation)], function (error, results) { diff --git a/test/unit/general/migration-test.js b/test/unit/general/migration-test.js index 03e04e57b..a53de0a7a 100644 --- a/test/unit/general/migration-test.js +++ b/test/unit/general/migration-test.js @@ -25,7 +25,7 @@ const migration = require('../../../lib/command/migration'); const mongo = require('mongodb').MongoClient; -const mongoUtils = require('../mongodb/mongoDBUtils'); +const mongoUtils = require('../../tools/mongoDBUtils'); const utils = require('../../tools/utils'); const logger = require('logops'); const async = require('async'); diff --git a/test/unit/general/statistics-persistence_test.js b/test/unit/general/statistics-persistence_test.js index f9ca24dfe..89565bf2b 100644 --- a/test/unit/general/statistics-persistence_test.js +++ b/test/unit/general/statistics-persistence_test.js @@ -28,7 +28,7 @@ const commonConfig = require('../../../lib/commonConfig'); const iotAgentLib = require('../../../lib/fiware-iotagent-lib'); const should = require('should'); const async = require('async'); -const mongoUtils = require('../mongodb/mongoDBUtils'); +const mongoUtils = require('../../tools/mongoDBUtils'); const iotAgentConfig = { logLevel: 'FATAL', contextBroker: { diff --git a/test/unit/mongodb/mongodb-cache-test.js b/test/unit/mongodb/mongodb-cache-test.js new file mode 100644 index 000000000..929e74589 --- /dev/null +++ b/test/unit/mongodb/mongodb-cache-test.js @@ -0,0 +1,403 @@ +/* + * Copyright 2016 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of fiware-iotagent-lib + * + * fiware-iotagent-lib is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * fiware-iotagent-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with fiware-iotagent-lib. + * If not, seehttp://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License + * please contact with::[contacto@tid.es] + * + * Modified by: Daniel Calvo - ATOS Research & Innovation + */ + +/* eslint-disable no-unused-vars */ + +const iotAgentLib = require('../../../lib/fiware-iotagent-lib'); +const utils = require('../../tools/utils'); +const should = require('should'); +const logger = require('logops'); +const nock = require('nock'); +const mongoUtils = require('../../tools/mongoDBUtils'); +const redisUtils = require('../../tools/redisUtils'); +const request = require('request'); +let contextBrokerMock; +let statusAttributeMock; +const iotAgentConfig = { + contextBroker: { + host: '192.168.1.1', + port: '1026', + ngsiVersion: 'v2' + }, + server: { + port: 4041 + }, + types: { + Light: { + commands: [], + lazy: [ + { + name: 'temperature', + type: 'centigrades' + } + ], + active: [ + { + name: 'pressure', + type: 'Hgmm' + } + ] + }, + Termometer: { + commands: [], + lazy: [ + { + name: 'temp', + type: 'kelvin' + } + ], + active: [] + }, + Motion: { + commands: [], + lazy: [ + { + name: 'moving', + type: 'Boolean' + } + ], + staticAttributes: [ + { + name: 'location', + type: 'Vector', + value: '(123,523)' + } + ], + active: [] + }, + Robot: { + commands: [ + { + name: 'position', + type: 'Array' + } + ], + lazy: [], + staticAttributes: [], + active: [] + } + }, + deviceRegistry: { + type: 'mongodb' + }, + + memCache: { + enabled: true, + deviceSize: 1000, + deviceTTL: 10, + groupSize: 100, + groupTTL: 10 + }, + + redis: { + enabled: true, + deviceDB: 0 + }, + + mongodb: { + host: 'localhost', + port: '27017', + db: 'iotagent' + }, + service: 'smartgondor', + subservice: 'gardens', + providerUrl: 'http://smartgondor.com', + pollingExpiration: 200, + pollingDaemonFrequency: 20 +}; +const device3 = { + id: 'cachedDevice', + type: 'Robot', + service: 'smartgondor', + subservice: 'gardens', + polling: true, + cache: true +}; + +describe('Mongo-DB Redis cache ', function () { + beforeEach(function (done) { + logger.setLevel('FATAL'); + + nock.cleanAll(); + + contextBrokerMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/registrations') + .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' }); + + contextBrokerMock + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .post('/v2/entities?options=upsert') + .reply(204); + + iotAgentLib.activate(iotAgentConfig, done); + }); + + afterEach(function (done) { + delete device3.registrationId; + iotAgentLib.clearAll(function () { + iotAgentLib.deactivate(function () { + mongoUtils.cleanDbs(function () { + redisUtils.cleanDbs('localhost', 6379, function () { + nock.cleanAll(); + iotAgentLib.setDataUpdateHandler(); + iotAgentLib.setCommandHandler(); + done(); + }); + }); + }); + }); + }); + + describe('When caching is enabled and a command update arrives to the IoT Agent for a device with polling', function () { + const options = { + url: 'http://localhost:' + iotAgentConfig.server.port + '/v2/op/update', + method: 'POST', + json: { + actionType: 'update', + entities: [ + { + id: 'Robot:cachedDevice', + type: 'Robot', + position: { + type: 'Array', + value: '[28, -104, 23]' + } + } + ] + }, + headers: { + 'fiware-service': 'smartgondor', + 'fiware-servicepath': 'gardens' + } + }; + + beforeEach(function (done) { + statusAttributeMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .patch( + '/v2/entities/Robot:cachedDevice/attrs?type=Robot', + utils.readExampleFile('./test/unit/ngsiv2/examples/contextRequests/updateContextCommandStatus.json') + ) + .reply(204); + + iotAgentLib.register(device3, function (error) { + done(); + }); + }); + + it('should not call the client handler', function (done) { + let handlerCalled = false; + + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + handlerCalled = true; + callback(null, { + id, + type, + attributes: [ + { + name: 'position', + type: 'Array', + value: '[28, -104, 23]' + } + ] + }); + }); + + request(options, function (error, response, body) { + should.not.exist(error); + handlerCalled.should.equal(false); + done(); + }); + }); + it('should create the attribute with the "_status" prefix in the Context Broker', function (done) { + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + callback(null); + }); + + request(options, function (error, response, body) { + should.not.exist(error); + statusAttributeMock.done(); + done(); + }); + }); + it('should store the commands in the queue', function (done) { + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + callback(null); + }); + + request(options, function (error, response, body) { + iotAgentLib.commandQueue('smartgondor', 'gardens', 'cachedDevice', function (error, listCommands) { + should.not.exist(error); + listCommands.count.should.equal(1); + listCommands.commands[0].name.should.equal('position'); + listCommands.commands[0].type.should.equal('Array'); + listCommands.commands[0].value.should.equal('[28, -104, 23]'); + done(); + }); + }); + }); + }); + + describe('When caching is enabled and a command arrives with multiple values in the value field', function () { + const options = { + url: 'http://localhost:' + iotAgentConfig.server.port + '/v2/op/update', + method: 'POST', + json: { + actionType: 'update', + entities: [ + { + id: 'Robot:cachedDevice', + type: 'Robot', + position: { + type: 'Array', + value: { + attr1: 12, + attr2: 24 + } + } + } + ] + }, + headers: { + 'fiware-service': 'smartgondor', + 'fiware-servicepath': 'gardens' + } + }; + + beforeEach(function (done) { + statusAttributeMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .patch( + '/v2/entities/Robot:cachedDevice/attrs?type=Robot', + utils.readExampleFile('./test/unit/ngsiv2/examples/contextRequests/updateContextCommandStatus.json') + ) + .reply(204); + + iotAgentLib.register(device3, function (error) { + done(); + }); + }); + + it('should return a 200 OK both in HTTP and in the status code', function (done) { + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + callback(null); + }); + + request(options, function (error, response, body) { + should.not.exist(error); + + response.statusCode.should.equal(204); + + done(); + }); + }); + }); + + describe('When caching is enabled and a polling command expires', function () { + const options = { + url: 'http://localhost:' + iotAgentConfig.server.port + '/v2/op/update', + method: 'POST', + json: { + actionType: 'update', + entities: [ + { + id: 'Robot:cachedDevice', + type: 'Robot', + position: { + type: 'Array', + value: '[28, -104, 23]' + } + } + ] + }, + headers: { + 'fiware-service': 'smartgondor', + 'fiware-servicepath': 'gardens' + } + }; + + beforeEach(function (done) { + statusAttributeMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .patch( + '/v2/entities/Robot:cachedDevice/attrs?type=Robot', + utils.readExampleFile('./test/unit/ngsiv2/examples/contextRequests/updateContextCommandStatus.json') + ) + .reply(204); + + statusAttributeMock = nock('http://192.168.1.1:1026') + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', 'gardens') + .patch( + '/v2/entities/Robot:cachedDevice/attrs?type=Robot', + utils.readExampleFile( + './test/unit//ngsiv2/examples/contextRequests/updateContextCommandExpired.json' + ) + ) + .reply(204); + + iotAgentLib.register(device3, function (error) { + done(); + }); + }); + + it('should remove it from the queue', function (done) { + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + callback(null); + }); + + request(options, function (error, response, body) { + setTimeout(function () { + iotAgentLib.commandQueue('smartgondor', 'gardens', 'cachedDevice', function (error, listCommands) { + should.not.exist(error); + listCommands.count.should.equal(0); + done(); + }); + }, 300); + }); + }); + + it('should mark it as ERROR in the Context Broker', function (done) { + iotAgentLib.setCommandHandler(function (id, type, service, subservice, attributes, callback) { + callback(null); + }); + + request(options, function (error, response, body) { + setTimeout(function () { + iotAgentLib.commandQueue('smartgondor', 'gardens', 'cachedDevice', function (error, listCommands) { + statusAttributeMock.done(); + done(); + }); + }, 300); + }); + }); + }); +}); diff --git a/test/unit/mongodb/mongodb-group-registry-test.js b/test/unit/mongodb/mongodb-group-registry-test.js index 556fa319e..263ccd7bc 100644 --- a/test/unit/mongodb/mongodb-group-registry-test.js +++ b/test/unit/mongodb/mongodb-group-registry-test.js @@ -57,7 +57,7 @@ const iotAgentConfig = { deviceRegistrationDuration: 'P1M' }; const mongo = require('mongodb').MongoClient; -const mongoUtils = require('./mongoDBUtils'); +const mongoUtils = require('../../tools/mongoDBUtils'); const optionsCreation = { url: 'http://localhost:4041/iot/services', method: 'POST', diff --git a/test/unit/mongodb/mongodb-registry-test.js b/test/unit/mongodb/mongodb-registry-test.js index 26c8e2b25..2ad558389 100644 --- a/test/unit/mongodb/mongodb-registry-test.js +++ b/test/unit/mongodb/mongodb-registry-test.js @@ -30,7 +30,7 @@ const logger = require('logops'); const mongo = require('mongodb').MongoClient; const nock = require('nock'); const async = require('async'); -const mongoUtils = require('./mongoDBUtils'); +const mongoUtils = require('../../tools/mongoDBUtils'); let contextBrokerMock; const iotAgentConfig = { contextBroker: { diff --git a/test/unit/ngsi-ld/lazyAndCommands/active-devices-attribute-update-test.js b/test/unit/ngsi-ld/lazyAndCommands/active-devices-attribute-update-test.js index 862cfb71e..2667344b6 100644 --- a/test/unit/ngsi-ld/lazyAndCommands/active-devices-attribute-update-test.js +++ b/test/unit/ngsi-ld/lazyAndCommands/active-devices-attribute-update-test.js @@ -29,7 +29,7 @@ const iotAgentLib = require('../../../../lib/fiware-iotagent-lib'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); let contextBrokerMock; const iotAgentConfig = { diff --git a/test/unit/ngsi-ld/lazyAndCommands/command-test.js b/test/unit/ngsi-ld/lazyAndCommands/command-test.js index 1d5259109..f4b3f78b2 100644 --- a/test/unit/ngsi-ld/lazyAndCommands/command-test.js +++ b/test/unit/ngsi-ld/lazyAndCommands/command-test.js @@ -30,7 +30,7 @@ const utils = require('../../../tools/utils'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); const timekeeper = require('timekeeper'); let contextBrokerMock; diff --git a/test/unit/ngsi-ld/lazyAndCommands/lazy-devices-test.js b/test/unit/ngsi-ld/lazyAndCommands/lazy-devices-test.js index d30ce02a4..e31eab0c4 100644 --- a/test/unit/ngsi-ld/lazyAndCommands/lazy-devices-test.js +++ b/test/unit/ngsi-ld/lazyAndCommands/lazy-devices-test.js @@ -32,7 +32,7 @@ const apply = async.apply; const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); const timekeeper = require('timekeeper'); let contextBrokerMock; diff --git a/test/unit/ngsi-ld/lazyAndCommands/polling-commands-test.js b/test/unit/ngsi-ld/lazyAndCommands/polling-commands-test.js index e838941ee..19b605757 100644 --- a/test/unit/ngsi-ld/lazyAndCommands/polling-commands-test.js +++ b/test/unit/ngsi-ld/lazyAndCommands/polling-commands-test.js @@ -30,7 +30,7 @@ const utils = require('../../../tools/utils'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); let contextBrokerMock; let statusAttributeMock; diff --git a/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js b/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js index 55e8c5229..67b702f5e 100644 --- a/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js +++ b/test/unit/ngsi-mixed/provisioning/ngsi-versioning-test.js @@ -55,7 +55,7 @@ const iotAgentConfig = { deviceRegistrationDuration: 'P1M' }; const mongo = require('mongodb').MongoClient; -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const optionsCreationDefault = { url: 'http://localhost:4041/iot/services', method: 'POST', diff --git a/test/unit/ngsiv2/general/startup-test.js b/test/unit/ngsiv2/general/startup-test.js index 9f4ca0918..f54e8d8e7 100644 --- a/test/unit/ngsiv2/general/startup-test.js +++ b/test/unit/ngsiv2/general/startup-test.js @@ -145,6 +145,90 @@ describe('NGSI-v2 - Startup tests', function () { }); }); + describe('When the IoT Agent is started with memcache environment variables', function () { + beforeEach(function () { + process.env.IOTA_MEMCACHE_ENABLED = 'true'; + process.env.IOTA_MEMCACHE_DEVICE_MAX = 9990; + process.env.IOTA_MEMCACHE_DEVICE_TTL = 99; + process.env.IOTA_MEMCACHE_GROUP_MAX = 90; + process.env.IOTA_MEMCACHE_GROUP_TTL = 9; + }); + + afterEach(function () { + delete process.env.IOTA_MEMCACHE_ENABLED; + delete process.env.IOTA_MEMCACHE_DEVICE_MAX; + delete process.env.IOTA_MEMCACHE_DEVICE_TTL; + delete process.env.IOTA_MEMCACHE_GROUP_MAX; + delete process.env.IOTA_MEMCACHE_GROUP_TTL; + }); + + afterEach(function (done) { + iotAgentLib.deactivate(done); + }); + + it('should load the correct configuration parameters', function (done) { + iotAgentLib.activate(iotAgentConfig, function (error) { + config.getConfig().memCache.enabled.should.equal(true); + config.getConfig().memCache.deviceMax.should.equal('9990'); + config.getConfig().memCache.deviceTTL.should.equal('99'); + config.getConfig().memCache.groupMax.should.equal('90'); + config.getConfig().memCache.groupTTL.should.equal('9'); + done(); + }); + }); + }); + + describe('When the IoT Agent is started with Redis environment variables', function () { + beforeEach(function () { + process.env.IOTA_REDIS_ENABLED = 'true'; + process.env.IOTA_REDIS_DEVICE_HOST = 'redishost1'; + process.env.IOTA_REDIS_DEVICE_PORT = 6999; + process.env.IOTA_REDIS_DEVICE_PASSWORD = 'xxx'; + process.env.IOTA_REDIS_DEVICE_DB = 2; + process.env.IOTA_REDIS_DEVICE_TTL = 9999; + process.env.IOTA_REDIS_GROUP_HOST = 'redishost2'; + process.env.IOTA_REDIS_GROUP_PORT = 6998; + process.env.IOTA_REDIS_GROUP_PASSWORD = 'yyy'; + process.env.IOTA_REDIS_GROUP_DB = 3; + process.env.IOTA_REDIS_GROUP_TTL = 999; + }); + + afterEach(function () { + delete process.env.IOTA_REDIS_ENABLED; + delete process.env.IOTA_REDIS_DEVICE_HOST; + delete process.env.IOTA_REDIS_DEVICE_PORT; + delete process.env.IOTA_REDIS_DEVICE_PASSWORD; + delete process.env.IOTA_REDIS_DEVICE_DB; + delete process.env.IOTA_REDIS_DEVICE_TTL; + delete process.env.IOTA_REDIS_GROUP_HOST; + delete process.env.IOTA_REDIS_GROUP_PORT; + delete process.env.IOTA_REDIS_GROUP_PASSWORD; + delete process.env.IOTA_REDIS_GROUP_DB; + delete process.env.IOTA_REDIS_GROUP_TTL; + }); + + afterEach(function (done) { + iotAgentLib.deactivate(done); + }); + + it('should load the correct configuration parameters', function (done) { + iotAgentLib.activate(iotAgentConfig, function (error) { + config.getConfig().redis.enabled.should.equal(true); + config.getConfig().redis.deviceHost.should.equal('redishost1'); + config.getConfig().redis.devicePort.should.equal('6999'); + config.getConfig().redis.devicePassword.should.equal('xxx'); + config.getConfig().redis.deviceDB.should.equal('2'); + config.getConfig().redis.deviceTTL.should.equal('9999'); + config.getConfig().redis.groupHost.should.equal('redishost2'); + config.getConfig().redis.groupPort.should.equal('6998'); + config.getConfig().redis.groupPassword.should.equal('yyy'); + config.getConfig().redis.groupDB.should.equal('3'); + config.getConfig().redis.groupTTL.should.equal('999'); + done(); + }); + }); + }); + describe('When the IoT Agent is started with mongodb params', function () { beforeEach(function () { process.env.IOTA_MONGO_HOST = 'mongohost'; diff --git a/test/unit/ngsiv2/lazyAndCommands/active-devices-attribute-update-test.js b/test/unit/ngsiv2/lazyAndCommands/active-devices-attribute-update-test.js index 057e94c44..3abb9b851 100644 --- a/test/unit/ngsiv2/lazyAndCommands/active-devices-attribute-update-test.js +++ b/test/unit/ngsiv2/lazyAndCommands/active-devices-attribute-update-test.js @@ -29,7 +29,7 @@ const iotAgentLib = require('../../../../lib/fiware-iotagent-lib'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); let contextBrokerMock; const iotAgentConfig = { diff --git a/test/unit/ngsiv2/lazyAndCommands/command-test.js b/test/unit/ngsiv2/lazyAndCommands/command-test.js index 8dc8f8bad..08c3a52c6 100644 --- a/test/unit/ngsiv2/lazyAndCommands/command-test.js +++ b/test/unit/ngsiv2/lazyAndCommands/command-test.js @@ -30,7 +30,7 @@ const utils = require('../../../tools/utils'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); const timekeeper = require('timekeeper'); let contextBrokerMock; diff --git a/test/unit/ngsiv2/lazyAndCommands/lazy-devices-test.js b/test/unit/ngsiv2/lazyAndCommands/lazy-devices-test.js index d1c7915e6..b45a89b0e 100644 --- a/test/unit/ngsiv2/lazyAndCommands/lazy-devices-test.js +++ b/test/unit/ngsiv2/lazyAndCommands/lazy-devices-test.js @@ -32,7 +32,7 @@ const apply = async.apply; const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); const timekeeper = require('timekeeper'); let contextBrokerMock; diff --git a/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js b/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js index cee110053..ad0c49dc5 100644 --- a/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js +++ b/test/unit/ngsiv2/lazyAndCommands/polling-commands-test.js @@ -30,7 +30,7 @@ const utils = require('../../../tools/utils'); const should = require('should'); const logger = require('logops'); const nock = require('nock'); -const mongoUtils = require('../../mongodb/mongoDBUtils'); +const mongoUtils = require('../../../tools/mongoDBUtils'); const request = require('request'); let contextBrokerMock; let statusAttributeMock;