diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2fbd4f7..50ab85c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: needs: unit-test services: mongodb: - image: mongo:4.4 + image: mongo:6.0 ports: - 27017:27017 steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index b287e6e4..e0084162 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,9 +1,13 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + mkdocs: configuration: mkdocs.yml python: - version: 3.8 install: - requirements: doc/requirements.txt diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 5dd9d5de..a64e74f5 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,3 @@ - Add: aggrMethod accept multiple values separated by comma (#432, partial) - Add: aggrMethod 'all' to get all the possible aggregations (#432, partial) -- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Remove: RPM stuff diff --git a/Changelog b/Changelog new file mode 100644 index 00000000..05664546 --- /dev/null +++ b/Changelog @@ -0,0 +1,81 @@ +2.11.0 (November 28th, 2023) + +- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) +- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions +- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) + +2.10.0 (November 2nd, 2022) + +- Fix: healthcheck over sth exposed port +- Fix: Dockerfile to include initial packages upgrade +- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) + +2.9.0 (March 25th, 2022) + +- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) +- Add: Docker healthcheck for STH API +- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) +- Add: allow connect with mongo uri without auth +- Fix: return 500 when DB is not connected (#570) +- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) +- Fix: update logs about get raw and aggregated data (#556) +- Update codebase to use ES6 + - Remove JSHint and jshint overrides + - Add esLint using standard tamia presets + - Replace var with let/const + - Fix or disable eslint errors +- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption +- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) +- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile + +2.8.0 (May 27th, 2020) + +- Fix: Filetype issue fixed for hLimit and hOffset (#539) +- Fix: Updated temporary csv filename for uniqueness (#532) +- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life +- Make optional PM2 usage in docker entrypoint +- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) + +2.7.0 (November 11th, 2019) + +- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) +- Add: CORS support (#500) +- Fix: check header response before use it (CSV response has no header) +- Fix: handler for reply a CSV: use new Hapi API (#513) +- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) +- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues + +2.6.0 (June 4th, 2019) + +- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security +- Add: logging feature to count number of requests attended and number of requests processed with error (#310) +- Fix: race condition causes erroneous count to 0 in some cases (#493) + +2.5.0 (December 19th, 2018) + +- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) +- Add: use NodeJS 8 in Dockerfile +- Add: use PM2 in Dockerfile +- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 +- Upgrade: logops dependence from ~1.0.5 to 2.1.0 +- Upgrade: joi dependence from ~5.1.0 to 14.0.6 +- Upgrade: boom dependence from ~2.7.2 to 7.2.2 +- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 +- Upgrade: request development dependence from ~2.79.0 to 2.88.0 +- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 +- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) + +2.4.0 (August 16th, 2018) + +- Upgrade lodash dependence to 4.17.5 +- Upgrade mongodb dependence to 2.2.35 +- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] +- Fix: Content Disposition header encoding issue (#433) +- Add count to query and related fiware-total-count header [#428] +- Using precise dependencies (~=) in packages.json +- Upgrade hapi to 11.1.3 + +2.3.0 (October 18th, 2017) + +- FEATURE update node version to 4.8.4 \ No newline at end of file diff --git a/README.md b/README.md index d11839b2..cf08459a 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ [![](https://nexus.lab.fiware.org/repository/raw/public/badges/chapters/core.svg)](https://www.fiware.org/developers/catalogue/) [![License](https://img.shields.io/github/license/telefonicaid/fiware-sth-comet.svg)](https://opensource.org/licenses/AGPL-3.0) -[![Docker badge](https://img.shields.io/docker/pulls/fiware/sth-comet.svg)](https://hub.docker.com/r/fiware/sth-comet-ngsi/) [![](https://img.shields.io/badge/tag-fiware--sth-comet-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-sth-comet) [![Support badge](https://img.shields.io/badge/support-askbot-yellowgreen.svg)](https://ask.fiware.org/questions/scope%3Aall/tags%3Asth-comet/) [![Join the chat at https://gitter.im/telefonicaid/fiware-sth-comet](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/telefonicaid/fiware-sth-comet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Quay badge](https://img.shields.io/badge/quay.io-fiware%2Fsth--comet-grey?logo=red%20hat&labelColor=EE0000)](https://quay.io/repository/fiware/sth-comet) +[![Docker badge](https://img.shields.io/badge/docker-telefonicaiot%2Ffiware--sth--comet-blue?logo=docker)](https://registry.hub.docker.com/r/telefonicaiot/fiware-sth-comet) +
[![Documentation badge](https://readthedocs.org/projects/fiware-sth-comet/badge/?version=latest)](https://fiware-sth-comet.readthedocs.io) [![CI](https://github.com/telefonicaid/fiware-sth-comet/workflows/CI/badge.svg)](https://github.com/telefonicaid/fiware-sth-comet/actions?query=workflow%3ACI) [![Coverage Status](https://coveralls.io/repos/github/telefonicaid/fiware-sth-comet/badge.svg?branch=master)](https://coveralls.io/github/telefonicaid/fiware-sth-comet?branch=master) @@ -25,8 +27,8 @@ interfaces. This project is part of [FIWARE](https://www.fiware.org/). For more information check the FIWARE Catalogue entry for the [Core Context Management](https://github.com/Fiware/catalogue/tree/master/core). -| :books: [Documentation](https://fiware-sth-comet.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/core/sth-comet) | :whale: [Docker Hub](https://hub.docker.com/r/fiware/sth-comet/) | :dart: [Roadmap](https://github.com/telefonicaid/fiware-sth-comet/blob/master/doc/roadmap.md) | -| --------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| :books: [Documentation](https://fiware-sth-comet.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/core/sth-comet) | [quay.io](https://quay.io/repository/fiware/sth-comet) | :whale: [Docker Hub](https://registry.hub.docker.com/r/telefonicaiot/fiware-sth-comet) | :dart: [Roadmap](https://github.com/telefonicaid/fiware-sth-comet/blob/master/doc/roadmap.md) | +| --------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | --- | # Contents diff --git a/doc/manuals/db_indexes.md b/doc/manuals/db_indexes.md index 38563bd8..8c8d3670 100644 --- a/doc/manuals/db_indexes.md +++ b/doc/manuals/db_indexes.md @@ -16,6 +16,16 @@ It is recommended to create the following index in this collection: The performance difference can be dramatic for large sets of data. For instance, for a collection with around ~3000000 query execution time can drop from 3 seconds to 1 millisecond. +Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :---------------- | :--------------------------------------------------------------- | +| dm-by-servicepath | recvTime, entityId, entityType, attrName, attrType, attrValue | +| dm-by-entity | recvTime, attrName, attrType, attrValue | +| dm-by-attribute | recvTime, attrType, attrValue | + +Note that datamodel others that the ones above are not allowed by Cygnus. + ## Indexes in the aggregated data collection It is recommended to create the following index in this collection: @@ -26,3 +36,13 @@ It is recommended to create the following index in this collection: The performance difference can be dramatic for large sets of data. For instance, for a collection with around ~6000000 query execution time can drop from 28 seconds to 20 millisecond. + +Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel: + +| datamodel | keys | +| :----------------- | :------------------------------------------------- | +| dm-by-servicepath | entityId, entityType, attrName, resolution, origin | +| dm-by-entity | attrName, resolution, origin | +| dm-by-attribute | resolution, origin | + +Note that datamodel others that the ones above are not allowed by Cygnus. diff --git a/doc/manuals/installation.md b/doc/manuals/installation.md index 75ce7060..f1230992 100644 --- a/doc/manuals/installation.md +++ b/doc/manuals/installation.md @@ -3,8 +3,7 @@ The STH component provides 3 alternatives for its installation: 1. Cloning the GitHub repository -2. Using a RPM package -3. Automatic deployment using Docker +2. Automatic deployment using Docker ## Cloning the GitHub repository @@ -24,118 +23,6 @@ npm install The STH component server is ready to be started as a Node application. -## Using a RPM package - -In the next sections we detail the steps to follow to install, update and remove the STH component using a RPM package. - -### Package generation - -**Prerequisites:** To generate the RPM package from the STH component sources it is needed to have the rpm build tools -(`rpmbuild` executable), Node and the npm utilities, as well as an Internet connection to download the required Node -modules. - -To generate the RPM package for the STH component, execute the following command from the root of the STH component: - -```bash -./rpm/create-rpm.sh -v -r -``` - -If everything goes fine, a new RPM package will be created and stored in the following location: -`./rpm/RPMS/x86_64/fiware-sth-comet--.x86_64.rpm`. - -Execute the next command to get additional information about the RPM package creation script: - -```bash -./rpm/create-rpm.sh -h -``` - -### Installation, upgrade and removal - -**Prerequisites:** Node is needed to install the generated STH component RPM package. - -To install or upgrade the STH component, execute: - -```bash -sudo rpm -Uvh fiware-sth-comet--.x86_64.rpm -``` - -After the installation, the following files and directories are created: - -```text -/etc/init.d -└── sth - -/etc/logrotate.d -└── logrotate-sth-daily - -/var/log/sth - -/var/run/sth - -/opt/sth -├── conf -│ └── Here is where instances are configured -├── node_modules -│ └── -├── package.json -└── src - └── -``` - -To remove a previous STH component installation, execute: - -```bash -sudo rpm -e fiware-sth-comet -``` - -### Configuration - -STH is able to start multiple instances using the -[`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script by -adding and configuring certain files as detailed next. - -To start multiple instances, one configuration file per instance has to be included in the `/opt/sth/conf` directory. It -is important to note that the default installation includes preconfigured instances. - -It is important to change the `STH_PORT` value included in the configuration files to a value not used by other STH -instances/services. It is also a good practice to change the `LOG_FILE_NAME` value to avoid getting the logs from -several instances mixed. - -The [`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script -includes the following operations: - -- **start**: if no `` (see below) is provided, the script starts an instance per configuration file found in - the `/opt/sth/conf` directory matching the `sth_*.conf` template. If `` is provided, a configuration file - named `sth_.conf` is searched in the `/opt/sth/conf` directory and the corresponding instance is started. -- **stop**: if no `` (see below) is provided, the script stops all the instances by listing all pid files - under `/var/run/sth` matching the pattern `sth_*.pid`. If `` is provided, the scripts stops the instance - with the associated pid file `/var/run/sth/sth_.pid` -- **status**: The status operation shows information about one or more running instances following the same procedure - detailed in the `stop` operation. -- **restart**: The restart operation executes a `stop` operation followed by a `start` operation according to the - procedure detailed in those operations. - -```bash -# start -sudo /sbin/service sth start [] -# stop -sudo /sbin/service sth stop [] -# status -sudo /sbin/service sth status [] -# restart -sudo /sbin/service sth restart [] -``` - -An example -[`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf) file -has been included in the component repository in GitHub to guide the STH instance configuration. - - - -Last but not least, the STH process (a `node` process) runs the as `sth` user. - - - ## Automatic deployment using Docker To ease the testing and deployment of the STH component, there also exists Docker images hosted at the diff --git a/doc/manuals/running.md b/doc/manuals/running.md index dfdaeeab..4d19b02c 100644 --- a/doc/manuals/running.md +++ b/doc/manuals/running.md @@ -9,7 +9,7 @@ To run the STH server, please execute the following command from the directory w The STH component provides the user with 2 mechanisms to configure the component to the concrete needs of the user: - Environment variables, which can be set assigning values to them or using the - [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf) + [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/sth_default.conf) file if a packaged version of the STH component is used. - The [`config.js`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/config.js) file located at the root of the STH component code, a JSON formatted file including the configuration properties. @@ -101,6 +101,7 @@ The environment variables accepted by the script (for which there exists counter - `PROOF_OF_LIFE_INTERVAL`: The time in seconds between proof of life logging messages informing that the server is up and running normally. Default value: "60". `PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL`: The time in seconds between processed requests statistics appear in the logs. Default value: "60". +- `CORS_ENABLED`: Boolean attribute (`true`|`false`) to enable cors configuration. Optional. Default value: "false". For example, to start the STH server listening on port 7777, connecting to a MongoDB instance listening on mymongo.com:27777 and without filtering out the empty results, use: diff --git a/doc/requirements.txt b/doc/requirements.txt index e3e60400..4252df91 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ -mkdocs==1.2.3 +mkdocs==1.2.4 Pygments==2.15.0 Markdown==3.3.4 -jinja2==3.0.0 \ No newline at end of file +jinja2==3.1.3 diff --git a/lib/configuration/sthConfiguration.js b/lib/configuration/sthConfiguration.js index 68272df4..224aa556 100644 --- a/lib/configuration/sthConfiguration.js +++ b/lib/configuration/sthConfiguration.js @@ -223,7 +223,10 @@ if (ENV.STH_PORT && !isNaN(ENV.STH_PORT)) { ); } -if (config && config.cors && config.cors.enabled) { +if (ENV.CORS_ENABLED && !(ENV.CORS_ENABLED == null)) { + module.exports.corsEnabled = JSON.parse(ENV.CORS_ENABLED); + sthLogger.info(module.exports.LOGGING_CONTEXT.STARTUP, 'CORS value set to: ' + module.exports.corsEnabled); +} else if (config && config.cors && config.cors.enabled) { module.exports.corsEnabled = JSON.parse(config.cors.enabled); sthLogger.info(module.exports.LOGGING_CONTEXT.STARTUP, 'CORS value set to: ' + module.exports.corsEnabled); } else { diff --git a/lib/database/model/sthDatabaseNameCodecTool.js b/lib/database/model/sthDatabaseNameCodecTool.js index 4296ca65..f302619d 100644 --- a/lib/database/model/sthDatabaseNameCodecTool.js +++ b/lib/database/model/sthDatabaseNameCodecTool.js @@ -389,7 +389,7 @@ function copyDatabase(options, databaseName, callback) { const adminDB = sthDatabase.connection.admin(); const listCommand = { listCollections: 1, - nameOnly: 1 + nameOnly: true }; var sourceDB = databaseName; var targetDB = options.encode diff --git a/lib/database/model/sthDatabaseNameMapperTool.js b/lib/database/model/sthDatabaseNameMapperTool.js index d75452d7..e8af1fa9 100644 --- a/lib/database/model/sthDatabaseNameMapperTool.js +++ b/lib/database/model/sthDatabaseNameMapperTool.js @@ -392,7 +392,7 @@ function copyDatabase(options, databaseName, callback) { const adminDB = sthDatabase.connection.admin(); const listCommand = { listCollections: 1, - nameOnly: 1 + nameOnly: true }; var sourceDB = databaseName; var targetDB = options.encode diff --git a/lib/database/sthDatabase.js b/lib/database/sthDatabase.js index 9bc18ae6..51e6056e 100644 --- a/lib/database/sthDatabase.js +++ b/lib/database/sthDatabase.js @@ -220,7 +220,7 @@ function isAggregated(collectionName) { function setTTLPolicy(collection) { // Set the TTL policy if required if (sthConfig.TRUNCATION_EXPIRE_AFTER_SECONDS > 0) { - if (!isAggregated(collection)) { + if (!isAggregated(collection.collectionName)) { if (sthConfig.TRUNCATION_SIZE === 0) { collection.ensureIndex( { diff --git a/package.json b/package.json index 4017ca6c..1d29296f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "fiware-sth-comet", "description": "The Short Time Historic (STH, aka. Comet) is a component of the FIWARE ecosystem in charge of providing aggregated time series information about the evolution in time of entity attribute values registered using the Orion Context Broker, an implementation of the publish/subscribe context management system exposing NGSI9 and NGSI10 interfaces.", - "version": "2.10.0-next", + "version": "2.11.0-next", "licence": "AGPL-3.0", "homepage": "https://github.com/telefonicaid/fiware-sth-comet", "author": { diff --git a/rpm/SPECS/sth.spec b/rpm/SPECS/sth.spec deleted file mode 100644 index bac5b604..00000000 --- a/rpm/SPECS/sth.spec +++ /dev/null @@ -1,228 +0,0 @@ -Summary: Short Time Historic (STH) -Name: fiware-sth-comet -Version: %{_product_version} -Release: %{_product_release} -License: AGPLv3 -BuildRoot: %{_topdir}/BUILDROOT/ -BuildArch: x86_64 -Requires: nodejs >= 0.10.42 -Requires: logrotate -Requires(post): /sbin/chkconfig, /usr/sbin/useradd, npm -Requires(preun): /sbin/chkconfig, /sbin/service -Requires(postun): /sbin/service -Group: Applications/Engineering -Vendor: Telefonica I+D -BuildRequires: npm - -# HISTORY CHANGES -# CJMM-DevOps 2016/03/21 - -%description -The Short Time Historic (STH, aka. Comet) is a component of the FIWARE ecosystem in charge of providing aggregated time series information about the evolution in time of entity attribute values registered using the Orion Context Broker, an implementation of the publish/subscribe context management system exposing NGSI9 and NGSI10 interfaces. - -# System folders -%define _srcdir $RPM_BUILD_ROOT/../../.. -%define _project_user sth -%define _service_name sth -%define _install_dir /opt/sth -%define _sth_log_dir /var/log/sth -%define _sth_pid_dir /var/run/sth - -# RPM Building folder -%define _build_root_project %{buildroot}%{_install_dir} -# -------------------------------------------------------------------------------------------- # -# prep section, setup macro: -# -------------------------------------------------------------------------------------------- # -%prep -echo "[INFO] Preparing installation" -# Create rpm/BUILDROOT folder -rm -Rf $RPM_BUILD_ROOT && mkdir -p $RPM_BUILD_ROOT -[ -d %{_build_root_project} ] || mkdir -p %{_build_root_project} - -# Copy src files -cp -R %{_srcdir}/lib \ - %{_srcdir}/bin \ - %{_srcdir}/config.js \ - %{_srcdir}/package.json \ - %{_build_root_project} - -[ -f %{_srcdir}/npm-shrinkwrap.json ] && /bin/cp %{_srcdir}/npm-shrinkwrap.json %{_build_root_project} - -cp -R %{_topdir}/SOURCES/etc %{buildroot} - -# Create conf dir -mkdir -p %{_build_root_project}/conf - -# -------------------------------------------------------------------------------------------- # -# Build section: -# -------------------------------------------------------------------------------------------- # -%build -echo "[INFO] Building RPM" -cd %{_build_root_project} - -# Only production modules. We have found that --force is required to make this work for Node v8 -rm -fR node_modules/ -npm cache clear --force -npm install --production - -# -------------------------------------------------------------------------------------------- # -# pre-install section: -# -------------------------------------------------------------------------------------------- # -%pre -echo "[INFO] Creating %{_project_user} user" -grep ^%{_project_user}: /etc/passwd -RET_VAL=$? -if [ "$RET_VAL" != "0" ]; then - /usr/sbin/useradd -s "/bin/bash" -d %{_install_dir} %{_project_user} - RET_VAL=$? - if [ "$RET_VAL" != "0" ]; then - echo "[ERROR] Unable create %{_project_user} user" \ - exit $RET_VAL - fi - -fi - -# -------------------------------------------------------------------------------------------- # -# post-install section: -# -------------------------------------------------------------------------------------------- # -%post -echo "[INFO] Configuring application" - - echo "[INFO] Creating the home Short Time Historic (STH) directory" - mkdir -p _install_dir - echo "[INFO] Creating log & run directory" - mkdir -p %{_sth_log_dir} - chown -R %{_project_user}:%{_project_user} %{_sth_log_dir} - chown -R %{_project_user}:%{_project_user} _install_dir - chmod g+s %{_sth_log_dir} - setfacl -d -m g::rwx %{_sth_log_dir} - setfacl -d -m o::rx %{_sth_log_dir} - - mkdir -p %{_sth_pid_dir} - chown -R %{_project_user}:%{_project_user} %{_sth_pid_dir} - chown -R %{_project_user}:%{_project_user} _install_dir - chmod g+s %{_sth_pid_dir} - setfacl -d -m g::rwx %{_sth_pid_dir} - setfacl -d -m o::rx %{_sth_pid_dir} - - echo "[INFO] Configuring application service" - cd /etc/init.d - chkconfig --add %{_service_name} - -echo "Done" - -# -------------------------------------------------------------------------------------------- # -# pre-uninstall section: -# -------------------------------------------------------------------------------------------- # -%preun - -echo "[INFO] stoping service %{_service_name}" -service %{_service_name} stop &> /dev/null - -if [ $1 == 0 ]; then - - echo "[INFO] Removing application log files" - # Log - [ -d %{_sth_log_dir} ] && rm -rfv %{_sth_log_dir} - - echo "[INFO] Removing application run files" - # Log - [ -d %{_sth_pid_dir} ] && rm -rfv %{_sth_pid_dir} - - echo "[INFO] Removing application files" - # Installed files - [ -d %{_install_dir} ] && rm -rfv %{_install_dir} - - echo "[INFO] Removing application user" - userdel -fr %{_project_user} - - echo "[INFO] Removing application service" - chkconfig --del %{_service_name} - rm -Rf /etc/init.d/%{_service_name} - echo "Done" -fi - -# -------------------------------------------------------------------------------------------- # -# post-uninstall section: -# clean section: -# -------------------------------------------------------------------------------------------- # -%postun -%clean -rm -rf $RPM_BUILD_ROOT - -# -------------------------------------------------------------------------------------------- # -# Files to add to the RPM -# -------------------------------------------------------------------------------------------- # -%files -%defattr(755,%{_project_user},%{_project_user},755) -%config /etc/init.d/%{_service_name} -%config /etc/logrotate.d/logrotate-sth-daily.conf -%{_install_dir} - -%changelog -* Wed Nov 02 2022 Alvaro Vega 2.10.0 -- Fix: healthcheck over sth exposed port -- Fix: Dockerfile to include initial packages upgrade -- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) - -* Fri Mar 25 2022 Fermin Galan 2.9.0 -- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) -- Add: Docker healthcheck for STH API -- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) -- Add: allow connect with mongo uri without auth -- Fix: return 500 when DB is not connected (#570) -- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) -- Fix: update logs about get raw and aggregated data (#556) -- Update codebase to use ES6 - - Remove JSHint and jshint overrides - - Add esLint using standard tamia presets - - Replace var with let/const - - Fix or disable eslint errors -- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption -- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) -- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile - -* Wed May 27 2020 Alvaro Vega 2.8.0 -- Fix: Filetype issue fixed for hLimit and hOffset (#539) -- Fix: Updated temporary csv filename for uniqueness (#532) -- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life -- Make optional PM2 usage in docker entrypoint -- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) - -* Mon Nov 11 2019 Fermin Galan 2.7.0 -- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) -- Add: CORS support (#500) -- Fix: check header response before use it (CSV response has no header) -- Fix: handler for reply a CSV: use new Hapi API (#513) -- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues - -* Tue Jun 04 2019 Fermin Galan 2.6.0 -- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security -- Add: logging feature to count number of requests attended and number of requests processed with error (#310) -- Fix: race condition causes erroneous count to 0 in some cases (#493) - -* Wed Dec 19 2018 Fermin Galan 2.5.0 -- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) -- Add: use NodeJS 8 in Dockerfile -- Add: use PM2 in Dockerfile -- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 -- Upgrade: logops dependence from ~1.0.5 to 2.1.0 -- Upgrade: joi dependence from ~5.1.0 to 14.0.6 -- Upgrade: boom dependence from ~2.7.2 to 7.2.2 -- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 -- Upgrade: request development dependence from ~2.79.0 to 2.88.0 -- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 -- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) - -* Thu Aug 16 2018 Alvaro Vega 2.4.0 -- Upgrade lodash dependence to 4.17.5 -- Upgrade mongodb dependence to 2.2.35 -- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] -- Fix: Content Disposition header encoding issue (#433) -- Add count to query and related fiware-total-count header [#428] -- Using precise dependencies (~=) in packages.json -- Upgrade hapi to 11.1.3 - -* Wed Oct 18 2017 Fermin Galan 2.3.0 -- FEATURE update node version to 4.8.4 diff --git a/rpm/create-rpm.sh b/rpm/create-rpm.sh deleted file mode 100755 index 77b82760..00000000 --- a/rpm/create-rpm.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U -# -# This file is part of the Short Time Historic (STH) component -# -# STH is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the License, -# or (at your option) any later version. -# -# STH is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with STH. -# If not, seehttp://www.gnu.org/licenses/. -# -# For those usages not covered by the GNU Affero General Public License -# please contact with: [german.torodelvalle@telefonica.com] - - -function usage() { - SCRIPT=$(basename $0) - - printf "\n" >&2 - printf "usage: ${SCRIPT} [options] \n" >&2 - printf "\n" >&2 - printf "Options:\n" >&2 - printf "\n" >&2 - printf " -h show usage\n" >&2 - printf " -v VERSION Mandatory parameter. Version for rpm product preferably in format x.y.z \n" >&2 - printf " -r RELEASE Mandatory parameter. Release for product. I.E. 0.ge58dffa \n" >&2 - printf "\n" >&2 -} - -while getopts ":v:r:u:a:h" opt - -do - case $opt in - v) - VERSION_ARG=${OPTARG} - ;; - r) - RELEASE_ARG=${OPTARG} - ;; - h) - usage - exit 0 - ;; - *) - echo "invalid argument: '${OPTARG}'" - exit 1 - ;; - esac -done - -BASE_DIR="$(cd ${0%/*} && pwd -P)/.." -RPM_BASE_DIR="${BASE_DIR}/rpm" - -if [[ ! -z ${VERSION_ARG} ]]; then - PRODUCT_VERSION=${VERSION_ARG} -else - echo "A product version must be specified with -v parameter." - usage - exit 2 -fi - -if [[ ! -z ${RELEASE_ARG} ]]; then - PRODUCT_RELEASE=${RELEASE_ARG} -else - echo "A product reslease must be specified with -r parameter." - usage - exit 2 -fi - -PROXY_USER="sth" - -# Clean previous builds of rpm -echo "INFO: Clean previous builds of rpm <${RPM_BASE_DIR}/{RPMS,BUILDROOT,BUILD,SRPMS}>" -rm -rf ${RPM_BASE_DIR}/{RPMS,BUILDROOT,BUILD,SRPMS} - -rpmbuild -ba ${RPM_BASE_DIR}/SPECS/sth.spec \ - --define "_topdir ${RPM_BASE_DIR}" \ - --define "_project_user ${PROXY_USER}" \ - --define "_product_version ${PRODUCT_VERSION}" \ - --define "_product_release ${PRODUCT_RELEASE}" - diff --git a/rpm/get_version_string.sh b/rpm/get_version_string.sh deleted file mode 100755 index f81c24d3..00000000 --- a/rpm/get_version_string.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# Bash lib to know the RPM version and revision from a GitHub repository -# Call method get_rpm_version_string to obtain them for rpmbuild -# The result appear on the vars ver and rel -# The requisites are tags similar to 0.1.0/KO. This tag must be created by 'git tag -a 0.1.0' -# The main purpose to use this script is to deploy CI on master branch. -# -# Steps to get version and release: -# 1 - source get_version_string.sh -# 2 - Execute any of the functions on the script -# - It will be use the command 'read ver rel < <(get_rpm_version_string)' in order to get version and release on different vars - -if [[ $(ps -hp $$ | grep bash) ]]; then - shopt -s extglob -elif [[ $(ps -hp $$ | grep zsh) ]]; then - setopt kshglob -fi - -get_branch() -{ - git rev-parse --abbrev-ref HEAD -} - -## Specific functions according the TID workflow -get_branch_type() -{ - local branch="$(get_branch)" - case $branch in - release/*) echo "release";; - master) echo "latest";; - *) echo "other";; - esac -} - -get_version_string() -{ - local branch branch_name describe_tags version ancestor release - case $(get_branch_type) in - master) - ## If we are in master use the total count of commits of the repo - total_commit_number=$(git rev-list --all --count) - short_hash=$(git rev-parse --short HEAD) - version="$(git describe --tags --long --match "[[:digit:]]*.[[:digit:]]*.[[:digit:]]*" 2>/dev/null)" - version="${version%-*-*}" - version="${version%KO}" - echo "${version}-${total_commit_number}-${short_hash}" - ;; - release) - ## in release branches the version is a tag named - branch_name="$(get_branch)" - branch_name="${branch_name#*/}" - describe_tags="$(git describe --tags --long --match ${branch_name} 2>/dev/null)" - version="${describe_tags%-*-*}" - version="${version%KO}" - release=${describe_tags#*.*.*-} - echo "${version}-${release}" - ;; - other) - ## We are in detached mode, use the last x-y-z tag - version="$(git describe --tags --long --match "[[:digit:]]*.[[:digit:]]*.[[:digit:]]*" 2>/dev/null)" - version="${version%-*-*}" - version="${version%KO}" - echo "${version}" - ;; - *) - # RMs don't stablish any standard here, we use branch name as version - version=$(get_branch) - # Using always master as parent branch does not describe correctly the number of revision - # for branches not starting there, but works as an incremental rev - ancestor="$(git merge-base $version master)" - version=${version#*/} - local res="$(git log --oneline ${ancestor}.. --pretty='format:%h')" - ## wc alone does not get the last line when there's no new line - [[ -z $res ]] && rel=0 || rel=$(echo "$res" | wc -l | tr -d ' ') - echo "${version}-${rel}-g$(git log --pretty=format:'%h' -1)" - esac -} - -get_rpm_version_string() { - local version_string ver rel - version_string="$(get_version_string)" - ver="${version_string%-*-*}" - rel="${version_string:$((${#ver}+1))}" - echo "${ver//[[:space:]-\/#]}" "${rel//[-]/.}" -} diff --git a/rpm/EXAMPLES/sth_default.conf b/sth_default.conf similarity index 100% rename from rpm/EXAMPLES/sth_default.conf rename to sth_default.conf diff --git a/test/acceptance/requirements.txt b/test/acceptance/requirements.txt index 9ac72ad0..4f621c1e 100644 --- a/test/acceptance/requirements.txt +++ b/test/acceptance/requirements.txt @@ -3,6 +3,6 @@ lettuce==0.2.20 requests==2.31.0 xmltodict==0.9.2 termcolor==1.1.0 -pymongo==2.7.2 +pymongo==4.6.3 # blindly changed from 2.7.2 due to vulnerability check in github fabric==1.10.1 git+https://github.com/telefonicaid/lettuce-tools.git#egg=lettuce_tools