Skip to content

Commit

Permalink
Merge branch 'rc-master-branch' into 'master'
Browse files Browse the repository at this point in the history
RC 1.10.0 merge to master

See merge request automation-toolchain/f5-telemetry!215
  • Loading branch information
kaustriaf5 committed Mar 9, 2020
2 parents 3790eb0 + 169695b commit 556cd01
Show file tree
Hide file tree
Showing 148 changed files with 25,250 additions and 6,226 deletions.
188 changes: 84 additions & 104 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,101 +10,112 @@ stages:
- teardown
- publish

lint:
image: node:8
stage: lint
##############################################################
# #
# Jobs and commands templates #
# #
##############################################################
.install_unittest_packages_cmd: &install_unittest_packages_cmd
- npm run install-test

.run_unittest_cmd: &run_unittest_cmd
- npm run test-only

.job_definition: &job_definition
tags:
- docker-executor
script:
- npm run install-test
- npm run lint

# BIG-IP 13.x and BIG-IP 14.0, unittests only (without coverage check)
test_node4:
image: node:4
.test_job_definition: &test_job_definition
extends:
- .job_definition
stage: test

.harness_deployment_definition: &harness_deployment_definition
image: ${CICD_CONTAINER_DEPLOY}
tags:
- docker-executor
- cm-official-docker-executor

.run_unittest:
extends:
- .test_job_definition
script:
- npm run install-test
- npm install [email protected]
- npm run test-only
- *install_unittest_packages_cmd
- *run_unittest_cmd
artifacts:
name: ${CI_COMMIT_REF_NAME}_unittests_artifacts
paths:
- test/artifacts
when: on_failure
expire_in: 3 days

##############################################################
# #
# Jobs #
# #
##############################################################

lint:
extends:
- .test_job_definition
image: node:8
stage: lint
script:
- *install_unittest_packages_cmd
- npm run lint

# BIG-IP 13.x and BIG-IP 14.0, unittests only (without coverage check)
test_node4:
extends:
- .run_unittest
image: node:4

# just in case, unittests only (without coverage check)
test_node6:
extends:
- .run_unittest
image: node:6
stage: test
tags:
- docker-executor
script:
- npm run install-test
- npm run test-only
artifacts:
name: ${CI_COMMIT_REF_NAME}_unittests_artifacts
paths:
- test/artifacts
when: on_failure
expire_in: 3 days

# BIG-IP 14.1+, unittests only (without coverage check)
test_node8:
stage: test
tags:
- docker-executor
script:
- npm run install-test
- npm run test-only
artifacts:
name: ${CI_COMMIT_REF_NAME}_unittests_artifacts
paths:
- test/artifacts
when: on_failure
expire_in: 3 days
extends:
- .run_unittest
image: node:8

# mostly for containers, unittests only (without coverage check)
test_node_latest:
extends:
- .run_unittest
image: node:latest
stage: test
tags:
- docker-executor
script:
- npm run install-test
- npm run test-only
artifacts:
name: ${CI_COMMIT_REF_NAME}_unittests_artifacts
paths:
- test/artifacts
when: on_failure
expire_in: 3 days

# run tests and check code coverage
coverage:
stage: test
# packages audit
npm_audit:
extends:
- .test_job_definition
allow_failure: true
script:
# install jq
- apt-get update
- apt-get install -y jq
# install node modules
- npm run install-test
- *install_unittest_packages_cmd
# npm audit - install includes audit, but perform specific check and fail if needed
- audit_report=$(npm audit --json)
- echo $audit_report
- audit_report=$(npm audit --json) || echo ""
- echo "$audit_report"
- actions=$(echo $audit_report | jq .actions | jq length)
- if [ $actions -ne 0 ]; then echo 'ERROR! vulnerabilities exist'; exit 1; fi
# unit tests

# run tests and check code coverage
coverage:
extends:
- .test_job_definition
script:
- *install_unittest_packages_cmd
# run tests with coverage report
- npm test
artifacts:
name: ${CI_COMMIT_REF_NAME}_unittests_coverage
paths:
- coverage
tags:
- cm-official-docker-executor

build_rpm:
image: f5devcentral/containthedocs:rpmbuild
Expand Down Expand Up @@ -145,23 +156,10 @@ build_docs:
- docs/_build/html
expire_in: 1 month

# for this job following variables should be defined:
# CICD_AUTH_OS_USERNAME - VIO user
# CICD_AUTH_OS_PASSWORD - VIO password
# CICD_AUTH_OS_PROJECT - VIO project
# or
# CICD_AUTH_OS_TOKEN - VIO auth token
# CICD_AUTH_OS_PROJECT - VIO project
# Also, variable to *enable* device pipeline should exist
# REQ_DEVICE_PIPELINE - boolean
deploy_env:
image: ${CICD_CONTAINER_DEPLOY}
extends:
- .harness_deployment_definition
stage: deploy
tags:
- cm-official-docker-executor
variables:
PROJECT_DECLARATION: ${CI_PROJECT_DIR}/test/functional/deployment/declaration.yml
CUSTOM_DECLARATION: "yes"
artifacts:
name: ${CI_COMMIT_REF_NAME}_bigip.harness_info
paths:
Expand All @@ -171,24 +169,10 @@ deploy_env:
variables:
- $REQ_DEVICE_PIPELINE == "true"
script:
- export PROJECT_NAME=$([ "${CICD_PROJECT_NAME}" == "" ] && echo "test_functional_harness" || echo "${CICD_PROJECT_NAME}")
- export PROJECT_DIR="/root/deploy-projects/${PROJECT_NAME}"
- declaration=$(sed "s/_DEPLOYMENT_NAME_/${PROJECT_NAME}/g" "${PROJECT_DECLARATION}")
- echo "$declaration" > "${PROJECT_DECLARATION}"
- cat "${PROJECT_DECLARATION}"
- cd /root/cicd-bigip-deploy && make configure &&
make printvars &&
make setup && ls -als ${PROJECT_DIR} &&
cp ${PROJECT_DIR}/harness_facts_flat.json ${CI_PROJECT_DIR}/harness_facts_flat.json
- $SHELL ./scripts/functional-testing/setup.sh

test_functional:
stage: functional test
script:
- export TEST_HARNESS_FILE=${CI_PROJECT_DIR}/harness_facts_flat.json
# really only need dev dependencies
- npm run install-test
- ls ./dist -ls
- npm run test-functional
# troubleshooting functional test failures typically requires looking at logs, one of which is
# the restnoded log that is captured by the functional tests. This saves off the folder
# containing that log as an artifact to speed up the troubleshooting process
Expand All @@ -204,29 +188,24 @@ test_functional:
variables:
# enable this job
- $RUN_FUNCTIONAL_TESTS == "true"
script:
- export TEST_HARNESS_FILE=${CI_PROJECT_DIR}/harness_facts_flat.json
- ls ./dist -ls
# really only need dev dependencies
- *install_unittest_packages_cmd
- npm install [email protected]
- npm run test-functional

# should be executed manually to remove the harness
teardown_env:
image: ${CICD_CONTAINER_DEPLOY}
extends:
- .harness_deployment_definition
stage: teardown
tags:
- cm-official-docker-executor
variables:
PROJECT_DECLARATION: ${CI_PROJECT_DIR}/test/functional/deployment/declaration.yml
CUSTOM_DECLARATION: "yes"
script:
- export PROJECT_NAME=$([ "${CICD_PROJECT_NAME}" == "" ] && echo "test_functional_harness" || echo "${CICD_PROJECT_NAME}")
- export PROJECT_DIR="/root/deploy-projects/${PROJECT_NAME}"
- declaration=$(sed "s/_DEPLOYMENT_NAME_/${PROJECT_NAME}/g" "${PROJECT_DECLARATION}")
- echo "$declaration" > "${PROJECT_DECLARATION}"
- cat "${PROJECT_DECLARATION}"
- cd /root/cicd-bigip-deploy && make configure &&
make printvars &&
make teardown
when: manual
only:
variables:
- $REQ_DEVICE_PIPELINE == "true"
script:
- $SHELL ./scripts/functional-testing/teardown.sh

# Publish to internal artifactory
# Note: Will publish when new tags are pushed and use the current build in dist directory
Expand Down Expand Up @@ -289,6 +268,7 @@ pages:
only:
# only update on designated, stable branch
- develop
- doc-release-branch

# Publish docs to clouddocs.f5networks.net
publish_docs_to_staging:
Expand Down
19 changes: 19 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,25 @@
# Changelog
Changes to this project are documented in this file. More detail and links can be found in the Telemetry Streaming [Document Revision History](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html).

## 1.10.0
### Added
- AUTOTOOL-1111: Enable configurable polling with Telemetry_Endpoints (BIG-IP paths) and multiple system poller support
- AUTOTOOL-1148: Allow 'OR' logic by adding ifAnyMatch functionality.
- AUTOTOOL-853: Support F5 systems (ex: Viprion) that have multiple hosts
### Fixed
- AUTOTOOL-1051: Event Listener unable to classify AFM DoS event
- AUTOTOOL-1037: Splunk legacy tmstats - include last_cycle_count
- AUTOTOOL-1019: Splunk legacy tmstats - add tenant and application data
- AUTOTOOL-1128: Declarations with large secrets may timeout
- AUTOTOOL-1154: Passphrases should be obfuscated in consumer trace files
- AUTOTOOL-1147: Add 'profiles' data (profiles attached to Virtual Server) to 'virtualServers'
- AUTOTOOL-896: [GitHub #26](https://github.com/F5Networks/f5-telemetry-streaming/pull/26): Use baseMac instead of hostname to fetch CM device
- AUTOTOOL-1160: cipherText validation when protected by SecureVault
- AUTOTOOL-1239: Caching data about the host device to speed up declaration processing
### Changed
- AUTOTOOL-1062: Update NPM packages
### Removed

## 1.9.0
### Added
- AUTOTOOL-725 and AUTOTOOL-755: Add support for GSLB WideIP and Pools Config and Stats
Expand Down
1 change: 1 addition & 0 deletions SUPPORT.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Currently supported versions:
|------------------|---------------|---------------------|-----------------|
| TS 1.8.0 | Feature | 03-Dec-2019 | 03-Mar-2020 |
| TS 1.9.0 | Feature | 28-Jan-2020 | 28-Apr-2020 |
| TS 1.10.0 | Feature | 10-Mar-2020 | 10-Jun-2020 |

Versions no longer supported:

Expand Down
8 changes: 4 additions & 4 deletions contributing/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ How does the project handle a typical `POST` request?
"trace": false,
"format": "default"
},
"schemaVersion": "1.9.0"
"schemaVersion": "1.10.0"
}
}
```
Expand Down Expand Up @@ -181,19 +181,19 @@ Collect the raw data from the device by adding a new endpoint to the paths confi
```javascript
{
"endpoint": "/mgmt/tm/sys/global-settings"
"path": "/mgmt/tm/sys/global-settings"
}
```
*Advanced Macros:* These macros signal the system to retrieve the data in specific, additional ways.
```javascript
{
"endpoint": "/mgmt/tm/sys/someEndpoint", // REST endpoint
"path": "/mgmt/tm/sys/someEndpoint", // REST endpoint
"includeStats": true, // Certain data is only available via /mgmt/tm/sys/someEndpoint as opposed to /mgmt/tm/sys/someEndpoint/stats, this property accomodates for this by making call to /stats (for each item) and adding that data to the original object
"expandReferences": { "membersReference": { "endpointSuffix": "/stats" } }, // Certain data requires getting a list of objects and then in each object expanding/following references to a child object. 'membersReference' is the name of that key (currently looking under 'items' in the data returned) and will result in self link data being retrived and 'membersReference' key being replaced with that data. If 'endpointSuffix' is supplied, a suffix is added to each self link prior to retrieval, otherwise, the value of self link as is will be used. In cases like gslb where both config and stats are needed, both the `link` and `link/stats` need to be fetched, hence, the resulting config is "expandReferences": { "membersReference": { "includeStats": true } }, which is equivalent to "expandReferences": { "membersReference": { "endpointSuffix": "", "includeStats": true } }. TODO: revisit keywords/ naming here to consolidate and avoid confusion
"endpointFields": [ "name", "fullPath", "selfLink", "ipProtocol", "mask" ], // Will collect only these fields from the endoint. Useful when using includeStats and the same property exists in both endpoints. Also can be used instead of a large exclude/include statement in properties.json
"body": "{ \"command\": \"run\", \"utilCmdArgs\": \"-c \\\"/bin/df -P | /usr/bin/tr -s ' ' ','\\\"\" }", // Certain information may require using POST instead of GET and require an HTTP body, if body is defined that gets used along with a POST
"body": "{ \"command\": \"run\", \"utilCmdArgs\": \"-c \\\"/bin/df -P | /usr/bin/tr -s ' ' ','\\\"\" }", // Certain information may require using POST instead of GET and require an HTTP body, if body is defined that gets used along with a POST. Body can be either string or object
"name": "someStatRef", // Alternate name to reference in properties.json, default is to use the endpoint
"ignoreCached": true // Invalidate cached response of previous request to endpoint
}
Expand Down
6 changes: 5 additions & 1 deletion contributing/process_release.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
* [package.json](package.json)
* [package-lock.json](package-lock.json)
* [project.spec](project.spec) (not required starting from 1.5)
* [src/lib/constants.js](src/lib/constants.js)
* [src/lib/constants.js](src/lib/constants.js) (not required starting from 1.10)
* [src/schema/latest/base_schema.json](src/schema/latest/base_schema.json)
* [contributing/README.md](contributing/README.md) (example of response, optional)
* [docs/conf.py](docs/conf.py)
Expand All @@ -42,13 +42,17 @@
* 1.4.0 - 8.6 MB
* 1.7.0 - 8.6 MB
* 1.8.0 - 9.5 MB
* 1.9.0 - 9.5 MB
* 1.10.0 - 9.5 MB
* Install build to BIG-IP, navigate to folder `/var/config/rest/iapps/f5-telemetry/` and check following:
* Run `du -sh` and check that folder's size (shouldn't be much greater than previous versions):
* 1.4.0 - 65 MB
* 1.5.0 - 65 MB
* 1.6.0 - 66 MB
* 1.7.0 - 66 MB
* 1.8.0 - 73 MB
* 1.9.0 - 73 MB
* 1.10.0 - 76 MB
* Check `nodejs/node_modules` folder - if you see `eslint`, `mocha` or something else from [package.json](package.json) `devDependencies` section - something wrong with build process. Probably some `npm` flags are work as not expected and it MUST BE FIXED before publishing.
* Ensure that all tests (unit tests and functional tests passed)
* Create pre-release tag and push it to GitLab:
Expand Down
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
# The short X.Y version.
version = u''
# The full version, including alpha/beta/rc tags.
release = u'1.9.0'
release = u'1.10.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
Expand Down
Loading

0 comments on commit 556cd01

Please sign in to comment.