Merge branch 'RC-1-11-0' into 'master'

Resolve RC-1-11-0 See merge request automation-toolchain/f5-telemetry!242
F5Networks · Apr 20, 2020 · 4b54941 · 4b54941
2 parents 556cd01 + d5c50eb
commit 4b54941
Show file tree

Hide file tree

Showing 137 changed files with 9,704 additions and 2,405 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -16,6 +16,10 @@ stages:
 #                                                            #
 ##############################################################
 .install_unittest_packages_cmd: &install_unittest_packages_cmd
+- echo "Node.js version - $(node --version)"
+- if node -e "process.exit(process.version.startsWith('v4.')?0:1);"; then
+- npm install --global [email protected]
+- fi
 - npm run install-test
 
 .run_unittest_cmd: &run_unittest_cmd
@@ -34,7 +38,7 @@ stages:
     image: ${CICD_CONTAINER_DEPLOY}
     tags:
         - cm-official-docker-executor
-
+ 
 .run_unittest:
     extends:
         - .test_job_definition
@@ -48,6 +52,44 @@ stages:
         when: on_failure
         expire_in: 3 days
 
+.base_azure: &base_azure
+    tags:
+        - docker-executor
+    only:
+        variables:
+            - $RUN_AZURE_TESTS == "true"
+
+.deploy_azure: &deploy_azure
+    extends:
+        - .base_azure
+    image: mcr.microsoft.com/azure-cli
+    stage: deploy
+    artifacts:
+        name: ${CI_COMMIT_REF_NAME}_azure
+        paths:
+            - deploy_output
+            - deploy_output_gov
+        expire_in: 4 days
+
+.test_azure: &test_azure
+    extends:
+        - .base_azure
+    stage: functional test
+    artifacts:
+        name: ${CI_JOB_NAME}_functests_artifacts
+        paths:
+            - test/artifacts
+        when: always
+        expire_in: 4 days
+
+# Tear-down Azure environment in separate job, since 'after_script' has a 5min timeout
+.teardown_azure: &teardown_azure
+    extends:
+        - .base_azure
+    image: mcr.microsoft.com/azure-cli
+    stage: teardown
+    when: always
+
 ##############################################################
 #                                                            #
 # Jobs                                                       #
@@ -98,6 +140,7 @@ npm_audit:
         - apt-get install -y jq
         # install node modules
         - *install_unittest_packages_cmd
+        - npm list --json
         # npm audit - install includes audit, but perform specific check and fail if needed
         - audit_report=$(npm audit --json) || echo ""
         - echo "$audit_report"
@@ -169,7 +212,7 @@ deploy_env:
         variables:
             - $REQ_DEVICE_PIPELINE == "true"
     script:
-        - $SHELL ./scripts/functional-testing/setup.sh
+        - $SHELL ./scripts/functional_testing/vio/setup.sh
 
 test_functional:
     stage: functional test
@@ -196,6 +239,7 @@ test_functional:
         - npm install [email protected]
         - npm run test-functional
 
+# should be executed manually to remove the harness
 teardown_env:
     extends:
         - .harness_deployment_definition
@@ -205,7 +249,65 @@ teardown_env:
         variables:
             - $REQ_DEVICE_PIPELINE == "true"
     script:
-        - $SHELL ./scripts/functional-testing/teardown.sh
+        - $SHELL ./scripts/functional_testing/vio/teardown.sh
+
+# Azure Public
+deploy_env_azure:
+    extends:
+        - .deploy_azure
+    script:
+        - mkdir -p './deploy_output'
+        - source ./scripts/functional_testing/azure/deploy_env.sh
+
+test_functional_azure:
+    extends:
+        - .test_azure
+    script:
+        - source ./deploy_output/azure.sh
+        - echo $AZURE_VM_HOSTNAME
+        - echo $AZURE_VM_IP
+        - *install_unittest_packages_cmd
+        - npm install [email protected]
+        - npm run test-functional-cloud-azure
+    needs:
+        - build_rpm
+        - deploy_env_azure
+
+teardown_env_azure:
+    extends:
+        - .teardown_azure
+    script:
+        - source ./scripts/functional_testing/azure/teardown_env.sh
+
+
+# Azure Gov
+deploy_env_azure_gov:
+    extends:
+        - .deploy_azure
+    script:
+        - mkdir -p './deploy_output_gov'
+        - source ./scripts/functional_testing/azure/deploy_env.sh 'gov'
+
+test_functional_azure_gov:
+    extends:
+        - .test_azure
+    script:
+        - source ./deploy_output_gov/azure_gov.sh
+        - echo $AZURE_VM_HOSTNAME
+        - echo $AZURE_VM_IP
+        - *install_unittest_packages_cmd
+        - npm install [email protected]
+        - npm run test-functional-cloud-azure
+    needs:
+        - build_rpm
+        - deploy_env_azure_gov
+
+
+teardown_env_azure_gov:
+    extends:
+        - .teardown_azure
+    script:
+        - source ./scripts/functional_testing/azure/teardown_env.sh 'gov'
 
 # Publish to internal artifactory
 # Note: Will publish when new tags are pushed and use the current build in dist directory

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,24 @@
 # Changelog
 Changes to this project are documented in this file. More detail and links can be found in the Telemetry Streaming [Document Revision History](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html).
 
+## 1.11.0
+### Added
+- AUTOTOOL-1448: Introduce new Pull Consumer class, Pull Consumer API, and 'default' Pull Consumer
+- AUTOTOOL-1187: Enable use of Managed Identities for Azure Log Analytics
+- AUTOTOOL-1186: New consumer Azure Application Insights
+- AUTOTOOL-1218: Enable use of Managed Identities for Azure Application Insights
+- AUTOTOOL-768: Enable support of AzureGov for Azure consumers and optional region property
+### Fixed
+- AUTOTOOL-1334: Fix [Regular Expression Denial of Service vulnerability](https://www.npmjs.com/advisories/1488) and improve start up time on node v4.x and v6.x
+- AUTOTOOL-1150: [GitHub #30](https://github.com/F5Networks/f5-telemetry-streaming/issues/30) Fix error when Splunk consumer (configured with 'legacy' format) tries to forward event from Event Listener
+- AUTOTOOL-491: [GitHub #17](https://github.com/F5Networks/f5-telemetry-streaming/issues/17) Fix crash in Kafka consumer on attempt to close idle connections to brokers
+- AUTOTOOL-1265: Reduce data copying in System Stats
+- AUTOTOOL-1266: Reduce data copying in Event Listener
+### Changed
+- AUTOTOOL-1376: Rename Google StackDriver consumer to 'Google Cloud Monitoring'
+- AUTOTOOL-1015: Update f5-teem to 1.4.2 for new reportRecord API
+### Removed
+
 ## 1.10.0
 ### Added
 - AUTOTOOL-1111: Enable configurable polling with Telemetry_Endpoints (BIG-IP paths) and multiple system poller support

diff --git a/SUPPORT.md b/SUPPORT.md
@@ -17,9 +17,9 @@ Currently supported versions:
 
 | Software Version | Release Type  | First Customer Ship | End of Support  |
 |------------------|---------------|---------------------|-----------------|
-| TS 1.8.0         | Feature       |  03-Dec-2019        | 03-Mar-2020     |
 | TS 1.9.0         | Feature       |  28-Jan-2020        | 28-Apr-2020     |
 | TS 1.10.0        | Feature       |  10-Mar-2020        | 10-Jun-2020     |
+| TS 1.11.0        | Feature       |  21-Apr-2020        | 21-Jul-2020     |
 
 Versions no longer supported:
 
@@ -32,6 +32,7 @@ Versions no longer supported:
 | TS 1.5.0         | Feature       |  30-Jul-2019        | 30-Oct-2019     |
 | TS 1.6.0         | Feature       |  10-Sep-2019        | 10-Dec-2019     |
 | TS 1.7.0         | Feature       |  22-Oct-2019        | 22-Jan-2020     |
+| TS 1.8.0         | Feature       |  03-Dec-2019        | 03-Mar-2020     |
 
 
 See the [Release notes](https://github.com/F5Networks/f5-telemetry-streaming/releases) and [Telemetry Streaming documentation](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/userguide/revision-history.html) for new features and issues resolved for each release.
diff --git a/contributing/README.md b/contributing/README.md
@@ -108,7 +108,7 @@ How does the project handle a typical `POST` request?
             "trace": false,
             "format": "default"
         },
-        "schemaVersion": "1.10.0"
+        "schemaVersion": "1.11.0"
     }
 }
 ```
@@ -190,9 +190,9 @@ Collect the raw data from the device by adding a new endpoint to the paths confi
 ```javascript
 {
     "path": "/mgmt/tm/sys/someEndpoint", // REST endpoint
-    "includeStats": true, // Certain data is only available via /mgmt/tm/sys/someEndpoint as opposed to /mgmt/tm/sys/someEndpoint/stats, this property accomodates for this by making call to /stats (for each item) and adding that data to the original object
-    "expandReferences": { "membersReference": { "endpointSuffix": "/stats" } }, // Certain data requires getting a list of objects and then in each object expanding/following references to a child object.  'membersReference' is the name of that key (currently looking under 'items' in the data returned) and will result in self link data being retrived and 'membersReference' key being replaced with that data. If 'endpointSuffix' is supplied, a suffix is added to each self link prior to retrieval, otherwise, the value of self link as is will be used. In cases like gslb where both config and stats are needed, both the `link` and `link/stats` need to be fetched, hence, the resulting config is "expandReferences": { "membersReference": { "includeStats": true } }, which is equivalent to "expandReferences": { "membersReference": { "endpointSuffix": "", "includeStats": true } }. TODO: revisit keywords/ naming here to consolidate and avoid confusion
-    "endpointFields": [ "name", "fullPath", "selfLink", "ipProtocol", "mask" ], // Will collect only these fields from the endoint. Useful when using includeStats and the same property exists in both endpoints. Also can be used instead of a large exclude/include statement in properties.json
+    "includeStats": true, // Certain data is only available via /mgmt/tm/sys/someEndpoint as opposed to /mgmt/tm/sys/someEndpoint/stats, this property accommodates for this by making call to /stats (for each item) and adding that data to the original object
+    "expandReferences": { "membersReference": { "endpointSuffix": "/stats" } }, // Certain data requires getting a list of objects and then in each object expanding/following references to a child object.  'membersReference' is the name of that key (currently looking under 'items' in the data returned) and will result in self link data being retrieved and 'membersReference' key being replaced with that data. If 'endpointSuffix' is supplied, a suffix is added to each self link prior to retrieval, otherwise, the value of self link as is will be used. In cases like gslb where both config and stats are needed, both the `link` and `link/stats` need to be fetched, hence, the resulting config is "expandReferences": { "membersReference": { "includeStats": true } }, which is equivalent to "expandReferences": { "membersReference": { "endpointSuffix": "", "includeStats": true } }. TODO: revisit keywords/ naming here to consolidate and avoid confusion
+    "endpointFields": [ "name", "fullPath", "selfLink", "ipProtocol", "mask" ], // Will collect only these fields from the endpoint. Useful when using includeStats and the same property exists in both endpoints. Also can be used instead of a large exclude/include statement in properties.json
     "body": "{ \"command\": \"run\", \"utilCmdArgs\": \"-c \\\"/bin/df -P | /usr/bin/tr -s ' ' ','\\\"\" }", // Certain information may require using POST instead of GET and require an HTTP body, if body is defined that gets used along with a POST. Body can be either string or object
     "name": "someStatRef", // Alternate name to reference in properties.json, default is to use the endpoint
     "ignoreCached": true // Invalidate cached response of previous request to endpoint
@@ -221,7 +221,7 @@ Enable and define how the data should look by adding a new key under *stats* in
 ```javascript
 {
     "someKey": {
-        "key": "/mgmt/tm/sys/someUri::someChildKey", // /uri (or alt name in paths.json) + key(s) seperated by '::' to navigate into object and get a specific value
+        "key": "/mgmt/tm/sys/someUri::someChildKey", // /uri (or alt name in paths.json) + key(s) separated by '::' to navigate into object and get a specific value
         "keyArgs": { // Arguments that can be passed to the associated alt name endpoint in paths.json
             "replaceStrings": { "\\$tmstatsTable": "cpu_info_stat" } // Key/value pairs that replace matching strings in request body. The key is treated as a regular expression
         }
@@ -235,10 +235,10 @@ Enable and define how the data should look by adding a new key under *stats* in
                 "includeFirstEntry": { "pattern": "/stats", "excludePattern": "/members/",  "runFunctions": [ {"name": "someCustomFunc" } ]  }, // This is useful if aggregating data from /endpoint and /endpoint/stats typically.  Allows a complex object to by merged instead of nesting down into entries, instead the values in the first entry of 'entries' will be copied to the top level object and then discarded.  There may be multiple 'entries', of which only some should follow this property, that is supported with an optional pattern, excludePattern and runFunctions.
             },
             {
-                "filterKeys": { "exclude": [ "removeMe"] }, // Filter all keys in object using either an inclusio or exclusion list - include also supported, not an exact match
+                "filterKeys": { "exclude": [ "removeMe"] }, // Filter all keys in object using either an inclusion or exclusion list - include also supported, not an exact match
             },
             {
-                "renameKeys": { "name/": { "pattern": "name\/(.*)", "group": 1 }, "~": { "replaceCharacter": "/" },  }, // Rename keys, useful if key contains unneccesary prefix/suffix or needs a specific character replaced. This can also be an array with 1+ rename key objects inside it to guarantee order.
+                "renameKeys": { "name/": { "pattern": "name\/(.*)", "group": 1 }, "~": { "replaceCharacter": "/" },  }, // Rename keys, useful if key contains unnecessary prefix/suffix or needs a specific character replaced. This can also be an array with 1+ rename key objects inside it to guarantee order.
             },
             {
                 "runFunctions": [{ "name": "getPercentFromKeys", "args": { "totalKey": "memoryTotal", "partialKey": "memoryUsed" } }], // Run custom functions, nail meet hammer.  This is to be used for one-offs where creating a standard macro does not make sense, keeping in mind each custom function could be used multiple times.  The function should already exist inside of normalizeUtil.js.
@@ -265,7 +265,7 @@ Certain properties require dynamic data to be pulled from the system prior to *s
 ```javascript
 {
     "someStat": {
-        "key": "/mgmt/tm/cm/device::items::{{HOSTNAME}}::description" // "HOSTNAME" (surrounded by '{{' and '}}') is context key which containts device's hostname
+        "key": "/mgmt/tm/cm/device::items::{{HOSTNAME}}::description" // "HOSTNAME" (surrounded by '{{' and '}}') is context key which contains device's hostname
     }
 }
 ```
@@ -278,7 +278,7 @@ This context data is defined on the same level as *stats* in the properties conf
 {
     "context": {
         "someCtxKey1": {
-            "key": "/mgmt/tm/sys/global-settings::hostname" // other Macros properties are available too. Context data is not availble!
+            "key": "/mgmt/tm/sys/global-settings::hostname" // other Macros properties are available too. Context data is not available!
         }
     }
 }
@@ -291,7 +291,7 @@ This context data is defined on the same level as *stats* in the properties conf
     "context": [
         {
             "someCtxKey1": {
-                "key": "/mgmt/tm/sys/global-settings::hostname" // other Macros properties are available too. Context data is not availble for the first set of Macros.
+                "key": "/mgmt/tm/sys/global-settings::hostname" // other Macros properties are available too. Context data is not available for the first set of Macros.
             }
         },
         {
@@ -345,7 +345,7 @@ Additional information about the testing methodology can be found in the [test r
 
 Build/publish makes heavy use of GitLab and [.gitlab-ci.yml](../.gitlab-ci.yml).  Check out CI file and GitLab documentation for more details.
 
-- Add *new* RPM to `dist/` directory (from build artifact on mainline developement branch)
+- Add *new* RPM to `dist/` directory (from build artifact on mainline development branch)
 - Publish to artifactory (automated on new tags)
 - Push to GitLab (mainline release branch)
 - Push to GitHub (mainline release branch)

diff --git a/contributing/process_release.md b/contributing/process_release.md
@@ -36,6 +36,7 @@
 * Update [SUPPORT.md](SUPPORT.md) if not yet done (or at least check that everything looks valid):
   * add new version to the list of `Currently supported versions` with appropriate dates
   * remove no longer supported versions from `Currently supported versions` and add it to `Versions no longer supported`
+* Would be good to remove local `node_modules` directory and `package-lock.json` and do `npm run install-test`. Ideally `package-lock.json` will be the same as before. If not then it probably means that someone forgot to update it.
 * Push all changes to GitLab
 * Get build artifacts (`.rpm` and `.sha256` checksum) from latest build and:
   * Check `.rpm` size, ideally it should not exceed 10 MB.:
@@ -44,6 +45,7 @@
     * 1.8.0 - 9.5 MB
     * 1.9.0 - 9.5 MB
     * 1.10.0 - 9.5 MB
+    * 1.11.0 - 9.5 MB
   * Install build to BIG-IP, navigate to folder `/var/config/rest/iapps/f5-telemetry/` and check following:
     * Run `du -sh` and check that folder's size (shouldn't be much greater than previous versions):
       * 1.4.0 - 65 MB
@@ -53,22 +55,24 @@
       * 1.8.0 - 73 MB
       * 1.9.0 - 73 MB
       * 1.10.0 - 76 MB
+      * 1.11.0 - 75 MB
     * Check `nodejs/node_modules` folder - if you see `eslint`, `mocha` or something else from [package.json](package.json) `devDependencies` section - something wrong with build process. Probably some `npm` flags are work as not expected and it MUST BE FIXED before publishing.
 * Ensure that all tests (unit tests and functional tests passed)
 * Create pre-release tag and push it to GitLab:
   * git tag -m 'Release candidate X.Y.Z-#' vX.Y.Z-#
   * git push origin
   * git push origin --tags
 * Check pipeline for artifactory URL to package (or browse in artifactory)
-* Create and send release candidate email with features, bugs, artifactory URL
+* Send release candidate email with features, bugs, artifactory URL
 
 ## Release process
 
 * Create new branch from `master`, e.g. `rc-master-branch`. It will be easier to merge branches and resolve conflicts without any following up issues.
-* Merge RC branch into RC master branch - squash to avoid commits leaking sensitive url's, etc.
+* Merge RC branch into RC master branch - squash to avoid leaking sensitive URLs, etc. through commits
   * git checkout rc-master-branch
   * git merge --squash vX.Y.Z
   * git push origin
+* Would be good to remove local `node_modules` directory and `package-lock.json` and do `npm run install-test`. Ideally `package-lock.json` will be the same as before. If not then it probably means that someone forgot to update it.
 * Ideally it will be good to run functional and unit testing
 * After that merge `rc-master-branch` to `master` branch:
   * git checkout master
@@ -93,5 +97,6 @@
   * Navigate to the latest release, select `edit` and upload artifacts:
     * `.rpm` file
     * `.sha256` file
+* Remove all RC tags and branches and other stale branches that were used for release or RC process
 
 # ATTENTION: DO NOT FORGET TO MERGE 'MASTER' BRANCH INTO 'DEVELOP' WHEN YOU ARE DONE WITH RELEASE PROCESS