AVR data fields are sent to Elasticsearch as strings #228
Comments
ajgerace
added
bug
|
I have to agree with @ajgerace that we need better guidance on the data types for the AVR data that is exported off box by Telemetry Streaming. In the example of Elastic Search receiving AVR data, all fields are received as strings. I can use an ingest pipeline to convert some fields from strings to numbers, but doing this manually is difficult because I may not know ahead of time all of the different fields I will receive. Also, it's prone to errors, so sending data with correct data types should be the responsibility of the sender of the data, in my opinion. Optionally, within ElasticSearch, I can use an index template and dynamic mapping to automatically map all numeric strings as numbers. However then you have another problem: some fields look like numbers but should be strings. (example: HTTP response codes, or port numbers, should not be treated as numbers that you might sum or average). Here's what I think should not be text, if you were to auto-convert all numeric fields to numbers: DosL7AttackID - should be string All of this is to say that sending all fields as strings to ElasticSearch leaves the work on the customer to map these fields correctly, and we don't have sufficient public-facing documentation to make this option realistic. |
Environment
Summary
When AVR data is being sent to Elasticsearch via Telemetry Streaming all fields are being sent as strings. This requires customization on the elastic side to convert fields to their appropriate types.
example data trace
Steps To Reproduce
Steps to reproduce the behavior:
{ "class": "Telemetry", "controls": { "class": "Controls", "logLevel": "info" }, "My_System": { "class": "Telemetry_System", "enable": false, "systemPoller": { "interval": 0 }, "host": "localhost", "port": 8100, "protocol": "http", "allowSelfSignedCert": true }, "My_Listener": { "class": "Telemetry_Listener", "port": 6514 }, "my_elastic_consumer": { "class": "Telemetry_Consumer", "type": "ElasticSearch", "dataType": "_doc", "host": "10.0.20.27", "index": "avr", "protocol": "http", "port": 9200, "apiVersion": "7.17.6", "trace": true }, "schemaVersion": "1.31.0" }{
"_index": "avr",
"_type": "_doc",
"_id": "uy2Bx4MBBX2B1AIZbVuo",
"_version": 1,
"_score": 1,
"_ignored": [
"data.originalRawData.keyword"
],
"_source": {
"data": {
"hostname": "bigtest01.f5demo.org",
"SlotId": "0",
"errdefs_msgno": "22282261",
"Entity": "DosVisibilityVips",
"AggrInterval": "300",
"EOCTimestamp": "1665499500",
"HitCount": "1",
"VipName": "/Common/bodgeit_avr_test_vs",
"ServerLatency": "0",
"ServerLatencyHitCount": "0",
"ClientConcurrentConns": "0",
"ServerConcurrentConns": "0",
"MaxClientConcurrentConns": "0",
"MaxServerConcurrentConns": "0",
"ClientNewConns": "0",
"ServerNewConns": "0",
"FailedConns": "0",
"ExpiredConns": "0",
"AbandonedConns": "0",
"ClientBytesIn": "0",
"ServerBytesOut": "0",
"ServerBytesIn": "0",
"ClientOutBytes": "0",
"ClientPktsIn": "0",
"ServerPktsOut": "0",
"ServerPktsIn": "0",
"ClientPktsOut": "0",
"ConcurrentIps": "0",
"ConcurrentBlockedIps": "0",
"ConcurrentIpsParticipatingInAttacks": "0",
"ConcurrentAttacks": "0",
"ServerLatencyHealth": "0",
"ConcurrentConnectionsHealth": "0",
"ThroughputHealth": "0",
"specialConcurrentIpsForAllVips": "1",
"Rounds": "30",
"originalRawData": "Hostname="bigtest01.f5demo.org",SlotId="0",errdefs_msgno="22282261",Entity="DosVisibilityVips",AggrInterval="300",EOCTimestamp="1665499500",HitCount="1",VipName="/Common/bodgeit_avr_test_vs",ServerLatency="0",ServerLatencyHitCount="0",ClientConcurrentConns="0",ServerConcurrentConns="0",MaxClientConcurrentConns="0",MaxServerConcurrentConns="0",ClientNewConns="0",ServerNewConns="0",FailedConns="0",ExpiredConns="0",AbandonedConns="0",ClientBytesIn="0",ServerBytesOut="0",ServerBytesIn="0",ClientOutBytes="0",ClientPktsIn="0",ServerPktsOut="0",ServerPktsIn="0",ClientPktsOut="0",ConcurrentIps="0",ConcurrentBlockedIps="0",ConcurrentIpsParticipatingInAttacks="0",ConcurrentAttacks="0",ServerLatencyHealth="0",ConcurrentConnectionsHealth="0",ThroughputHealth="0",specialConcurrentIpsForAllVips="1",Rounds="30""
},
"telemetryEventCategory": "AVR"
},
"fields": {
"data.ServerLatency.keyword": [
"0"
],
"data.ClientPktsOut": [
"0"
],
"data.ConcurrentIpsParticipatingInAttacks": [
"0"
],
"data.ServerBytesOut": [
"0"
],
"data.specialConcurrentIpsForAllVips": [
"1"
],
"data.ConcurrentIpsParticipatingInAttacks.keyword": [
"0"
],
"data.EOCTimestamp": [
"1665499500"
],
"data.ExpiredConns.keyword": [
"0"
],
"data.ExpiredConns": [
"0"
],
"data.ClientBytesIn": [
"0"
],
"data.ConcurrentIps": [
"0"
],
"data.ClientPktsOut.keyword": [
"0"
],
"data.ServerConcurrentConns": [
"0"
],
"data.AggrInterval.keyword": [
"300"
],
"data.ServerPktsOut": [
"0"
],
"data.Rounds": [
"30"
],
"data.hostname": [
"bigtest01.f5demo.org"
],
"data.Entity": [
"DosVisibilityVips"
],
"data.ServerNewConns.keyword": [
"0"
],
"data.MaxServerConcurrentConns.keyword": [
"0"
],
"data.ServerConcurrentConns.keyword": [
"0"
],
"data.ConcurrentIps.keyword": [
"0"
],
"data.Entity.keyword": [
"DosVisibilityVips"
],
"data.ServerPktsIn": [
"0"
],
"data.ServerBytesIn": [
"0"
],
"data.errdefs_msgno.keyword": [
"22282261"
],
"data.Rounds.keyword": [
"30"
],
"telemetryEventCategory.keyword": [
"AVR"
],
"data.ConcurrentAttacks.keyword": [
"0"
],
"data.MaxClientConcurrentConns": [
"0"
],
"data.VipName.keyword": [
"/Common/bodgeit_avr_test_vs"
],
"data.ServerLatencyHitCount": [
"0"
],
"data.ClientNewConns": [
"0"
],
"data.ServerNewConns": [
"0"
],
"data.ClientConcurrentConns.keyword": [
"0"
],
"data.SlotId": [
"0"
],
"data.FailedConns.keyword": [
"0"
],
"data.ServerLatency": [
"0"
],
"data.AbandonedConns": [
"0"
],
"data.ConcurrentBlockedIps.keyword": [
"0"
],
"data.ThroughputHealth.keyword": [
"0"
],
"data.AggrInterval": [
"300"
],
"data.errdefs_msgno": [
"22282261"
],
"data.ClientNewConns.keyword": [
"0"
],
"data.ServerPktsOut.keyword": [
"0"
],
"data.specialConcurrentIpsForAllVips.keyword": [
"1"
],
"data.originalRawData": [
"Hostname="bigtest01.f5demo.org",SlotId="0",errdefs_msgno="22282261",Entity="DosVisibilityVips",AggrInterval="300",EOCTimestamp="1665499500",HitCount="1",VipName="/Common/bodgeit_avr_test_vs",ServerLatency="0",ServerLatencyHitCount="0",ClientConcurrentConns="0",ServerConcurrentConns="0",MaxClientConcurrentConns="0",MaxServerConcurrentConns="0",ClientNewConns="0",ServerNewConns="0",FailedConns="0",ExpiredConns="0",AbandonedConns="0",ClientBytesIn="0",ServerBytesOut="0",ServerBytesIn="0",ClientOutBytes="0",ClientPktsIn="0",ServerPktsOut="0",ServerPktsIn="0",ClientPktsOut="0",ConcurrentIps="0",ConcurrentBlockedIps="0",ConcurrentIpsParticipatingInAttacks="0",ConcurrentAttacks="0",ServerLatencyHealth="0",ConcurrentConnectionsHealth="0",ThroughputHealth="0",specialConcurrentIpsForAllVips="1",Rounds="30""
],
"data.ClientPktsIn.keyword": [
"0"
],
"data.ServerLatencyHealth": [
"0"
],
"data.ConcurrentBlockedIps": [
"0"
],
"data.ConcurrentAttacks": [
"0"
],
"data.ClientConcurrentConns": [
"0"
],
"data.ClientBytesIn.keyword": [
"0"
],
"data.hostname.keyword": [
"bigtest01.f5demo.org"
],
"data.ServerBytesIn.keyword": [
"0"
],
"data.ServerLatencyHitCount.keyword": [
"0"
],
"data.FailedConns": [
"0"
],
"data.ServerPktsIn.keyword": [
"0"
],
"data.EOCTimestamp.keyword": [
"1665499500"
],
"data.ClientPktsIn": [
"0"
],
"data.MaxServerConcurrentConns": [
"0"
],
"data.ServerBytesOut.keyword": [
"0"
],
"data.VipName": [
"/Common/bodgeit_avr_test_vs"
],
"telemetryEventCategory": [
"AVR"
],
"data.ClientOutBytes": [
"0"
],
"data.HitCount.keyword": [
"1"
],
"data.AbandonedConns.keyword": [
"0"
],
"data.ConcurrentConnectionsHealth.keyword": [
"0"
],
"data.SlotId.keyword": [
"0"
],
"data.HitCount": [
"1"
],
"data.ConcurrentConnectionsHealth": [
"0"
],
"data.ServerLatencyHealth.keyword": [
"0"
],
"data.ClientOutBytes.keyword": [
"0"
],
"data.MaxClientConcurrentConns.keyword": [
"0"
],
"data.ThroughputHealth": [
"0"
]
},
"ignored_field_values": {
"data.originalRawData.keyword": [
"Hostname="bigtest01.f5demo.org",SlotId="0",errdefs_msgno="22282261",Entity="DosVisibilityVips",AggrInterval="300",EOCTimestamp="1665499500",HitCount="1",VipName="/Common/bodgeit_avr_test_vs",ServerLatency="0",ServerLatencyHitCount="0",ClientConcurrentConns="0",ServerConcurrentConns="0",MaxClientConcurrentConns="0",MaxServerConcurrentConns="0",ClientNewConns="0",ServerNewConns="0",FailedConns="0",ExpiredConns="0",AbandonedConns="0",ClientBytesIn="0",ServerBytesOut="0",ServerBytesIn="0",ClientOutBytes="0",ClientPktsIn="0",ServerPktsOut="0",ServerPktsIn="0",ClientPktsOut="0",ConcurrentIps="0",ConcurrentBlockedIps="0",ConcurrentIpsParticipatingInAttacks="0",ConcurrentAttacks="0",ServerLatencyHealth="0",ConcurrentConnectionsHealth="0",ThroughputHealth="0",specialConcurrentIpsForAllVips="1",Rounds="30""
]
}
}
Expected Behavior
We should know what data types each of the fields should be. Dates should be sent as dates and numeric fields should be sent as numbers not strings.
Actual Behavior
All fields are being sent as strings.
The text was updated successfully, but these errors were encountered: