From 88315e454a842ff212aa010b8f9819071460a809 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Fri, 22 Sep 2023 11:13:20 -0400 Subject: [PATCH] HPCC-30295 Add ALA ContainerLogV2 Support - Adds ContainerLogV2 based KQL query logic - Provides new sample helm values to enable v2 support - Ensures V1 backward compatability - Updates README documentation - Provides config files for v2 enablement - Provides logic to enable v2 on AKS deployment - Projects-away superfluous columns - Honors lookup key config for instance type - Fixes getXReturnColumns signature - Adds directory content description to README - Report error if invalid log message col detected Signed-off-by: Rodrigo Pastrana --- helm/examples/azure/log-analytics/README.md | 26 ++- .../container-azm-ms-agentconfig.yaml | 217 ++++++++++++++++++ .../log-analytics/dataCollectionSettings.json | 7 + .../log-analytics/enable-loganalytics.sh | 9 + .../azure/log-analytics/env-loganalytics | 5 +- .../loganalytics-hpcc-logaccessV2.yaml | 91 ++++++++ .../AzureLogAnalyticsCurlClient.cpp | 89 ++++++- .../AzureLogAnalyticsCurlClient.hpp | 6 +- 8 files changed, 432 insertions(+), 18 deletions(-) create mode 100644 helm/examples/azure/log-analytics/container-azm-ms-agentconfig.yaml create mode 100644 helm/examples/azure/log-analytics/dataCollectionSettings.json create mode 100644 helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml diff --git a/helm/examples/azure/log-analytics/README.md b/helm/examples/azure/log-analytics/README.md index 3d566d7225c..6fe91fb4255 100644 --- a/helm/examples/azure/log-analytics/README.md +++ b/helm/examples/azure/log-analytics/README.md @@ -22,6 +22,16 @@ The user should populate the following values in order to create a new Azure Log For example: "admin=MyName email=my.email@mycompany.com environment=myenv justification=testing" - AZURE_SUBSCRIPTION (Optional - Ensures this subscription is set before creating the new workspace) +- AKS_RESOURCE_LOCATION (Optional e.g. eastus) + +- ENABLE_CONTAINER_LOG_V2 (true|false) Enables the ContainerLog V2 schema. + If set to true, the stdout/stderr Logs are forwarded to ContainerLogV2 table, otherwise the container logs continue to be forwarded to ContainerLog table. + Utilizes ./helm/examples/azure/log-analytics/dataCollectionSettings.json, and + ./helm/examples/azure/log-analytics/container-azm-ms-agentconfig.yaml which creates and applies a new configmap 'kube-system/container-azm-ms-agentconfig'. + + Details on benefits of V2 schema: https://learn.microsoft.com/en-us/azure/azure-monitor/containers/container-insights-logging-v2?tabs=configure-portal + + #### b - Execute enable-loganalytics.sh This helper script attempts to create new Azure LogAnalytics workspace (user can provide pre-existing), associates the workspace with the target AKS cluster, and enables the Azure Log Analytics feature. This script is dependant on the values provided in the previous step. @@ -37,7 +47,7 @@ Depending on your Azure subscription structure, it might be necessary to request The Registered Application must provide a 'client secret' which is used to gain access to the Log Analytics API. -#### b - Provide AAD registered application inforation and target ALA Workspace +#### b - Provide AAD registered application information and target ALA Workspace HPCC logAccess requires access to the AAD Tenant ID, client ID, and secret which are provided by the registered app from section '2.a' above. The target workspace ID is also required, and can be retrieved after the step in section '1.b' is successfully completed. Those four values must be provided via a secure secret object. The secret is expected to be in the 'esp' category, and be named 'azure-logaccess'. @@ -62,9 +72,19 @@ Example manual secret creation command (assuming ./secrets-templates contains a ``` #### c - Configure HPCC logAccess -The target HPCC deployment should be directed to use the above Azure Log Analytics workspace, and the newly created secret by providing appropriate logAccess values (such as ./loganalytics-hpcc-logaccess.yaml). +The target HPCC deployment should be directed to use the above Azure Log Analytics workspace, and the newly created secret by providing appropriate logAccess values (such as ./loganalytics-hpcc-logaccess.yaml or ./loganalytics-hpcc-logaccessV2.yaml if targeting Azure Log Analytics ContainerLogV2 - recommended ). Example use: ```console - helm install myhpcc hpcc/hpcc -f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml + helm install myhpcc hpcc/hpcc -f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml ``` +## Directory Contents + +- 'create-azure-logaccess-secret.sh' - Script for creating 'azure-logaccess' secret needed for accessing logs stored in Azure Log Analytics +- 'secrets-templates' - Contains placeholders for information required to create 'azure-logaccess' secret via 'create-azure-logaccess-secret.sh' script +- 'enable-loganalytics.sh' - Script for enabling Azure LogAnalytics upon a given AKS cluster +- 'env-loganalytics' - Environment information required to enable ALA upon target AKS cluster. +- 'dataCollectionSettings.json' - Provided to enable ContainerLogV2 schema on target AKS cluster +- 'container-azm-ms-agentconfig.yaml' - Defines ConfigMap used to configure ALA log collection. Provided to re-direct ALA log collection to ContainerLogV2 schema. +- 'loganalytics-hpcc-logaccess.yaml' - Used to configure ALA -> HPCC LogAccess. Provides mapping between ALA log tables to HPCC's known log categories +- 'loganalytics-hpcc-logaccessV2.yaml' - Used to configure ALA -> HPCC LogAccess. Provides mapping between ALA ContainerLogV2 log table to HPCC's known log categories diff --git a/helm/examples/azure/log-analytics/container-azm-ms-agentconfig.yaml b/helm/examples/azure/log-analytics/container-azm-ms-agentconfig.yaml new file mode 100644 index 00000000000..0ed5b561d81 --- /dev/null +++ b/helm/examples/azure/log-analytics/container-azm-ms-agentconfig.yaml @@ -0,0 +1,217 @@ +# Azure Log Analytics collects stdout, stderr, and environmental variables from container +# workloads deployed to managed Kubernetes clusters from the containerized agent. +# Users can customize agent data collection settings by creating a custom Kubernetes ConfigMap +# This ConfiMap can be used to enable/disable log collection, exclude namespaces, and enable/disable +# It is included here specifically to enable Azure Log Analytics ContainerLogV2 schema +# See this document for details: +# https://learn.microsoft.com/en-us/azure/azure-monitor/containers/container-insights-agent-config#configure-and-deploy-configmaps + +kind: ConfigMap +apiVersion: v1 +data: + schema-version: + #string.used by agent to parse config. supported versions are {v1}. Configs with other schema versions will be rejected by the agent. + v1 + config-version: + #string.used by customer to keep track of this config file's version in their source control/repository (max allowed 10 chars, other chars will be truncated) + ver1 + log-data-collection-settings: |- + # Log data collection settings + # Any errors related to config map settings can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to. + + [log_collection_settings] + [log_collection_settings.stdout] + # In the absense of this configmap, default value for enabled is true + enabled = true + # exclude_namespaces setting holds good only if enabled is set to true + # kube-system,gatekeeper-system log collection are disabled by default in the absence of 'log_collection_settings.stdout' setting. If you want to enable kube-system,gatekeeper-system, remove them from the following setting. + # If you want to continue to disable kube-system,gatekeeper-system log collection keep the namespaces in the following setting and add any other namespace you want to disable log collection to the array. + # In the absense of this configmap, default value for exclude_namespaces = ["kube-system","gatekeeper-system"] + exclude_namespaces = ["kube-system","gatekeeper-system"] + + [log_collection_settings.stderr] + # Default value for enabled is true + enabled = true + # exclude_namespaces setting holds good only if enabled is set to true + # kube-system,gatekeeper-system log collection are disabled by default in the absence of 'log_collection_settings.stderr' setting. If you want to enable kube-system,gatekeeper-system, remove them from the following setting. + # If you want to continue to disable kube-system,gatekeeper-system log collection keep the namespaces in the following setting and add any other namespace you want to disable log collection to the array. + # In the absense of this configmap, default value for exclude_namespaces = ["kube-system","gatekeeper-system"] + exclude_namespaces = ["kube-system","gatekeeper-system"] + + [log_collection_settings.env_var] + # In the absense of this configmap, default value for enabled is true + enabled = true + [log_collection_settings.enrich_container_logs] + # In the absense of this configmap, default value for enrich_container_logs is false + enabled = false + # When this is enabled (enabled = true), every container log entry (both stdout & stderr) will be enriched with container Name & container Image + [log_collection_settings.collect_all_kube_events] + # In the absense of this configmap, default value for collect_all_kube_events is false + # When the setting is set to false, only the kube events with !normal event type will be collected + enabled = false + # When this is enabled (enabled = true), all kube events including normal events will be collected + [log_collection_settings.schema] + # In the absence of this configmap, default value for containerlog_schema_version is "v1" + # Supported values for this setting are "v1","v2" + # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema + containerlog_schema_version = "v2" + #[log_collection_settings.enable_multiline_logs] + # fluent-bit based multiline log collection for go (stacktrace), dotnet (stacktrace) + # if enabled will also stitch together container logs split by docker/cri due to size limits(16KB per log line) + # enabled = "false" + + + prometheus-data-collection-settings: |- + # Custom Prometheus metrics data collection settings + [prometheus_data_collection_settings.cluster] + # Cluster level scrape endpoint(s). These metrics will be scraped from agent's Replicaset (singleton) + # Any errors related to prometheus scraping can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to. + + #Interval specifying how often to scrape for metrics. This is duration of time and can be specified for supporting settings by combining an integer value and time unit as a string value. Valid time units are ns, us (or µs), ms, s, m, h. + interval = "1m" + + ## Uncomment the following settings with valid string arrays for prometheus scraping + #fieldpass = ["metric_to_pass1", "metric_to_pass12"] + + #fielddrop = ["metric_to_drop"] + + # An array of urls to scrape metrics from. + # urls = ["http://myurl:9101/metrics"] + + # An array of Kubernetes services to scrape metrics from. + # kubernetes_services = ["http://my-service-dns.my-namespace:9102/metrics"] + + # When monitor_kubernetes_pods = true, replicaset will scrape Kubernetes pods for the following prometheus annotations: + # - prometheus.io/scrape: Enable scraping for this pod + # - prometheus.io/scheme: Default is http + # - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation. + # - prometheus.io/port: If port is not 9102 use this annotation + monitor_kubernetes_pods = false + + ## Restricts Kubernetes monitoring to namespaces for pods that have annotations set and are scraped using the monitor_kubernetes_pods setting. + ## This will take effect when monitor_kubernetes_pods is set to true + ## ex: monitor_kubernetes_pods_namespaces = ["default1", "default2", "default3"] + # monitor_kubernetes_pods_namespaces = ["default1"] + + ## Label selector to target pods which have the specified label + ## This will take effect when monitor_kubernetes_pods is set to true + ## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + # kubernetes_label_selector = "env=dev,app=nginx" + + ## Field selector to target pods which have the specified field + ## This will take effect when monitor_kubernetes_pods is set to true + ## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/ + ## eg. To scrape pods on a specific node + # kubernetes_field_selector = "spec.nodeName=$HOSTNAME" + + [prometheus_data_collection_settings.node] + # Node level scrape endpoint(s). These metrics will be scraped from agent's DaemonSet running in every node in the cluster + # Any errors related to prometheus scraping can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to. + + #Interval specifying how often to scrape for metrics. This is duration of time and can be specified for supporting settings by combining an integer value and time unit as a string value. Valid time units are ns, us (or µs), ms, s, m, h. + interval = "1m" + + ## Uncomment the following settings with valid string arrays for prometheus scraping + + # An array of urls to scrape metrics from. $NODE_IP (all upper case) will substitute of running Node's IP address + # urls = ["http://$NODE_IP:9103/metrics"] + + #fieldpass = ["metric_to_pass1", "metric_to_pass12"] + + #fielddrop = ["metric_to_drop"] + + metric_collection_settings: |- + # Metrics collection settings for metrics sent to Log Analytics and MDM + [metric_collection_settings.collect_kube_system_pv_metrics] + # In the absense of this configmap, default value for collect_kube_system_pv_metrics is false + # When the setting is set to false, only the persistent volume metrics outside the kube-system namespace will be collected + enabled = false + # When this is enabled (enabled = true), persistent volume metrics including those in the kube-system namespace will be collected + + alertable-metrics-configuration-settings: |- + # Alertable metrics configuration settings for container resource utilization + [alertable_metrics_configuration_settings.container_resource_utilization_thresholds] + # The threshold(Type Float) will be rounded off to 2 decimal points + # Threshold for container cpu, metric will be sent only when cpu utilization exceeds or becomes equal to the following percentage + container_cpu_threshold_percentage = 95.0 + # Threshold for container memoryRss, metric will be sent only when memory rss exceeds or becomes equal to the following percentage + container_memory_rss_threshold_percentage = 95.0 + # Threshold for container memoryWorkingSet, metric will be sent only when memory working set exceeds or becomes equal to the following percentage + container_memory_working_set_threshold_percentage = 95.0 + + # Alertable metrics configuration settings for persistent volume utilization + [alertable_metrics_configuration_settings.pv_utilization_thresholds] + # Threshold for persistent volume usage bytes, metric will be sent only when persistent volume utilization exceeds or becomes equal to the following percentage + pv_usage_threshold_percentage = 60.0 + + # Alertable metrics configuration settings for completed jobs count + [alertable_metrics_configuration_settings.job_completion_threshold] + # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold + job_completion_threshold_time_minutes = 360 + integrations: |- + [integrations.azure_network_policy_manager] + collect_basic_metrics = false + collect_advanced_metrics = false + [integrations.azure_subnet_ip_usage] + enabled = false + +# Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md + agent-settings: |- + # prometheus scrape fluent bit settings for high scale + # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # settings scoped to prometheus sidecar container. all values in mb + [agent_settings.prometheus_fbit_settings] + tcp_listener_chunk_size = 10 + tcp_listener_buffer_size = 10 + tcp_listener_mem_buf_limit = 200 + + # prometheus scrape fluent bit settings for high scale + # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # settings scoped to daemonset container. all values in mb + # [agent_settings.node_prometheus_fbit_settings] + # tcp_listener_chunk_size = 1 + # tcp_listener_buffer_size = 1 + # tcp_listener_mem_buf_limit = 10 + + # prometheus scrape fluent bit settings for high scale + # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # settings scoped to replicaset container. all values in mb + # [agent_settings.cluster_prometheus_fbit_settings] + # tcp_listener_chunk_size = 1 + # tcp_listener_buffer_size = 1 + # tcp_listener_mem_buf_limit = 10 + + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. + # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage. + ## Ref for more details about Ignore_Older - https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail + # [agent_settings.fbit_config] + # log_flush_interval_secs = "1" # default value is 15 + # tail_mem_buf_limit_megabytes = "10" # default value is 10 + # tail_buf_chunksize_megabytes = "1" # default value is 32kb (comment out this line for default) + # tail_buf_maxsize_megabytes = "1" # default value is 32kb (comment out this line for default) + # tail_ignore_older = "5m" # default value same as fluent-bit default i.e.0m + + # On both AKS & Arc K8s enviornments, if Cluster has configured with Forward Proxy then Proxy settings automatically applied and used for the agent + # Certain configurations, proxy config should be ignored for example Cluster with AMPLS + Proxy + # in such scenarios, use the following config to ignore proxy settings + # [agent_settings.proxy_config] + # ignore_proxy_settings = "true" # if this is not applied, default value is false + + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. + # Configuration settings for the waittime for the network listeners to be available + # [agent_settings.network_listener_waittime] + # tcp_port_25226 = 45 # Port 25226 is used for telegraf to fluent-bit data in ReplicaSet + # tcp_port_25228 = 60 # Port 25228 is used for telegraf to fluentd data + # tcp_port_25229 = 45 # Port 25229 is used for telegraf to fluent-bit data in DaemonSet + + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. + # [agent_settings.mdsd_config] + # monitoring_max_event_rate = "50000" # default 20K eps + # backpressure_memory_threshold_in_mb = "1500" # default 3500MB + # upload_max_size_in_mb = "20" # default 2MB + # upload_frequency_seconds = "1" # default 60 upload_frequency_seconds + # compression_level = "0" # supported levels 0 to 9 and 0 means no compression + +metadata: + name: container-azm-ms-agentconfig + namespace: kube-system diff --git a/helm/examples/azure/log-analytics/dataCollectionSettings.json b/helm/examples/azure/log-analytics/dataCollectionSettings.json new file mode 100644 index 00000000000..4b6eb3bcc10 --- /dev/null +++ b/helm/examples/azure/log-analytics/dataCollectionSettings.json @@ -0,0 +1,7 @@ +{ + "interval": "1m", + "namespaceFilteringMode": "Include", + "namespaces": ["kube-system"], + "enableContainerLogV2": true, + "streams": ["Microsoft-Perf", "Microsoft-ContainerLogV2"] + } diff --git a/helm/examples/azure/log-analytics/enable-loganalytics.sh b/helm/examples/azure/log-analytics/enable-loganalytics.sh index 4ad63ecea92..1e47cf353b9 100755 --- a/helm/examples/azure/log-analytics/enable-loganalytics.sh +++ b/helm/examples/azure/log-analytics/enable-loganalytics.sh @@ -83,6 +83,10 @@ else fi echo "Enabling workspace on target AKS cluster '$AKS_CLUSTER_NAME'..." + +if $ENABLE_CONTAINER_LOG_V2 ; then DATA_COLLECTION_SETTINGS="--workspace-resource-id $wsid --data-collection-settings dataCollectionSettings.json"; fi + +echo "aks enable-addons -g $AKS_RESOURCE_GROUP -n $AKS_CLUSTER_NAME -a monitoring $DATA_COLLECTION_SETTINGS" az aks enable-addons -g $AKS_RESOURCE_GROUP -n $AKS_CLUSTER_NAME -a monitoring --workspace-resource-id $wsid if [[ $? -ne 0 ]] then @@ -91,3 +95,8 @@ then else echo "Success, workspace id: '$wsid' enabled on AKS $AKS_CLUSTER_NAME" fi + +if $ENABLE_CONTAINER_LOG_V2 ; then + echo "Setting ContainerLogV2 schema via container-azm-ms-agentconfig" + kubectl apply -f ${WORK_DIR}/container-azm-ms-agentconfig.yaml; +fi diff --git a/helm/examples/azure/log-analytics/env-loganalytics b/helm/examples/azure/log-analytics/env-loganalytics index b3e01320ea5..30ccfb8f344 100755 --- a/helm/examples/azure/log-analytics/env-loganalytics +++ b/helm/examples/azure/log-analytics/env-loganalytics @@ -21,4 +21,7 @@ AKS_RESOURCE_GROUP= AZURE_SUBSCRIPTION= # Azure resource location -AKS_RESOURCE_LOCATION=eastus \ No newline at end of file +AKS_RESOURCE_LOCATION=eastus + +# Enable enableContainerLogV2 +ENABLE_CONTAINER_LOG_V2=true diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml new file mode 100644 index 00000000000..db4bb0b2780 --- /dev/null +++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml @@ -0,0 +1,91 @@ +# Configures HPCC logAccess to target Azure Log Analytics Workspace +global: + logAccess: + name: "Azure LogAnalytics LogAccess" + type: "AzureLogAnalyticsCurl" + #connection: + #All connection attributes are optional. + #It is preferable to provide connection values as secret values category 'esp', secret name 'azure_logaccess' + # NOTE: secret 'azure_logaccess' must include 'aad-client-secret' and it cannot be provided in configuration + # + #workspaceID: "XYZ" #ID of the Azure LogAnalytics workspace to query logs from + # Secret value equivalent: 'ala-workspace-id' + #clientID: "DEF" #ID of Azure Active Directory registered application with api.loganalytics.io access - format: 00000000-0000-0000-0000-000000000000 + # Secret value equivalent: 'aad-client-id' + #tenantID: "ABC" #The Azure Active Directory Tenant ID, required for KQL API access + # Secret value equivalent: 'aad-tenant-id' + logMaps: + - type: "global" + storeName: "ContainerLogV2" + searchColumn: "LogMessage" + timeStampColumn: "hpcc_log_timestamp" + columnType: "dynamic" + columnMode: "ALL" + - type: "workunits" + searchColumn: "hpcc_log_jobid" + columnMode: "DEFAULT" + columnType: "string" + - type: "components" + storeName: "ContainerLogV2" + searchColumn: "ContainerName" # Container name happens to coincide with component name + keyColumn: "ContainerName" + columnMode: "DEFAULT" + columnType: "string" + - type: "audience" + searchColumn: "hpcc_log_audience" + enumValues: + - code: OPR + - code: USR + - code: PRO + - code: ADT + columnMode: "DEFAULT" + columnType: "enum" + - type: "class" + searchColumn: "hpcc_log_class" + enumValues: + - code: DIS + - code: ERR + - code: WRN + - code: INF + - code: PRO + - code: MET + columnMode: "DEFAULT" + columnType: "enum" + - type: "instance" + searchColumn: "PodName" + keyColumn: "PodName" + columnMode: "DEFAULT" + columnType: "string" + - type: "node" + columnMode: "DEFAULT" + searchColumn: "Computer" + columnMode: "ALL" + columnType: "string" + - type: "message" + searchColumn: "hpcc_log_message" + columnMode: "MIN" + columnType: "string" + - type: "logid" + searchColumn: "hpcc_log_sequence" + columnMode: "DEFAULT" + columnType: "numeric" + - type: "processid" + searchColumn: "hpcc_log_procid" + columnMode: "ALL" + columnType: "numeric" + - type: "threadid" + searchColumn: "hpcc_log_threadid" + columnMode: "DEFAULT" + columnType: "numeric" + - type: "timestamp" + searchColumn: "hpcc_log_timestamp" + columnMode: "MIN" + columnType: "datetime" +secrets: + esp: + azure-logaccess: "azure-logaccess" +vaults: + esp: + - name: my-azure-logaccess-vault + url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret} + kind: kv-v2 diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index 9aa987a51f8..667fa671e5e 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -46,7 +46,6 @@ static constexpr const char * logMapTimeStampColAtt = "@timeStampColumn"; static constexpr const char * logMapKeyColAtt = "@keyColumn"; static constexpr const char * logMapDisableJoinsAtt = "@disableJoins"; - static constexpr std::size_t defaultMaxRecordsPerFetch = 100; static size_t captureIncomingCURLReply(void* contents, size_t size, size_t nmemb, void* userp) @@ -344,7 +343,10 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce if (streq(logMapType, "global")) { if (logMap.hasProp(logMapIndexPatternAtt)) + { m_globalIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); + targetIsContainerLogV2 = strcmp("ContainerLogV2", m_globalIndexSearchPattern)==0; + } if (logMap.hasProp(logMapSearchColAtt)) m_globalSearchColName = logMap.queryProp(logMapSearchColAtt); if (logMap.hasProp(logMapTimeStampColAtt)) @@ -371,6 +373,15 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce m_componentsTimestampField = defaultHPCCLogComponentTSCol; m_disableComponentNameJoins = logMap.getPropBool(logMapDisableJoinsAtt, false); + if (targetIsContainerLogV2) + m_disableComponentNameJoins = true; //Don't attempt a join on ContainerLogV2 + else + { + if (strcmp("ContainerLogV2", m_componentsIndexSearchPattern)==0) + targetIsContainerLogV2 = true; + + m_disableComponentNameJoins = !m_disableComponentNameJoins && logMap.getPropBool(logMapDisableJoinsAtt, false); + } } else if (streq(logMapType, "class")) { @@ -392,6 +403,8 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce m_instanceIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); if (logMap.hasProp(logMapSearchColAtt)) m_instanceSearchColName = logMap.queryProp(logMapSearchColAtt); + if (logMap.hasProp(logMapKeyColAtt)) + m_instanceLookupKeyColumn = logMap.queryProp(logMapKeyColAtt); } else if (streq(logMapType, "node")) { @@ -420,26 +433,59 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce } } -void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, bool & includeComponentName) +void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, const bool includeComponentName) { columns.append("\n| project "); if (includeComponentName) - columns.appendf("%s, ", defaultHPCCLogComponentCol); + { + if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0) + { + columns.append(m_componentsSearchColName.str()); + if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str())) + columns.appendf("=%s", m_componentsLookupKeyColumn.str()); + } + else + columns.append(defaultHPCCLogComponentCol); + columns.append(", "); + } columns.appendf("%s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol); } -void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName) +void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName) { columns.append("\n| project "); + if (includeComponentName) - columns.appendf("%s, ", defaultHPCCLogComponentCol); + { + if (targetIsContainerLogV2 && m_componentsSearchColName.length() > 0) + { + columns.append(m_componentsSearchColName.str()); + if (m_componentsLookupKeyColumn.length() > 0 && !strsame(m_componentsSearchColName.str(), m_componentsLookupKeyColumn.str())) + columns.appendf("=%s", m_componentsLookupKeyColumn.str()); + } + else + { + columns.append(defaultHPCCLogComponentCol); + } + columns.append(", "); + } + + if (targetIsContainerLogV2) + { + columns.appendf("%s", m_instanceSearchColName.str()); + + if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str())) + columns.appendf("=%s, ", m_instanceLookupKeyColumn.str()); + else + columns.append(", "); + } columns.appendf("%s, %s, %s, %s, %s, %s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(), m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol); } -bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) +bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2) { if (isEmptyString(colName)) { @@ -447,7 +493,15 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) return false; } - kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", colName); + StringBuffer sourceCol; + if (targetsV2 && strcmp(colName, "LogMessage")==0) + sourceCol.set("tostring(LogMessage)"); + else if (!targetsV2 && strcmp(colName, "LogEntry")==0) + sourceCol.append(colName); + else + throw makeStringExceptionV(-1, "%s: Invalid Azure Log Analytics log message column name detected: '%s'. Review logAccess configuration.", COMPONENT_NAME, colName); + + kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s+\\\"(.*)\\\"$', %s)[0]", sourceCol.str()); kql.appendf("\n| extend %s = tostring(hpcclogfields.[0])", defaultHPCCLogSeqCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[1])", defaultHPCCLogAudCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[2])", defaultHPCCLogTypeCol); @@ -456,6 +510,13 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName) kql.appendf("\n| extend %s = toint(hpcclogfields.[5])", defaultHPCCLogThreadIDCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[6])", defaultHPCCLogJobIDCol); kql.appendf("\n| extend %s = tostring(hpcclogfields.[7])", defaultHPCCLogMessageCol); + kql.appendf("\n| project-away hpcclogfields, Type, TenantId, _ResourceId, %s, ", colName); + + if (targetsV2) + kql.append("LogSource, SourceSystem"); + else + kql.append("LogEntrySource, TimeOfCommand, SourceSystem"); + return true; } @@ -630,7 +691,10 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr if (m_instanceSearchColName.isEmpty()) throw makeStringExceptionV(-1, "%s: 'Instance' log entry field not configured", COMPONENT_NAME); - queryField = m_instanceSearchColName.str(); + if (m_instanceLookupKeyColumn.length()>0 && !strsame(m_instanceLookupKeyColumn.str(),m_instanceSearchColName.str())) + queryField = m_instanceLookupKeyColumn.str(); + else + queryField = m_instanceSearchColName.str(); if (!m_instanceIndexSearchPattern.isEmpty()) { @@ -708,13 +772,13 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr queryIndex.set(m_globalIndexSearchPattern.str()); StringBuffer searchColumns; - bool includeComponentName = !m_disableComponentNameJoins; + bool includeComponentName = !m_disableComponentNameJoins || targetIsContainerLogV2; searchMetaData(searchColumns, options.getReturnColsMode(), options.getLogFieldNames(), includeComponentName, options.getLimit(), options.getStartFrom()); - if (includeComponentName) + if (!m_disableComponentNameJoins && !targetIsContainerLogV2) declareContainerIndexJoinTable(queryString, options); queryString.append(queryIndex); - generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str()); + generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str(), targetIsContainerLogV2); if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter { @@ -730,7 +794,8 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr StringBuffer range; azureLogAnalyticsTimestampQueryRangeString(range, m_globalIndexTimestampField.str(), trange.getStartt().getSimple(),trange.getEndt().isNull() ? -1 : trange.getEndt().getSimple()); queryString.append("\n| where ").append(range.str()); - if (includeComponentName) + //if (includeComponentName) + if (!m_disableComponentNameJoins && !targetIsContainerLogV2) queryString.append("\n) on ").append(m_componentsLookupKeyColumn); queryString.append(searchColumns); diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp index f64af4817fd..b622db4d749 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp @@ -65,12 +65,14 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf StringBuffer m_aadClientSecret; StringBuffer m_componentsLookupKeyColumn; + StringBuffer m_instanceLookupKeyColumn; + bool targetIsContainerLogV2 = false; public: AzureLogAnalyticsCurlClient(IPropertyTree & logAccessPluginConfig); - void getMinReturnColumns(StringBuffer & columns, bool & includeComponentName); - void getDefaultReturnColumns(StringBuffer & columns, bool & includeComponentName); + void getMinReturnColumns(StringBuffer & columns, const bool includeComponentName); + void getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName); void searchMetaData(StringBuffer & search, const LogAccessReturnColsMode retcolmode, const StringArray & selectcols, bool & includeComponentName, unsigned size = defaultEntryLimit, offset_t from = defaultEntryStart); void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const LogAccessConditions & options); void populateKQLQueryString(StringBuffer & queryString, StringBuffer& queryIndex, const ILogAccessFilter * filter);