From a8c939bfbe4c18172d997b09a424c342584e8203 Mon Sep 17 00:00:00 2001 From: Akhil Dangore Date: Tue, 10 Sep 2024 11:41:32 +0530 Subject: [PATCH 1/3] Updated log-group-connector sam version to 1.0.14 --- aws-observability-terraform/source-module/main.tf | 4 ++-- aws-observability/apps/autoenable/auto_enable.template.yaml | 2 +- .../permissioncheck.nested.template.test.yaml | 2 +- .../permissionchecker/permissioncheck.nested.template.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws-observability-terraform/source-module/main.tf b/aws-observability-terraform/source-module/main.tf index c6912d52..1c5a1da5 100644 --- a/aws-observability-terraform/source-module/main.tf +++ b/aws-observability-terraform/source-module/main.tf @@ -230,7 +230,7 @@ module "cloudwatch_logs_lambda_log_forwarder_module" { } auto_enable_logs_subscription = var.auto_enable_logs_subscription - app_semantic_version = "1.0.12" + app_semantic_version = "1.0.14" auto_enable_logs_subscription_options = { filter = var.auto_enable_logs_subscription_options.filter } @@ -259,7 +259,7 @@ module "kinesis_firehose_for_logs_module" { } auto_enable_logs_subscription = var.auto_enable_logs_subscription - app_semantic_version = "1.0.12" + app_semantic_version = "1.0.14" auto_enable_logs_subscription_options = { filter = var.auto_enable_logs_subscription_options.filter } diff --git a/aws-observability/apps/autoenable/auto_enable.template.yaml b/aws-observability/apps/autoenable/auto_enable.template.yaml index 44eae706..688ae275 100644 --- a/aws-observability/apps/autoenable/auto_enable.template.yaml +++ b/aws-observability/apps/autoenable/auto_enable.template.yaml @@ -154,7 +154,7 @@ Resources: Properties: Location: ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-loggroup-connector - SemanticVersion: 1.0.12 + SemanticVersion: 1.0.14 Parameters: DestinationArnType: !Ref AutoSubscribeDestinationARNType DestinationArnValue: !Ref AutoSubscribeDestinationARN diff --git a/aws-observability/apps/permissionchecker/permissioncheck.nested.template.test.yaml b/aws-observability/apps/permissionchecker/permissioncheck.nested.template.test.yaml index 0c01739e..7593b1db 100644 --- a/aws-observability/apps/permissionchecker/permissioncheck.nested.template.test.yaml +++ b/aws-observability/apps/permissionchecker/permissioncheck.nested.template.test.yaml @@ -663,7 +663,7 @@ Resources: Properties: Location: ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-loggroup-connector - SemanticVersion: 1.0.12 + SemanticVersion: 1.0.14 Parameters: DestinationArnType: "Lambda" DestinationArnValue: !GetAtt CloudWatchEventFunction.Outputs.SumoCWLogsLambdaArn diff --git a/aws-observability/apps/permissionchecker/permissioncheck.nested.template.yaml b/aws-observability/apps/permissionchecker/permissioncheck.nested.template.yaml index 0682c4d1..a2d9af35 100644 --- a/aws-observability/apps/permissionchecker/permissioncheck.nested.template.yaml +++ b/aws-observability/apps/permissionchecker/permissioncheck.nested.template.yaml @@ -671,7 +671,7 @@ Resources: Properties: Location: ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-loggroup-connector - SemanticVersion: 1.0.12 + SemanticVersion: 1.0.14 Parameters: DestinationArnType: "Lambda" DestinationArnValue: !GetAtt CloudWatchEventFunction.Outputs.SumoCWLogsLambdaArn From 0bbd489daceb04a7f1c5cc6753134bd685ce9088 Mon Sep 17 00:00:00 2001 From: Akhil Dangore Date: Tue, 10 Sep 2024 16:48:00 +0530 Subject: [PATCH 2/3] Added support for AutoSubscribeLogGroupByTags --- .../apps/autoenable/auto_enable.template.yaml | 5 ++++ ...mologic_observability.master.template.yaml | 26 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/aws-observability/apps/autoenable/auto_enable.template.yaml b/aws-observability/apps/autoenable/auto_enable.template.yaml index 688ae275..c19bc4ea 100644 --- a/aws-observability/apps/autoenable/auto_enable.template.yaml +++ b/aws-observability/apps/autoenable/auto_enable.template.yaml @@ -70,6 +70,10 @@ Parameters: Type: String Default: '\/aws\/(lambda|apigateway|rds)' Description: "Enter regex for matching logGroups. Regex will check for the name. Visit https://help.sumologic.com/docs/send-data/collect-from-other-data-sources/autosubscribe-arn-destination/#configuringparameters" + AutoSubscribeLogGroupByTags: + Type: String + Default: "" + Description: "Enter comma separated key value pairs for filtering logGroups using tags. Ex KeyName1=string,KeyName2=string. This is optional leave it blank if tag based filtering is not needed. Visit https://help.sumologic.com/docs/send-data/collect-from-other-data-sources/autosubscribe-arn-destination/#configuringparameters" AutoSubscribeRoleArn: Type: String Default: "" @@ -159,5 +163,6 @@ Resources: DestinationArnType: !Ref AutoSubscribeDestinationARNType DestinationArnValue: !Ref AutoSubscribeDestinationARN LogGroupPattern: !Ref AutoSubscribeLambdaLogGroupPattern + LogGroupTags: !Ref AutoSubscribeLogGroupByTags UseExistingLogs: !If [auto_subscribe_exisitng_log_groups, "true", "false"] RoleArn: !Ref AutoSubscribeRoleArn \ No newline at end of file diff --git a/aws-observability/templates/sumologic_observability.master.template.yaml b/aws-observability/templates/sumologic_observability.master.template.yaml index 04603341..5d464f5f 100644 --- a/aws-observability/templates/sumologic_observability.master.template.yaml +++ b/aws-observability/templates/sumologic_observability.master.template.yaml @@ -56,6 +56,7 @@ Metadata: - Section7bLambdaCloudWatchLogsSourceUrl - Section7cAutoSubscribeLogGroupsOptions - Section7dAutoSubscribeLogGroupPattern + - Section7eAutoSubscribeLogGroupByTags - Label: default: "8. Sumo Logic Root Cause Explorer Sources" @@ -134,6 +135,9 @@ Metadata: default: "Subscribe log groups to Destination (Lambda or kinesis firehose delivery stream)" Section7dAutoSubscribeLogGroupPattern: default: "Regex for filtering CloudWatch Log Groups. If you want to collect from all CloudWatch Log Groups use .*" + Section7eAutoSubscribeLogGroupByTags: + default: "Tags for filtering CloudWatch Log Groups." + Section8aRootCauseExplorerOptions: default: "Select the Sumo Logic Root Cause Explorer Sources" @@ -324,6 +328,10 @@ Parameters: Type: String Default: '\/aws\/(lambda|apigateway|rds)' Description: "Enter regex for matching CloudWatch Log groups name. Regex will check for the Log group name. Visit https://help.sumologic.com/03Send-Data/Collect-from-Other-Data-Sources/Auto-Subscribe_AWS_Log_Groups_to_a_Lambda_Function#Configuring_parameters" + Section7eAutoSubscribeLogGroupByTags: + Type: String + Default: "" + Description: "Enter comma separated key value pairs for filtering logGroups using tags. Ex KeyName1=string,KeyName2=string. This is optional leave it blank if tag based filtering is not needed. Visit https://help.sumologic.com/docs/send-data/collect-from-other-data-sources/autosubscribe-arn-destination/#configuringparameters" Section8aRootCauseExplorerOptions: Type: String @@ -475,7 +483,7 @@ Mappings: CommonData: NestedTemplate: BucketName: "sumologic-appdev-aws-sam-apps" - Version: "v2.10.0" + Version: "v2.11.0" CollectorDetails: SumoLogicAccountID: 926226587429 CollectorNamePrefix: "aws-observability" @@ -627,7 +635,7 @@ Resources: scanInterval: 60 ToUpdate: Fn::Base64: !Sub - - "${a}-${b}-${c}-${d}-${e}-${f}-${g}-${h}-${i}-${j}-${k}-${l}-${m}-${n}-${o}-${p}-${q}-${r}-${s}-${t}-${u}-${v}" + - "${a}-${b}-${c}-${d}-${e}-${f}-${g}-${h}-${i}-${j}-${k}-${l}-${m}-${n}-${o}-${p}-${q}-${r}-${s}-${t}-${u}-${v}-${w}" - a: !Ref Section2aAccountAlias b: !Ref Section3aInstallObservabilityApps c: !Ref Section4aCreateMetricsSourceOptions @@ -644,12 +652,13 @@ Resources: n: !Ref Section7bLambdaCloudWatchLogsSourceUrl o: !Ref Section7cAutoSubscribeLogGroupsOptions p: !Ref Section7dAutoSubscribeLogGroupPattern - q: !Ref Section9aAutoEnableS3LogsELBResourcesOptions - r: !Ref Section9bELBCreateLogSource - s: !Ref Section9cELBLogsSourceUrl - t: !Ref Section9dELBS3LogsBucketName - u: !Ref Section10aAppInstallLocation - v: !FindInMap [CommonData, NestedTemplate, Version] + q: !Ref Section7eAutoSubscribeLogGroupByTags + r: !Ref Section9aAutoEnableS3LogsELBResourcesOptions + s: !Ref Section9bELBCreateLogSource + t: !Ref Section9cELBLogsSourceUrl + u: !Ref Section9dELBS3LogsBucketName + v: !Ref Section10aAppInstallLocation + w: !FindInMap [CommonData, NestedTemplate, Version] CreateCommonResources: Type: AWS::CloudFormation::Stack @@ -729,6 +738,7 @@ Resources: AutoSubscribeDestinationARNType: !If [create_kf_logs_source, "Kinesis", "Lambda" ] AutoSubscribeDestinationARN: !If [create_kf_logs_source, !GetAtt CreateCommonResources.Outputs.KinesisLogsDeliveryStreamARN, !If [ create_cw_logs_source, !GetAtt CreateCommonResources.Outputs.CloudWatchLambdaARN, "" ] ] AutoSubscribeLambdaLogGroupPattern: !Ref Section7dAutoSubscribeLogGroupPattern + AutoSubscribeLogGroupByTags: !Ref Section7eAutoSubscribeLogGroupByTags AutoSubscribeRoleArn: !If [create_kf_logs_source, !GetAtt CreateCommonResources.Outputs.KinesisLogsRoleARN, "" ] ELBAutoEnableS3Logs: !If [auto_enable_s3_logs_elb, "Yes", "No"] ELBAutoEnableS3LogsOptions: !Ref Section9aAutoEnableS3LogsELBResourcesOptions From a2bb09e2efd92721190afda8b9c1da1c146b466d Mon Sep 17 00:00:00 2001 From: Akhil Dangore Date: Fri, 20 Sep 2024 10:26:32 +0530 Subject: [PATCH 3/3] Addded support for tags_filter in TF --- aws-observability-terraform/source-module/README.md | 2 +- aws-observability-terraform/source-module/main.tf | 2 ++ aws-observability-terraform/source-module/variables.tf | 10 ++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/aws-observability-terraform/source-module/README.md b/aws-observability-terraform/source-module/README.md index 46fba312..cf742ee8 100644 --- a/aws-observability-terraform/source-module/README.md +++ b/aws-observability-terraform/source-module/README.md @@ -71,7 +71,7 @@ | [auto\_enable\_access\_logs](#input\_auto\_enable\_access\_logs) | Enable Application Load Balancer (ALB) Access logging.
You have the following options:
New - Automatically enables access logging for newly created ALB resources to collect logs for ALB resources. This does not affect ALB resources already collecting logs.
Existing - Automatically enables access logging for existing ALB resources to collect logs for ALB resources.
Both - Automatically enables access logging for new and existing ALB resources.
None - Skips Automatic access Logging enable for ALB resources. | `string` | `"Both"` | no | | [auto\_enable\_classic\_lb\_access\_logs](#input\_auto\_enable\_classic\_lb\_access\_logs) | Enable Classic Load Balancer (CLB) Access logging.
You have the following options:
New - Automatically enables access logging for newly created CLB resources to collect logs for CLB resources. This does not affect CLB resources already collecting logs.
Existing - Automatically enables access logging for existing CLB resources to collect logs for CLB resources.
Both - Automatically enables access logging for new and existing CLB resources.
None - Skips Automatic access Logging enable for CLB resources. | `string` | `"Both"` | no | | [auto\_enable\_logs\_subscription](#input\_auto\_enable\_logs\_subscription) | Subscribe log groups to Sumo Logic Lambda Forwarder.
You have the following options:
New - Automatically subscribes new log groups to send logs to Sumo Logic.
Existing - Automatically subscribes existing log groups to send logs to Sumo Logic.
Both - Automatically subscribes new and existing log groups.
None - Skips Automatic subscription. | `string` | `"Both"` | no | -| [auto\_enable\_logs\_subscription\_options](#input\_auto\_enable\_logs\_subscription\_options) | filter - Enter regex for matching CloudWatch Log groups name. Regex will check for the Log groups name. Visit https://help.sumologic.com/03Send-Data/Collect-from-Other-Data-Sources/Auto-Subscribe_AWS_Log_Groups_to_a_Lambda_Function#Configuring_parameters | 
object({
    filter = string
  })
| 
{
  "filter": "apigateway|lambda|rds"
}
| no | +| [auto\_enable\_logs\_subscription\_options](#input\_auto\_enable\_logs\_subscription\_options) | filter - Enter regex for matching logGroups. Regex will check for the name.
tags_filter - Enter comma separated key value pairs for filtering logGroups using tags. Ex KeyName1=string,KeyName2=string. This is optional leave it blank if tag based filtering is not needed.
Visit https://help.sumologic.com/docs/send-data/collect-from-other-data-sources/autosubscribe-arn-destination/#configuringparameters | 
object({
    filter = string 
    tags_filter = string 
 })
| 
{
  "filter": "lambda" 
  "tags_filter" = "" 
 }
| no | | [aws\_account\_alias](#input\_aws\_account\_alias) | Provide the Name/Alias for the AWS environment from which you are collecting data. This name will appear in the Sumo Logic Explorer View, metrics, and logs.
Please leave this blank if you are going to deploy the solution in multiple AWS accounts.
Do not include special characters in the alias. | `string` | n/a | yes | | [classic\_lb\_log\_source\_url](#input\_classic\_lb\_log\_source\_url) | Required if you are already collecting Classic LB logs. Provide the existing Sumo Logic Classic LB Source API URL. The account, accountid, and region fields will be added to the Source. For information on how to determine the URL, see [View or Download Source JSON Configuration](https://help.sumologic.com/03Send-Data/Sources/03Use-JSON-to-Configure-Sources/Local-Configuration-File-Management/View-or-Download-Source-JSON-Configuration). | `string` | `""` | no | | [classic\_lb\_source\_details](#input\_classic\_lb\_source\_details) | Provide details for the Sumo Logic Classic Load Balancer source. If not provided, then defaults will be used.
To enable collection of classic load balancer logs, set collect\_classic\_lb\_logs to true and provide configuration information for the bucket.
If create\_bucket is false, provide a name of an existing S3 bucket where you would like to store loadbalancer logs. If this is empty, a new bucket will be created in the region.
If create\_bucket is true, the script creates a bucket, the name of the bucket has to be unique; this is achieved internally by generating a random-id and then post-fixing it to the “aws-observability-” string.
path\_expression - This is required in case the above existing bucket is already configured to receive Classic LB access logs. If this is blank, Sumo Logic will store logs in the path expression: *classicloadbalancing/AWSLogs/*/elasticloadbalancing/*/* | 
object({
    source_name     = string
    source_category = string
    description     = string
    bucket_details = object({
      create_bucket        = bool
      bucket_name          = string
      path_expression      = string
      force_destroy_bucket = bool
    })
    fields = map(string)
  })
| 
{
  "bucket_details": {
    "bucket_name": "aws-observability-random-id",
    "create_bucket": true,
    "force_destroy_bucket": true,
    "path_expression": "*classicloadbalancing/AWSLogs//elasticloadbalancing//*"
  },
  "description": "This source is created using Sumo Logic terraform AWS Observability module to collect AWS Classic LoadBalancer logs.",
  "fields": {},
  "source_category": "aws/observability/clb/logs",
  "source_name": "Classic lb Logs (Region)"
}
| no | diff --git a/aws-observability-terraform/source-module/main.tf b/aws-observability-terraform/source-module/main.tf index 1c5a1da5..2f69a56d 100644 --- a/aws-observability-terraform/source-module/main.tf +++ b/aws-observability-terraform/source-module/main.tf @@ -233,6 +233,7 @@ module "cloudwatch_logs_lambda_log_forwarder_module" { app_semantic_version = "1.0.14" auto_enable_logs_subscription_options = { filter = var.auto_enable_logs_subscription_options.filter + tags_filter = var.auto_enable_logs_subscription_options.tags_filter } } @@ -262,6 +263,7 @@ module "kinesis_firehose_for_logs_module" { app_semantic_version = "1.0.14" auto_enable_logs_subscription_options = { filter = var.auto_enable_logs_subscription_options.filter + tags_filter = var.auto_enable_logs_subscription_options.tags_filter } } diff --git a/aws-observability-terraform/source-module/variables.tf b/aws-observability-terraform/source-module/variables.tf index 45dd2f08..d082e086 100644 --- a/aws-observability-terraform/source-module/variables.tf +++ b/aws-observability-terraform/source-module/variables.tf @@ -499,12 +499,18 @@ variable "auto_enable_logs_subscription" { variable "auto_enable_logs_subscription_options" { type = object({ filter = string + tags_filter = string }) + description = <