From b2fe35e109193fbcff21c11b01512dff019863ec Mon Sep 17 00:00:00 2001 From: Abhinav Vedmala Date: Wed, 31 Jul 2024 10:13:23 -0400 Subject: [PATCH] Moved tag to cloudformation param --- aws/logs_monitoring/README.md | 5 ++++- aws/logs_monitoring/settings.py | 6 +++++- aws/logs_monitoring/steps/enrichment.py | 25 ++----------------------- aws/logs_monitoring/template.yaml | 17 +++++++++++++++++ 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/aws/logs_monitoring/README.md b/aws/logs_monitoring/README.md index 32351ac5..0ef644f9 100644 --- a/aws/logs_monitoring/README.md +++ b/aws/logs_monitoring/README.md @@ -148,7 +148,7 @@ Starting version 3.107.0 a new feature is added to enable Lambda function to sto ### Upgrade an older version to +3.106.0 -Starting version 3.106.0 Lambda function has been updated to add a prefix to cache filenames stored in the S3 bucket configured in `DD_S3_BUCKET_NAME`. This allows to use the same bucket to store cache files from several functions. +Starting version 3.106.0 Lambda function has been updated to add a prefix to cache filenames stored in the S3 bucket configured in `DD_S3_BUCKET_NAME`. This allows to use the same bucket to store cache files from several functions. Additionally, starting this version, the forwarder will attach custom S3 bucket tags by default to all logs exported to S3. For example, if a service is configured to send logs to a destiantion S3 bucket, the forwarder will add the bucket's tags to the logs while pulling and forwarding the logs. ### Upgrade an older version to +3.99.0 @@ -388,6 +388,9 @@ SSL encrypted TCP connection, set this parameter to true. `DdForwardLog` : Set to false to disable log forwarding, while continuing to forward other observability data, such as metrics and traces from Lambda functions. +`DdStepFunctionTraceEnabled` +: Set to false to disable log to trace conversion for Step Functions, while continuing to forward logs. + `DdFetchLambdaTags` : Let the Forwarder fetch Lambda tags using GetResources API calls and apply them to logs, metrics, and traces. If set to true, permission `tag:GetResources` will be automatically added to the Lambda execution IAM role. diff --git a/aws/logs_monitoring/settings.py b/aws/logs_monitoring/settings.py index 10e2c202..5e567f79 100644 --- a/aws/logs_monitoring/settings.py +++ b/aws/logs_monitoring/settings.py @@ -58,6 +58,11 @@ def get_env_var(envvar, default, boolean=False): # DD_FORWARD_LOG = get_env_var("DD_FORWARD_LOG", "true", boolean=True) +## @param DD_STEP_FUNCTION_TRACE_ENABLED - boolean - optional - default: true +## Set this variable to `False` to disable log to trace conversion for Step Functions, while continuing to forward logs. +# +DD_STEP_FUNCTION_TRACE_ENABLED = get_env_var("DD_STEP_FUNCTION_TRACE_ENABLED", default="true", boolean=True) + ## @param DD_USE_TCP - boolean - optional -default: false ## Change this value to `true` to send your logs and metrics using the TCP network client ## By default, it uses the HTTP client. @@ -250,7 +255,6 @@ def __init__(self, name, pattern, placeholder): FORWARDERVERSION_STRING = "forwarder_version" GOV_STRING = "gov" CN_STRING = "cn" -DD_STEP_FUNCTION_TRACE_ENABLED = "dd_step_function_trace_enabled" # Additional target lambda invoked async with event data DD_ADDITIONAL_TARGET_LAMBDAS = get_env_var("DD_ADDITIONAL_TARGET_LAMBDAS", default=None) diff --git a/aws/logs_monitoring/steps/enrichment.py b/aws/logs_monitoring/steps/enrichment.py index f3356455..742e7cda 100644 --- a/aws/logs_monitoring/steps/enrichment.py +++ b/aws/logs_monitoring/steps/enrichment.py @@ -95,8 +95,8 @@ def add_metadata_to_lambda_log(event, cache_layer, context): tags += custom_lambda_tags # Set tracing behavior for all step functions - if context: - tags += get_step_function_tracing_tags(context) + if DD_STEP_FUNCTION_TRACE_ENABLED: + tags += ["dd_step_function_trace_enabled:true"] # Dedup tags, so we don't end up with functionname twice tags = list(set(tags)) @@ -105,27 +105,6 @@ def add_metadata_to_lambda_log(event, cache_layer, context): event[DD_CUSTOM_TAGS] = ",".join([event[DD_CUSTOM_TAGS]] + tags) -def get_step_function_tracing_tags(context): - """Fetch and add the `dd_step_function_trace_enabled` tag which can enable/disable tracing for step functions at the - forwarder level. - - Saves the tag as an environment variable to avoid fetching the tag everytime. - - Args: - context (LambdaContext): used to fetch the tag we need - """ - trace_enabled = os.environ.get(DD_STEP_FUNCTION_TRACE_ENABLED, "") - if not trace_enabled: - client = boto3.client("lambda") - response = client.list_tags(Resource=context.function_name) - lambda_tags = response.get("Tags", {}) - - trace_enabled = lambda_tags.get(DD_STEP_FUNCTION_TRACE_ENABLED, "false") - os.environ[DD_STEP_FUNCTION_TRACE_ENABLED] = trace_enabled - - return [f"{DD_STEP_FUNCTION_TRACE_ENABLED}:{trace_enabled}"] - - def get_enriched_lambda_log_tags(log_event, cache_layer): """Retrieves extra tags from lambda, either read from the function arn, or by fetching lambda tags from the function itself. diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml index 6736a7b6..aa7b5d00 100644 --- a/aws/logs_monitoring/template.yaml +++ b/aws/logs_monitoring/template.yaml @@ -161,6 +161,13 @@ Parameters: - true - false Description: Set to false to disable log forwarding, while continuing to forward other observability data, such as metrics and traces from Lambda functions. + DdStepFunctionTraceEnabled: + Type: String + Default: true + AllowedValues: + - true + - false + Description: Set to false to disable log to trace conversion for Step Functions, while continuing to forward logs. DdUseCompression: Type: String Default: true @@ -391,6 +398,10 @@ Conditions: Fn::Equals: - Ref: DdForwardLog - false + SetDdStepFunctionTraceEnabled: + Fn::Equals: + - Ref: DdStepFunctionTraceEnabled + - true SetDdUseCompression: Fn::Equals: - Ref: DdUseCompression @@ -615,6 +626,11 @@ Resources: - SetDdForwardLog - Ref: DdForwardLog - Ref: AWS::NoValue + DD_STEP_FUNCTION_TRACE_ENABLED: + Fn::If: + - SetDdStepFunctionTraceEnabled + - Ref: DdStepFunctionTraceEnabled + - Ref: AWS::NoValue DD_USE_COMPRESSION: Fn::If: - SetDdUseCompression @@ -1130,6 +1146,7 @@ Metadata: - DdCompressionLevel - DdMaxWorkers - DdForwardLog + - DdStepFunctionTraceEnabled - Label: default: Log Scrubbing (Optional) Parameters: