Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update and add callback alarms #1507

Merged
merged 5 commits into from
Sep 3, 2024
Merged

Update and add callback alarms #1507

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6136137
Update and add callback alarms
whabanks Aug 29, 2024
db2f54b
Adjust query name
whabanks Aug 29, 2024
0b59950
typo
whabanks Sep 3, 2024
4a4cdea
Merge branch 'main' into callback-alarms
whabanks Sep 3, 2024
e811acb
Merge branch 'main' into callback-alarms
whabanks Sep 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions aws/eks/cloudwatch_alarms.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -958,14 +958,29 @@ resource "aws_cloudwatch_metric_alarm" "github-arc-runner-error-alarm" {
resource "aws_cloudwatch_metric_alarm" "service-callback-too-many-failures-warning" {
count = var.cloudwatch_enabled ? 1 : 0
alarm_name = "service-callback-too-many-failures-warning"
alarm_description = "Service reached the max number of callback retries 5 times in 30 minutes"
alarm_description = "Service reached the max number of callback retries 25 times in 5 minutes"
comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "1"
metric_name = aws_cloudwatch_log_metric_filter.callback-max-retry-failures[0].metric_transformation[0].name
namespace = aws_cloudwatch_log_metric_filter.callback-max-retry-failures[0].metric_transformation[0].namespace
period = 60 * 30
metric_name = aws_cloudwatch_log_metric_filter.callback-request-failures[0].metric_transformation[0].name
namespace = aws_cloudwatch_log_metric_filter.callback-request-failures[0].metric_transformation[0].namespace
period = 60 * 5
statistic = "Sum"
threshold = 5
threshold = 25
treat_missing_data = "notBreaching"
alarm_actions = [var.sns_alert_warning_arn]
}

resource "aws_cloudwatch_metric_alarm" "service-callback-too-many-failures-critical" {
count = var.cloudwatch_enabled ? 1 : 0
alarm_name = "service-callback-too-many-failures-warning"
alarm_description = "Service reached the max number of callback retries 100 times in 10 minutes"
comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "1"
metric_name = aws_cloudwatch_log_metric_filter.callback-request-failures[0].metric_transformation[0].name
namespace = aws_cloudwatch_log_metric_filter.callback-request-failures[0].metric_transformation[0].namespace
period = 60 * 10
statistic = "Sum"
threshold = 100
treat_missing_data = "notBreaching"
alarm_actions = [var.sns_alert_critical_arn]
}
6 changes: 3 additions & 3 deletions aws/eks/cloudwatch_log.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,10 @@ resource "aws_cloudwatch_log_metric_filter" "github-arc-runner-alarm" {
}
}

resource "aws_cloudwatch_log_metric_filter" "callback-max-retry-failures" {
resource "aws_cloudwatch_log_metric_filter" "callback-request-failures" {
count = var.cloudwatch_enabled ? 1 : 0
name = "callback-max-retry-failures"
pattern = "send_delivery_status_to_service has retried the max num of times for callback url"
name = "allback-request-failures"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

name = "callback-request-failures"

pattern = "send_delivery_status_to_service request failed for notification_id:"
log_group_name = aws_cloudwatch_log_group.notification-canada-ca-eks-application-logs[0].name

metric_transformation {
Expand Down
21 changes: 19 additions & 2 deletions aws/eks/cloudwatch_queries.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,9 +272,9 @@ fields @timestamp, @service_id, @bounce_type
QUERY
}

resource "aws_cloudwatch_query_definition" "callback-failures-by-service" {
resource "aws_cloudwatch_query_definition" "callback-max-retry-failures-by-service" {
count = var.cloudwatch_enabled ? 1 : 0
name = "Callbacks / Count of callbacks that exceeded MaxRetries by service"
name = "Callbacks / Callbacks that exceeded MaxRetries by service"

log_group_names = [
local.eks_application_log_group
Expand All @@ -291,6 +291,23 @@ fields @timestamp, @service_id, @callback_url, @notification_id
QUERY
}

resource "aws_cloudwatch_query_definition" "callback-failures" {
count = var.cloudwatch_enabled ? 1 : 0
name = "Callbacks / Callback errors by notification_id"

log_group_names = [
local.eks_application_log_group
]

query_string = <<QUERY
fields @timestamp, @notification_id, @url, @error
| filter kubernetes.container_name like /^celery/
| filter @message like /send_delivery_status_to_service request failed for notification_id:/
| parse @message 'send_delivery_status_to_service request failed for notification_id: * and url: * exc: *' as @notification_id, @url, @error
| limit 10000
QUERY
}

resource "aws_cloudwatch_query_definition" "gh-arc-errors" {
count = var.cloudwatch_enabled ? 1 : 0
name = "CICD / GitHub ARC Errors"
Expand Down
Loading