Skip to content

Commit

Permalink
feat: Include status code and body in alerts, style Slack
Browse files Browse the repository at this point in the history
  • Loading branch information
stevensdavid committed Jun 26, 2024
1 parent 5adfe1a commit c99ce1c
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 15 deletions.
19 changes: 19 additions & 0 deletions src/alerts/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,28 @@ pub struct WebhookNotification {
pub error_message: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub trace_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub status_code: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub body: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlackNotification {
pub blocks: Vec<SlackBlock>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlackBlock {
pub r#type: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub elements: Option<Vec<SlackTextBlock>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub text: Option<SlackTextBlock>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SlackTextBlock {
pub r#type: String,
pub text: String,
}
88 changes: 75 additions & 13 deletions src/alerts/outbound_webhook.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
use std::time::Duration;

use crate::alerts::model::WebhookNotification;
use crate::errors::MapToSendError;
use crate::probe::model::ProbeAlert;
use crate::{alerts::model::WebhookNotification, probe::model::ProbeResponse};
use chrono::{DateTime, Utc};
use lazy_static::lazy_static;
use tracing::{info, warn};

use super::model::SlackNotification;
use super::model::{SlackBlock, SlackNotification, SlackTextBlock};

const REQUEST_TIMEOUT_SECS: u64 = 10;

Expand All @@ -21,6 +21,7 @@ lazy_static! {
pub async fn alert_if_failure(
success: bool,
error: Option<&str>,
probe_response: Option<&ProbeResponse>,
probe_name: &str,
failure_timestamp: DateTime<Utc>,
alerts: &Option<Vec<ProbeAlert>>,
Expand All @@ -30,16 +31,20 @@ pub async fn alert_if_failure(
return Ok(());
}
let error_message = error.unwrap_or("No error message");
let truncated_body = probe_response.map(|r| r.truncated_body(500));
warn!(
"Probe {probe_name} failed at {failure_timestamp} with trace ID {}. Error: {error_message}",
trace_id.as_ref().unwrap_or(&"N/A".to_owned())
"Probe {probe_name} failed at {failure_timestamp} with trace ID {}. Status code: {}. Error: {error_message}. Body: {}",
trace_id.as_ref().unwrap_or(&"N/A".to_owned()),
probe_response.map_or("N/A".to_owned(), |r| r.status_code.to_string()),
truncated_body.unwrap_or("N/A".to_owned()),
);
let mut errors = Vec::new();
if let Some(alerts_vec) = alerts {
for alert in alerts_vec {
if let Err(e) = send_alert(
alert,
probe_name.to_owned(),
probe_response,
error_message,
failure_timestamp,
trace_id.clone(),
Expand Down Expand Up @@ -81,6 +86,7 @@ pub async fn send_generic_webhook(
pub async fn send_webhook_alert(
url: &String,
probe_name: String,
probe_response: Option<&ProbeResponse>,
error_message: &str,
failure_timestamp: DateTime<Utc>,
trace_id: Option<String>,
Expand All @@ -91,6 +97,8 @@ pub async fn send_webhook_alert(
error_message: error_message.to_owned(),
failure_timestamp,
trace_id,
body: probe_response.map(|r| r.truncated_body(500)),
status_code: probe_response.map(|r| r.status_code),
};

let json = serde_json::to_string(&request_body).map_to_send_err()?;
Expand All @@ -100,26 +108,77 @@ pub async fn send_webhook_alert(
pub async fn send_slack_alert(
webhook_url: &String,
probe_name: String,
probe_response: Option<&ProbeResponse>,
error_message: &str,
failure_timestamp: DateTime<Utc>,
trace_id: Option<String>,
) -> Result<(), Box<dyn std::error::Error + Send>> {
let request_body = SlackNotification {
text: format!(
"Probe {} failed at {}. Trace ID: {}. Error: {}",
probe_name,
failure_timestamp,
trace_id.unwrap_or_else(|| "N/A".to_owned()),
error_message,
),
};
// Uses Slack's Block Kit UI to make the message prettier
let mut blocks = vec![
SlackBlock {
r#type: "header".to_owned(),
text: Some(SlackTextBlock {
r#type: "plain_text".to_owned(),
text: format!("\"{}\" failed.", probe_name),
}),
elements: None,
},
SlackBlock {
r#type: "section".to_owned(),
text: Some(SlackTextBlock {
r#type: "mrkdwn".to_owned(),
text: format!("Error message:\n\n> {}", error_message),
}),
elements: None,
},
];

if let Some(response) = probe_response {
blocks.extend([
SlackBlock {
r#type: "divider".to_owned(),
elements: None,
text: None,
},
SlackBlock {
r#type: "section".to_owned(),
elements: None,
text: Some(SlackTextBlock {
r#type: "mrkdwn".to_owned(),
text: format!(
"Received status code *{}* and (truncated) body:\n```\n{}\n```",
response.status_code,
response.body.chars().take(500).collect::<String>()
),
}),
},
])
}

blocks.push(SlackBlock {
r#type: "context".to_owned(),
elements: Some(vec![
SlackTextBlock {
r#type: "mrkdwn".to_owned(),
text: format!("Time: *{}*", failure_timestamp),
},
SlackTextBlock {
r#type: "mrkdwn".to_owned(),
text: format!("Trace ID: *{}*", trace_id.unwrap_or("N/A".to_owned())),
},
]),
text: None,
});
let request_body = SlackNotification { blocks };
let json = serde_json::to_string(&request_body).map_to_send_err()?;
println!("{}", json);
send_generic_webhook(webhook_url, json).await
}

pub async fn send_alert(
alert: &ProbeAlert,
probe_name: String,
probe_response: Option<&ProbeResponse>,
error_message: &str,
failure_timestamp: DateTime<Utc>,
trace_id: Option<String>,
Expand All @@ -130,6 +189,7 @@ pub async fn send_alert(
send_slack_alert(
&alert.url,
probe_name.clone(),
probe_response,
error_message,
failure_timestamp,
trace_id.clone(),
Expand All @@ -140,6 +200,7 @@ pub async fn send_alert(
send_webhook_alert(
&alert.url,
probe_name.clone(),
probe_response,
error_message,
failure_timestamp,
trace_id.clone(),
Expand Down Expand Up @@ -181,6 +242,7 @@ mod webhook_tests {
let alert_result = alert_if_failure(
false,
Some("Test error"),
None,
&probe_name,
failure_timestamp,
&alerts,
Expand Down
4 changes: 2 additions & 2 deletions src/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ impl std::fmt::Display for ExpectationFailedError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
"Failed to meet expectation for field '{:?}' with operation {:?} {:?}. Received: status '{}', body '{}' (truncatated to 100 characters).",
self.field, self.operation, self.expected, self.status_code, self.body.chars().take(100).collect::<String>()
"Failed to meet expectation for field '{:?}' with operation {:?} {:?}.",
self.field, self.operation, self.expected,
)
}
}
Expand Down
6 changes: 6 additions & 0 deletions src/probe/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ pub struct ProbeResponse {
pub body: String,
}

impl ProbeResponse {
pub fn truncated_body(&self, n: usize) -> String {
self.body.chars().take(n).collect()
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Story {
pub name: String,
Expand Down
2 changes: 2 additions & 0 deletions src/probe/probe_logic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ impl Monitorable for Story {
let send_alert_result = alert_if_failure(
story_success,
last_step.error_message.as_deref(),
last_step.response.as_ref(),
&self.name,
timestamp_started,
&self.alerts,
Expand Down Expand Up @@ -270,6 +271,7 @@ impl Monitorable for Probe {
let send_alert_result = alert_if_failure(
probe_result.success,
probe_result.error_message.as_deref(),
probe_result.response.as_ref(),
&self.name,
timestamp,
&self.alerts,
Expand Down

0 comments on commit c99ce1c

Please sign in to comment.