Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allow for a signal to be triggered on flush #35

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

allow for a signal to be triggered on flush #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
962338a
implement batch signal
ogrman Sep 19, 2022
f69b4cb
test flush_signal
ogrman Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ serde_json = "1"
tokio = { version = "1", features = ["sync", "time"] }

[dev-dependencies]
env_logger = "*"
proptest = "1"
async-trait = "0.1.24"
criterion = { version = "0.3", features = ["async_tokio"] }
tokio = { version = "1", features = ["test-util"] }
tokio-stream = "0.1.9"

[features]
default = ["native-tls"]
Expand Down
5 changes: 3 additions & 2 deletions src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::{collections::BTreeMap, fmt, pin::Pin};

use futures_util::{future, FutureExt, Stream};
use rusoto_cloudwatch::CloudWatch;
use tokio::sync::oneshot;

use crate::{
collector::{self, Config, Resolution},
Expand All @@ -17,7 +18,7 @@ pub struct Builder {
client: Box<dyn CloudWatch + Send + Sync>,
shutdown_signal: Option<BoxFuture<'static, ()>>,
metric_buffer_size: usize,
force_flush_stream: Option<Pin<Box<dyn Stream<Item = ()> + Send>>>,
force_flush_stream: Option<Pin<Box<dyn Stream<Item = Option<oneshot::Sender<()>>> + Send>>>,
}

pub fn builder(region: rusoto_core::Region) -> Builder {
Expand Down Expand Up @@ -56,7 +57,7 @@ impl Builder {
/// held metric data in the same way as shutdown_signal will.
pub fn force_flush_stream(
mut self,
force_flush_stream: Pin<Box<dyn Stream<Item = ()> + Send>>,
force_flush_stream: Pin<Box<dyn Stream<Item = Option<oneshot::Sender<()>>> + Send>>,
) -> Self {
self.force_flush_stream = Some(force_flush_stream);
self
Expand Down
47 changes: 30 additions & 17 deletions src/collector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use {
metrics::{GaugeValue, Key, Recorder, Unit},
rusoto_cloudwatch::{CloudWatch, Dimension, MetricDatum, PutMetricDataInput, StatisticSet},
rusoto_core::Region,
tokio::sync::mpsc,
tokio::sync::{mpsc, oneshot},
};

use crate::{error::Error, BoxFuture};
Expand All @@ -42,7 +42,7 @@ pub struct Config {
pub client: Box<dyn CloudWatch + Send + Sync>,
pub shutdown_signal: future::Shared<BoxFuture<'static, ()>>,
pub metric_buffer_size: usize,
pub force_flush_stream: Option<Pin<Box<dyn Stream<Item = ()> + Send>>>,
pub force_flush_stream: Option<Pin<Box<dyn Stream<Item = Option<oneshot::Sender<()>>> + Send>>>,
}

struct CollectorConfig {
Expand All @@ -60,7 +60,8 @@ enum Message {
Datum(Datum),
SendBatch {
send_all_before: Timestamp,
emit_sender: mpsc::Sender<Vec<MetricDatum>>,
emit_sender: mpsc::Sender<(Vec<MetricDatum>, Option<oneshot::Sender<()>>)>,
flush_signal: Option<oneshot::Sender<()>>,
},
}

Expand Down Expand Up @@ -164,13 +165,14 @@ pub fn new(mut config: Config) -> (RecorderHandle, impl Future<Output = ()>) {
.force_flush_stream
.take()
.unwrap_or_else(|| {
Box::pin(futures_util::stream::empty::<()>()) as Pin<Box<dyn Stream<Item = ()> + Send>>
Box::pin(futures_util::stream::empty::<Option<oneshot::Sender<()>>>()) as Pin<Box<dyn Stream<Item = Option<oneshot::Sender<()>>> + Send>>
})
.map({
let emit_sender = emit_sender.clone();
move |()| Message::SendBatch {
move |signal| Message::SendBatch {
send_all_before: std::u64::MAX,
emit_sender: emit_sender.clone(),
flush_signal: signal,
}
});

Expand Down Expand Up @@ -200,6 +202,7 @@ pub fn new(mut config: Config) -> (RecorderHandle, impl Future<Output = ()>) {
.accept(Message::SendBatch {
send_all_before: std::u64::MAX,
emit_sender,
flush_signal: None,
})
.await;
};
Expand Down Expand Up @@ -232,13 +235,13 @@ where
}

async fn mk_emitter(
mut emit_receiver: mpsc::Receiver<Vec<MetricDatum>>,
mut emit_receiver: mpsc::Receiver<(Vec<MetricDatum>, Option<oneshot::Sender<()>>)>,
cloudwatch_client: Box<dyn CloudWatch + Send + Sync>,
cloudwatch_namespace: String,
) {
let cloudwatch_client = &cloudwatch_client;
let cloudwatch_namespace = &cloudwatch_namespace;
while let Some(metrics) = emit_receiver.recv().await {
while let Some((metrics, flush_signal)) = emit_receiver.recv().await {
let chunks: Vec<_> = metrics_chunks(&metrics).collect();
stream::iter(chunks)
.for_each(|metric_data| async move {
Expand All @@ -261,11 +264,17 @@ async fn mk_emitter(
e,
),
Err(tokio::time::error::Elapsed { .. }) => {
log::warn!("Failed to send metrics: send timeout")
log::warn!("Failed to send metrics: send timeout");
}
}
})
.await;

if let Some(signal) = flush_signal {
if let Err(_) = signal.send(()) {
log::warn!("Unable to send flush complete signal");
}
}
}
}

Expand Down Expand Up @@ -353,7 +362,7 @@ fn jitter_interval_at(
}

fn mk_send_batch_timer(
emit_sender: mpsc::Sender<Vec<MetricDatum>>,
emit_sender: mpsc::Sender<(Vec<MetricDatum>, Option<oneshot::Sender<()>>)>,
config: &Config,
) -> impl Stream<Item = Message> {
let interval = Duration::from_secs(config.send_interval_secs);
Expand All @@ -363,6 +372,7 @@ fn mk_send_batch_timer(
Message::SendBatch {
send_all_before,
emit_sender: emit_sender.clone(),
flush_signal: None,
}
})
}
Expand Down Expand Up @@ -476,8 +486,9 @@ impl Collector {
Message::SendBatch {
send_all_before,
emit_sender,
flush_signal,
} => {
self.accept_send_batch(send_all_before, emit_sender)?;
self.accept_send_batch(send_all_before, emit_sender, flush_signal)?;
break;
}
},
Expand All @@ -493,7 +504,8 @@ impl Collector {
Message::SendBatch {
send_all_before,
emit_sender,
} => self.accept_send_batch(send_all_before, emit_sender),
flush_signal,
} => self.accept_send_batch(send_all_before, emit_sender, flush_signal),
}
};
if let Err(e) = result.await {
Expand Down Expand Up @@ -550,12 +562,13 @@ impl Collector {
fn accept_send_batch(
&mut self,
send_all_before: Timestamp,
emit_sender: mpsc::Sender<Vec<MetricDatum>>,
emit_sender: mpsc::Sender<(Vec<MetricDatum>, Option<oneshot::Sender<()>>)>,
flush_signal: Option<oneshot::Sender<()>>,
) -> Result<(), Box<dyn std::error::Error>> {
let mut range = self.metrics_data.split_off(&send_all_before);
mem::swap(&mut range, &mut self.metrics_data);

let mut metrics_batch = vec![];
let mut metrics_batch = (vec![], flush_signal);

for (timestamp, stats_by_key) in range {
let timestamp = timestamp_string(time::UNIX_EPOCH + Duration::from_secs(timestamp));
Expand Down Expand Up @@ -609,7 +622,7 @@ impl Collector {
maximum: sum,
minimum: sum,
};
metrics_batch.push(stats_set_datum(stats_set, unit.or(Some("Count"))));
metrics_batch.0.push(stats_set_datum(stats_set, unit.or(Some("Count"))));
}

if let Some(Gauge {
Expand All @@ -625,7 +638,7 @@ impl Collector {
minimum,
maximum,
};
metrics_batch.push(stats_set_datum(statistic_set, unit));
metrics_batch.0.push(stats_set_datum(statistic_set, unit));
}

let histogram_datum = &mut |Histogram { values, counts }, unit| MetricDatum {
Expand Down Expand Up @@ -656,12 +669,12 @@ impl Collector {
if histogram.values.is_empty() {
break;
};
metrics_batch.push(histogram_datum(histogram, unit.map(|s| s.into())));
metrics_batch.0.push(histogram_datum(histogram, unit.map(|s| s.into())));
}
}
}
}
if !metrics_batch.is_empty() {
if !metrics_batch.0.is_empty() {
emit_sender.try_send(metrics_batch)?;
}
Ok(())
Expand Down
0 tests/test.rs → tests/flush_on_shutdown.rs
View file
Open in desktop
File renamed without changes.
59 changes: 59 additions & 0 deletions tests/flush_signal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
use std::time::Duration;

use futures_util::FutureExt;

use common::MockCloudWatchClient;

mod common;

#[tokio::test]
async fn test_manual_flush() {
let _ = env_logger::try_init();

let client = MockCloudWatchClient::default();

tokio::time::pause();
let (tx, rx) = tokio::sync::oneshot::channel();

let (metrics_force_flush_sender, metrics_force_flush_receiver) =
tokio::sync::mpsc::channel::<Option<tokio::sync::oneshot::Sender<()>>>(1);
let metrics_force_flush_receiver =
tokio_stream::wrappers::ReceiverStream::from(metrics_force_flush_receiver);

let backend_fut = Box::pin(
metrics_cloudwatch::Builder::new_with(client.clone())
.cloudwatch_namespace("test-ns")
.default_dimension("dimension", "default")
.send_interval_secs(100)
.storage_resolution(metrics_cloudwatch::Resolution::Second)
.shutdown_signal(Box::pin(rx.map(|_| ())))
.force_flush_stream(Box::pin(metrics_force_flush_receiver))
.init_future(metrics::set_boxed_recorder),
);

let join_handle = tokio::spawn(backend_fut);
tokio::time::advance(Duration::from_millis(5)).await;

for i in 0..150 {
metrics::histogram!("test", i as f64);
metrics::counter!("count", 1);
metrics::gauge!("gauge", i as f64);
}

{
let (tx, rx) = tokio::sync::oneshot::channel();

// Send flush with flush signal:
metrics_force_flush_sender.send(Some(tx)).await.unwrap();

// Wait for a flush signal:
match rx.await {
Ok(()) => {},
_ => panic!("Expected a flush signal"),
}

}

tx.send(()).unwrap();
join_handle.await.unwrap().unwrap();
}