Skip to content

Commit

Permalink
[nexus] deflake test_instance_watcher_metrics (#5768)
Browse files Browse the repository at this point in the history
Presently, `test_instance_watcher_metrics` will wait for the
`instance_watcher` background task to have run before making assertions
about metrics, but it does *not* ensure that oximeter has actually
collected those metrics. This can result in flaky failures --- see
#5752.

This commit adds explicit calls to `oximeter.force_collect()` prior to
making assertions, to ensure that the latest metrics have been
collected.

Fixes #5752
  • Loading branch information
hawkw authored May 15, 2024
1 parent c8c7b41 commit 7566128
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions nexus/tests/integration_tests/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ async fn test_instance_watcher_metrics(
let client = &cptestctx.external_client;
let internal_client = &cptestctx.internal_client;
let nexus = &cptestctx.server.server_context().nexus;
let oximeter = &cptestctx.oximeter;

// TODO(eliza): consider factoring this out to a generic
// `activate_background_task` function in `nexus-test-utils` eventually?
Expand Down Expand Up @@ -399,6 +400,8 @@ async fn test_instance_watcher_metrics(
)
.await
.unwrap();
// Make sure that the latest metrics have been collected.
oximeter.force_collect().await;
};

#[track_caller]
Expand Down Expand Up @@ -443,11 +446,8 @@ async fn test_instance_watcher_metrics(
let project = create_project_and_pool(&client).await;
let project_name = project.identity.name.as_str();
// Wait until Nexus registers as a producer with Oximeter.
wait_for_producer(
&cptestctx.oximeter,
cptestctx.server.server_context().nexus.id(),
)
.await;
wait_for_producer(&oximeter, cptestctx.server.server_context().nexus.id())
.await;

eprintln!("--- creating instance 1 ---");
let instance1 = create_instance(&client, project_name, "i-1").await;
Expand Down

0 comments on commit 7566128

Please sign in to comment.