From 9db79b903e5fb2e6d2dfb5b4d04641a4e9244c26 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 27 Dec 2023 16:12:53 +0000 Subject: [PATCH] Test harness progress Next up: putting into action my thoughts on improved idempotency. --- nexus/src/app/instance_network.rs | 9 +- nexus/src/app/sagas/instance_common.rs | 4 +- nexus/src/app/sagas/instance_create.rs | 3 +- nexus/src/app/sagas/instance_ip_attach.rs | 215 ++++++++++++++------ nexus/src/app/sagas/instance_ip_detach.rs | 228 +++++++++++++++++++++- nexus/test-utils/src/resource_helpers.rs | 4 +- sled-agent/src/params.rs | 4 +- sled-agent/src/sim/http_entrypoints.rs | 39 +++- sled-agent/src/sim/sled_agent.rs | 67 ++++++- 9 files changed, 488 insertions(+), 85 deletions(-) diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index a0fb217a4e..ca45025b5e 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -358,12 +358,9 @@ impl super::Nexus { // This is performed so that an IP attach/detach will block the // instance_start saga. Return service unavailable to indicate // the request is retryable. - if ips_of_interest - .iter() - .any(|ip| { - must_all_be_attached && ip.state != IpAttachState::Attached - }) - { + if ips_of_interest.iter().any(|ip| { + must_all_be_attached && ip.state != IpAttachState::Attached + }) { return Err(Error::unavail( "cannot push all DPD state: IP attach/detach in progress", )); diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index b325a2da7a..c94aea8fb3 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -362,7 +362,7 @@ pub async fn instance_ip_remove_opte( ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); - // If we didn't push OPTE before, don't undo it. + // No physical sled? Don't inform OPTE. let Some(sled_uuid) = sagactx.lookup::("instance_state")?.sled_id else { @@ -382,7 +382,7 @@ pub async fn instance_ip_remove_opte( "sled agent client went away mid-attach", )) })? - .instance_put_external_ip(&authz_instance.id(), &sled_agent_body) + .instance_delete_external_ip(&authz_instance.id(), &sled_agent_body) .await .map_err(|e| { ActionError::action_failed(match e { diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 779fa140a3..d921275402 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -733,7 +733,8 @@ async fn sic_allocate_instance_external_ip_undo( error!( osagactx.log(), "sic_allocate_instance_external_ip_undo: failed to \ - completely detach ip {}", ip.id + completely detach ip {}", + ip.id ); } } diff --git a/nexus/src/app/sagas/instance_ip_attach.rs b/nexus/src/app/sagas/instance_ip_attach.rs index 8a6841b339..2e877cc065 100644 --- a/nexus/src/app/sagas/instance_ip_attach.rs +++ b/nexus/src/app/sagas/instance_ip_attach.rs @@ -246,15 +246,28 @@ impl NexusSaga for SagaInstanceIpAttach { #[cfg(test)] pub(crate) mod test { - use crate::app::sagas::test_helpers; - use super::*; + use crate::app::{ + saga::create_saga_dag, + sagas::test_helpers::{self, instance_simulate}, + }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; use dropshot::test_util::ClientTestContext; - use nexus_db_model::Name; + use nexus_db_model::{IpKind, Name}; use nexus_db_queries::context::OpContext; - use nexus_test_utils::resource_helpers::{populate_ip_pool, create_project, create_disk, create_floating_ip, object_create}; + use nexus_test_utils::resource_helpers::{ + create_disk, create_floating_ip, create_instance, create_project, + object_create, populate_ip_pool, + }; use nexus_test_utils_macros::nexus_test; - use omicron_common::api::external::{SimpleIdentity, IdentityMetadataCreateParams}; + use nexus_types::external_api::params::ExternalIpCreate; + use omicron_common::api::external::{ + ByteCount, IdentityMetadataCreateParams, InstanceCpuCount, + SimpleIdentity, + }; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -262,42 +275,10 @@ pub(crate) mod test { const PROJECT_NAME: &str = "cafe"; const INSTANCE_NAME: &str = "menu"; const FIP_NAME: &str = "affogato"; - const DISK_NAME: &str = "my-disk"; - - // Test matrix: - // - instance started/stopped - // - fip vs ephemeral - - // async fn create_instance( - // client: &ClientTestContext, - // ) -> omicron_common::api::external::Instance { - // let instances_url = format!("/v1/instances?project={}", PROJECT_NAME); - // object_create( - // client, - // &instances_url, - // ¶ms::InstanceCreate { - // identity: IdentityMetadataCreateParams { - // name: INSTANCE_NAME.parse().unwrap(), - // description: format!("instance {:?}", INSTANCE_NAME), - // }, - // ncpus: InstanceCpuCount(2), - // memory: ByteCount::from_gibibytes_u32(2), - // hostname: String::from(INSTANCE_NAME), - // user_data: b"#cloud-config".to_vec(), - // network_interfaces: - // params::InstanceNetworkInterfaceAttachment::None, - // external_ips: vec![], - // disks: vec![], - // start: false, - // }, - // ) - // .await - // } pub async fn ip_manip_test_setup(client: &ClientTestContext) -> Uuid { populate_ip_pool(&client, "default", None).await; let project = create_project(client, PROJECT_NAME).await; - create_disk(&client, PROJECT_NAME, DISK_NAME).await; create_floating_ip( client, FIP_NAME, @@ -307,22 +288,33 @@ pub(crate) mod test { ) .await; - project.id() } - async fn new_test_params(opctx: &OpContext, datastore: &db::DataStore, project_id: Uuid, use_floating: bool) -> Params { + pub async fn new_test_params( + opctx: &OpContext, + datastore: &db::DataStore, + use_floating: bool, + ) -> Params { let create_params = if use_floating { - params::ExternalIpCreate::Floating { floating_ip_name: FIP_NAME.parse().unwrap() } + params::ExternalIpCreate::Floating { + floating_ip_name: FIP_NAME.parse().unwrap(), + } } else { params::ExternalIpCreate::Ephemeral { pool_name: None } }; - let (.., authz_instance) = LookupPath::new(opctx, datastore).project_id(project_id) - .instance_name(&Name(INSTANCE_NAME.parse().unwrap())).lookup_for(authz::Action::Modify).await.unwrap(); + let (.., authz_project, authz_instance) = + LookupPath::new(opctx, datastore) + .project_name(&Name(PROJECT_NAME.parse().unwrap())) + .instance_name(&Name(INSTANCE_NAME.parse().unwrap())) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + Params { serialized_authn: authn::saga::Serialized::for_opctx(opctx), - project_id, + project_id: authz_project.id(), create_params, authz_instance, } @@ -333,36 +325,143 @@ pub(crate) mod test { cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; - let nexus = &cptestctx.server.apictx().nexus; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + let opctx = test_helpers::test_opctx(cptestctx); - let instance = create_instance(client).await; - let db_instance = - test_helpers::instance_fetch(cptestctx, instance.identity.id) - .await - .instance() - .clone(); - let project_id = ip_manip_test_setup(&client); - todo!() + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let _instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + for use_float in [false, true] { + let params = new_test_params(&opctx, datastore, use_float).await; + + let dag = create_saga_dag::(params).unwrap(); + let saga = nexus.create_runnable_saga(dag).await.unwrap(); + nexus.run_saga(saga).await.expect("Attach saga should succeed"); + + // TODO: assert sled agent, dpd happy, ... + } + } + + pub(crate) async fn verify_clean_slate( + cptestctx: &ControlPlaneTestContext, + instance_id: Uuid, + ) { + use nexus_db_queries::db::schema::external_ip::dsl; + + let sled_agent = &cptestctx.sled_agent.sled_agent; + let datastore = cptestctx.server.apictx().nexus.datastore(); + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + // No Floating IPs exist in states other than 'detached'. + assert!(dsl::external_ip + .filter(dsl::kind.eq(IpKind::Floating)) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::parent_id.eq(instance_id)) + .filter(dsl::state.ne(IpAttachState::Detached)) + .select(ExternalIp::as_select()) + .first_async::(&*conn,) + .await + .optional() + .unwrap() + .is_none()); + + // All ephemeral IPs are removed. + assert!(dsl::external_ip + .filter(dsl::kind.eq(IpKind::Ephemeral)) + .filter(dsl::time_deleted.is_null()) + .select(ExternalIp::as_select()) + .first_async::(&*conn,) + .await + .optional() + .unwrap() + .is_none()); + + // No IP bindings remain on sled-agent. + let eips = &*sled_agent.external_ips.lock().await; + for (_nic_id, eip_set) in eips { + assert!(eip_set.is_empty()); + } } #[nexus_test(server = crate::Server)] async fn test_action_failure_can_unwind( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + for use_float in [false, true] { + test_helpers::action_failure_can_unwind::( + nexus, + || Box::pin(new_test_params(&opctx, datastore, use_float) ), + || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + log, + ) + .await; + } } #[nexus_test(server = crate::Server)] async fn test_action_failure_can_unwind_idempotently( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + for use_float in [false, true] { + test_helpers::action_failure_can_unwind_idempotently::< + SagaInstanceIpAttach, + _, + _, + >( + nexus, + || Box::pin(new_test_params(&opctx, datastore, use_float)), + || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + log, + ) + .await; + } } #[nexus_test(server = crate::Server)] async fn test_actions_succeed_idempotently( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let _instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + for use_float in [false, true] { + let params = new_test_params(&opctx, datastore, use_float).await; + let dag = create_saga_dag::(params).unwrap(); + test_helpers::actions_succeed_idempotently(nexus, dag).await; + } } } diff --git a/nexus/src/app/sagas/instance_ip_detach.rs b/nexus/src/app/sagas/instance_ip_detach.rs index 3db0e92700..46edca1d0c 100644 --- a/nexus/src/app/sagas/instance_ip_detach.rs +++ b/nexus/src/app/sagas/instance_ip_detach.rs @@ -252,37 +252,247 @@ impl NexusSaga for SagaInstanceIpDetach { #[cfg(test)] pub(crate) mod test { - + use super::*; + use crate::{ + app::{ + saga::create_saga_dag, + sagas::{ + instance_ip_attach::{self, test::ip_manip_test_setup}, + test_helpers, + }, + }, + Nexus, + }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; + use nexus_db_model::Name; + use nexus_db_queries::context::OpContext; + use nexus_test_utils::resource_helpers::create_instance; use nexus_test_utils_macros::nexus_test; + use omicron_common::api::external::SimpleIdentity; + use std::sync::Arc; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; + const PROJECT_NAME: &str = "cafe"; + const INSTANCE_NAME: &str = "menu"; + const FIP_NAME: &str = "affogato"; + + async fn new_test_params( + opctx: &OpContext, + datastore: &db::DataStore, + use_floating: bool, + ) -> Params { + let delete_params = if use_floating { + params::ExternalIpDelete::Floating { + floating_ip_name: FIP_NAME.parse().unwrap(), + } + } else { + params::ExternalIpDelete::Ephemeral + }; + + let (.., authz_project, authz_instance) = + LookupPath::new(opctx, datastore) + .project_name(&Name(PROJECT_NAME.parse().unwrap())) + .instance_name(&Name(INSTANCE_NAME.parse().unwrap())) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + project_id: authz_project.id(), + delete_params, + authz_instance, + } + } + + async fn attach_instance_ips(nexus: &Arc, opctx: &OpContext) { + let datastore = &nexus.db_datastore; + + let proj_name = Name(PROJECT_NAME.parse().unwrap()); + let inst_name = Name(INSTANCE_NAME.parse().unwrap()); + let lookup = LookupPath::new(opctx, datastore) + .project_name(&proj_name) + .instance_name(&inst_name); + + for use_float in [false, true] { + let params = instance_ip_attach::test::new_test_params( + opctx, datastore, use_float, + ) + .await; + nexus + .instance_attach_external_ip( + opctx, + &lookup, + ¶ms.create_params, + ) + .await + .unwrap(); + } + } + #[nexus_test(server = crate::Server)] async fn test_saga_basic_usage_succeeds( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let _ = ip_manip_test_setup(&client).await; + let _instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + attach_instance_ips(nexus, &opctx).await; + + for use_float in [false, true] { + let params = new_test_params(&opctx, datastore, use_float).await; + + let dag = create_saga_dag::(params).unwrap(); + let saga = nexus.create_runnable_saga(dag).await.unwrap(); + nexus.run_saga(saga).await.expect("Detach saga should succeed"); + } + } + + pub(crate) async fn verify_clean_slate( + cptestctx: &ControlPlaneTestContext, + instance_id: Uuid, + ) { + use nexus_db_queries::db::schema::external_ip::dsl; + + let opctx = test_helpers::test_opctx(cptestctx); + let sled_agent = &cptestctx.sled_agent.sled_agent; + let datastore = cptestctx.server.apictx().nexus.datastore(); + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + // No IPs in transitional states w/ current instance. + assert!(dsl::external_ip + .filter(dsl::time_deleted.is_null()) + .filter(dsl::parent_id.eq(instance_id)) + .filter(dsl::state.ne(IpAttachState::Attached)) + .select(ExternalIp::as_select()) + .first_async::(&*conn,) + .await + .optional() + .unwrap() + .is_none()); + + // No external IPs in detached state. + assert!(dsl::external_ip + .filter(dsl::time_deleted.is_null()) + .filter(dsl::state.eq(IpAttachState::Detached)) + .select(ExternalIp::as_select()) + .first_async::(&*conn,) + .await + .optional() + .unwrap() + .is_none()); + + // Instance still has one Ephemeral IP, and one Floating IP. + let db_eips = datastore + .instance_lookup_external_ips(&opctx, instance_id) + .await + .unwrap(); + assert_eq!(db_eips.len(), 3); + assert!(db_eips.iter().find(|v| v.kind == IpKind::Ephemeral).is_some()); + assert!(db_eips.iter().find(|v| v.kind == IpKind::Floating).is_some()); + assert!(db_eips.iter().find(|v| v.kind == IpKind::SNat).is_some()); + + // No IP bindings remain on sled-agent. + let eips = &*sled_agent.external_ips.lock().await; + for (_nic_id, eip_set) in eips { + assert_eq!(eip_set.len(), 2); + } } #[nexus_test(server = crate::Server)] async fn test_action_failure_can_unwind( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + attach_instance_ips(nexus, &opctx).await; + + for use_float in [false, true] { + test_helpers::action_failure_can_unwind::( + nexus, + || Box::pin(new_test_params(&opctx, datastore, use_float) ), + || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + log, + ) + .await; + } } #[nexus_test(server = crate::Server)] async fn test_action_failure_can_unwind_idempotently( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + attach_instance_ips(nexus, &opctx).await; + + for use_float in [false, true] { + test_helpers::action_failure_can_unwind_idempotently::< + SagaInstanceIpDetach, + _, + _, + >( + nexus, + || Box::pin(new_test_params(&opctx, datastore, use_float)), + || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + log, + ) + .await; + } } #[nexus_test(server = crate::Server)] async fn test_actions_succeed_idempotently( - _cptestctx: &ControlPlaneTestContext, + cptestctx: &ControlPlaneTestContext, ) { - todo!() + let log = &cptestctx.logctx.log; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + let opctx = test_helpers::test_opctx(cptestctx); + let datastore = &nexus.db_datastore; + let project_id = ip_manip_test_setup(&client).await; + let _instance = + create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + + attach_instance_ips(nexus, &opctx).await; + + for use_float in [false, true] { + let params = new_test_params(&opctx, datastore, use_float).await; + let dag = create_saga_dag::(params).unwrap(); + test_helpers::actions_succeed_idempotently(nexus, dag).await; + } } } diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index c72c7ad780..f564d73119 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -68,8 +68,8 @@ where .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .unwrap_or_else(|_| { - panic!("failed to make \"create\" request to {path}") + .unwrap_or_else(|e| { + panic!("failed to make \"create\" request to {path}: {e}") }) .parsed_body() .unwrap() diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index e5e1b82977..2263aa725d 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -827,7 +827,9 @@ pub struct CleanupContextUpdate { } /// Used to dynamically update external IPs attached to an instance. -#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +#[derive( + Copy, Clone, Debug, Eq, PartialEq, Hash, Deserialize, JsonSchema, Serialize, +)] #[serde(rename_all = "snake_case", tag = "type", content = "value")] pub enum InstanceExternalIpBody { Ephemeral(IpAddr), diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index f77da11b0e..d533db3252 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -8,8 +8,9 @@ use crate::bootstrap::early_networking::{ EarlyNetworkConfig, EarlyNetworkConfigBody, }; use crate::params::{ - DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, - InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, + DiskEnsureBody, InstanceEnsureBody, InstanceExternalIpBody, + InstancePutMigrationIdsBody, InstancePutStateBody, + InstancePutStateResponse, InstanceUnregisterResponse, VpcFirewallRulesEnsureBody, }; use dropshot::endpoint; @@ -45,6 +46,8 @@ pub fn api() -> SledApiDescription { api.register(instance_put_state)?; api.register(instance_register)?; api.register(instance_unregister)?; + api.register(instance_put_external_ip)?; + api.register(instance_delete_external_ip)?; api.register(instance_poke_post)?; api.register(disk_put)?; api.register(disk_poke_post)?; @@ -149,6 +152,38 @@ async fn instance_put_migration_ids( )) } +#[endpoint { + method = PUT, + path = "/instances/{instance_id}/external-ip", +}] +async fn instance_put_external_ip( + rqctx: RequestContext>, + path_params: Path, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let instance_id = path_params.into_inner().instance_id; + let body_args = body.into_inner(); + sa.instance_put_external_ip(instance_id, &body_args).await?; + Ok(HttpResponseUpdatedNoContent()) +} + +#[endpoint { + method = DELETE, + path = "/instances/{instance_id}/external-ip", +}] +async fn instance_delete_external_ip( + rqctx: RequestContext>, + path_params: Path, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let instance_id = path_params.into_inner().instance_id; + let body_args = body.into_inner(); + sa.instance_delete_external_ip(instance_id, &body_args).await?; + Ok(HttpResponseUpdatedNoContent()) +} + #[endpoint { method = POST, path = "/instances/{instance_id}/poke", diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index a16049dd2f..3a5633c6c3 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -13,9 +13,9 @@ use super::storage::Storage; use crate::nexus::NexusClient; use crate::params::{ - DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, - InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, + DiskStateRequested, InstanceExternalIpBody, InstanceHardware, + InstanceMigrationSourceParams, InstancePutStateResponse, + InstanceStateRequested, InstanceUnregisterResponse, }; use crate::sim::simulatable::Simulatable; use crate::updates::UpdateManager; @@ -32,7 +32,7 @@ use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use uuid::Uuid; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use dropshot::HttpServer; @@ -68,6 +68,8 @@ pub struct SledAgent { pub v2p_mappings: Mutex>>, mock_propolis: Mutex>, PropolisClient)>>, + /// lists of external IPs assigned to instances + pub external_ips: Mutex>>, instance_ensure_state_error: Mutex>, } @@ -160,6 +162,7 @@ impl SledAgent { nexus_client, disk_id_to_region_ids: Mutex::new(HashMap::new()), v2p_mappings: Mutex::new(HashMap::new()), + external_ips: Mutex::new(HashMap::new()), mock_propolis: Mutex::new(None), instance_ensure_state_error: Mutex::new(None), }) @@ -620,6 +623,62 @@ impl SledAgent { Ok(()) } + pub async fn instance_put_external_ip( + &self, + instance_id: Uuid, + body_args: &InstanceExternalIpBody, + ) -> Result<(), Error> { + if !self.instances.contains_key(&instance_id).await { + return Err(Error::internal_error( + "can't alter IP state for nonexistant instance", + )); + } + + let mut eips = self.external_ips.lock().await; + let my_eips = eips.entry(instance_id).or_default(); + + // High-level behaviour: this should always succeed UNLESS + // trying to add a double ephemeral. + if let InstanceExternalIpBody::Ephemeral(curr_ip) = &body_args { + if my_eips + .iter() + .find(|v| { + if let InstanceExternalIpBody::Ephemeral(other_ip) = v { + curr_ip != other_ip + } else { + false + } + }) + .is_some() + { + return Err(Error::invalid_request("cannot replace exisitng ephemeral IP without explicit removal")); + } + } + + my_eips.insert(*body_args); + + Ok(()) + } + + pub async fn instance_delete_external_ip( + &self, + instance_id: Uuid, + body_args: &InstanceExternalIpBody, + ) -> Result<(), Error> { + if !self.instances.contains_key(&instance_id).await { + return Err(Error::internal_error( + "can't alter IP state for nonexistant instance", + )); + } + + let mut eips = self.external_ips.lock().await; + let my_eips = eips.entry(instance_id).or_default(); + + my_eips.remove(&body_args); + + Ok(()) + } + /// Used for integration tests that require a component to talk to a /// mocked propolis-server API. // TODO: fix schemas so propolis-server's port isn't hardcoded in nexus