reject old generations and also ignore that on the client side

sled-agent/src/rack_setup/service.rs

@@ -275,11 +275,27 @@ impl ServiceInner {
                 self.log,
                 "attempting to set up sled's Omicron zones: {:?}", zones_config
             );
-            client
-                .omicron_zones_put(&zones_config.clone().into())
-                .await
-                .map_err(BackoffError::transient)?;
-            Ok::<(), BackoffError<SledAgentError<SledAgentTypes::Error>>>(())
+            let result =
+                client.omicron_zones_put(&zones_config.clone().into()).await;
+            let Err(error) = result else {
+                return Ok::<
+                    (),
+                    BackoffError<SledAgentError<SledAgentTypes::Error>>,
+                >(());
+            };
+
+            if let sled_agent_client::Error::ErrorResponse(response) = &error {
+                if let Some(code) = &response.error_code {
+                    if code == "RequestedConfigOutdated" {
+                        // XXX-dap is this really how to do this?  look at what
+                        // nexus does when propagating DNS maybe?  or maybe the
+                        // DNS tests?
+                        return Ok(());
+                    }
+                }
+            }
+
+            return Err(BackoffError::transient(error));
         };
         let log_failure = |error, delay| {
             warn!(
@@ -315,7 +331,6 @@ impl ServiceInner {
     ) -> Result<(), SetupServiceError> {
         futures::future::join_all(configs.iter().map(
             |(sled_address, zones_config)| async move {
-                // XXX-dap check for the error indicating an older generation
                 self.initialize_zones_on_sled(*sled_address, zones_config)
                     .await?;
                 Ok(())

sled-agent/src/services.rs

@@ -196,9 +196,6 @@ pub enum Error {
     #[error("Could not initialize service {service} as requested: {message}")]
     BadServiceRequest { service: String, message: String },
 
-    #[error("Services already configured for this Sled Agent")]
-    ServicesAlreadyConfigured,
-
     #[error("Failed to get address: {0}")]
     GetAddressFailure(#[from] illumos_utils::zone::GetAddressError),
 
@@ -223,6 +220,9 @@ pub enum Error {
     #[error("Error querying simnet devices")]
     Simnet(#[from] GetSimnetError),
 
+    #[error("Requested generation ({0}) is older than current ({0})")]
+    RequestedConfigOutdated(Generation, Generation),
+
     #[error("Error migrating old-format services ledger")]
     ServicesMigration(anyhow::Error),
 }
@@ -2601,6 +2601,12 @@ impl ServiceManager {
         };
 
         let ledger_zone_config = ledger.data_mut();
+        if ledger_zone_config.generation > request.generation {
+            return Err(Error::RequestedConfigOutdated(
+                request.generation,
+                ledger_zone_config.generation,
+            ));
+        }
 
         let new_config = self
             .ensure_all_omicron_zones(