Add a background task for update plan execution (#4891)

This PR is the first step in creating a background task that is capable
of taking a `Blueprint` and then reifying that blueprint into deployed
or updated software. This PR uses the initial version of a Blueprint
introduced in #4804. A basic executor that sends the related
`OmicronZonesConfig` to the appropriate sled-agents for newly added
sleds was created.

A background task that loads the target `Blueprint` from the database
and feeds it to the executor is also included, along with a test for each.

common/src/nexus_config.rs

@@ -334,6 +334,8 @@ pub struct BackgroundTaskConfig {
     pub inventory: InventoryConfig,
     /// configuration for phantom disks task
     pub phantom_disks: PhantomDiskConfig,
+    /// configuration for blueprint related tasks
+    pub blueprints: BlueprintTasksConfig,
     /// configuration for service zone nat sync task
     pub sync_service_zone_nat: SyncServiceZoneNatConfig,
     /// configuration for the bfd manager task
@@ -428,6 +430,20 @@ pub struct PhantomDiskConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct BlueprintTasksConfig {
+    /// period (in seconds) for periodic activations of the background task that
+    /// reads the latest target blueprint from the database
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs_load: Duration,
+
+    /// period (in seconds) for periodic activations of the background task that
+    /// executes the latest target blueprint
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs_execute: Duration,
+}
+
 /// Configuration for a nexus server
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct PackageConfig {
@@ -528,12 +544,12 @@ impl std::fmt::Display for SchemeName {
 mod test {
     use super::{
         default_techport_external_server_port, AuthnConfig,
-        BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig,
-        Database, DeploymentConfig, DnsTasksConfig, DpdConfig,
-        ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError,
-        LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig,
-        PhantomDiskConfig, SchemeName, TimeseriesDbConfig, Tunables,
-        UpdatesConfig,
+        BackgroundTaskConfig, BlueprintTasksConfig, Config,
+        ConfigDropshotWithTls, ConsoleConfig, Database, DeploymentConfig,
+        DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns,
+        InventoryConfig, LoadError, LoadErrorKind, MgdConfig, NatCleanupConfig,
+        PackageConfig, PhantomDiskConfig, SchemeName, TimeseriesDbConfig,
+        Tunables, UpdatesConfig,
     };
     use crate::address::{Ipv6Subnet, RACK_PREFIX};
     use crate::api::internal::shared::SwitchLocation;
@@ -687,6 +703,8 @@ mod test {
             inventory.nkeep = 11
             inventory.disable = false
             phantom_disks.period_secs = 30
+            blueprints.period_secs_load = 10
+            blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
@@ -795,6 +813,10 @@ mod test {
                         phantom_disks: PhantomDiskConfig {
                             period_secs: Duration::from_secs(30),
                         },
+                        blueprints: BlueprintTasksConfig {
+                            period_secs_load: Duration::from_secs(10),
+                            period_secs_execute: Duration::from_secs(60)
+                        },
                         sync_service_zone_nat: SyncServiceZoneNatConfig {
                             period_secs: Duration::from_secs(30)
                         }
@@ -857,6 +879,8 @@ mod test {
             inventory.nkeep = 3
             inventory.disable = false
             phantom_disks.period_secs = 30
+            blueprints.period_secs_load = 10
+            blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -256,6 +256,8 @@ async fn cmd_nexus_background_tasks_show(
         "dns_servers_external",
         "dns_propagation_external",
         "nat_v4_garbage_collector",
+        "blueprint_loader",
+        "blueprint_executor",
     ] {
         if let Some(bgtask) = tasks.remove(name) {
             print_task(&bgtask);

dev-tools/omdb/tests/env.out

@@ -28,6 +28,14 @@ task: "bfd_manager"
     switches
 
 
+task: "blueprint_executor"
+    Executes the target blueprint
+
+
+task: "blueprint_loader"
+    Loads the current target blueprint from the DB
+
+
 task: "dns_config_external"
     watches external DNS data stored in CockroachDB
 
@@ -106,6 +114,14 @@ task: "bfd_manager"
     switches
 
 
+task: "blueprint_executor"
+    Executes the target blueprint
+
+
+task: "blueprint_loader"
+    Loads the current target blueprint from the DB
+
+
 task: "dns_config_external"
     watches external DNS data stored in CockroachDB
 
@@ -171,6 +187,14 @@ task: "bfd_manager"
     switches
 
 
+task: "blueprint_executor"
+    Executes the target blueprint
+
+
+task: "blueprint_loader"
+    Loads the current target blueprint from the DB
+
+
 task: "dns_config_external"
     watches external DNS data stored in CockroachDB
 

dev-tools/omdb/tests/successes.out

@@ -222,6 +222,14 @@ task: "bfd_manager"
     switches
 
 
+task: "blueprint_executor"
+    Executes the target blueprint
+
+
+task: "blueprint_loader"
+    Loads the current target blueprint from the DB
+
+
 task: "dns_config_external"
     watches external DNS data stored in CockroachDB
 
@@ -344,6 +352,20 @@ task: "nat_v4_garbage_collector"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null)
 
+task: "blueprint_loader"
+  configured period: every 1m 40s
+  currently executing: no
+  last completed activation: iter 2, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "blueprint_loader" (don't know how to interpret details: Object {"status": String("no target blueprint")})
+
+task: "blueprint_executor"
+  configured period: every 10m
+  currently executing: no
+  last completed activation: iter 2, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    last completion reported error: no blueprint
+
 task: "bfd_manager"
   configured period: every 30s
   currently executing: no

nexus/examples/config.toml

@@ -106,6 +106,8 @@ inventory.nkeep = 5
 # Disable inventory collection altogether (for emergencies)
 inventory.disable = false
 phantom_disks.period_secs = 30
+blueprints.period_secs_load = 10
+blueprints.period_secs_execute = 60
 sync_service_zone_nat.period_secs = 30
 
 [default_region_allocation_strategy]