From d990790bd0f6c9e3eef8d9728011750a99d2a8e1 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 1 Aug 2023 20:57:16 +0000 Subject: [PATCH] WIP: internal API endpoints --- common/src/sql/dbinit.sql | 4 + nexus/db-model/src/ipv4_nat_entry.rs | 62 +++++++- .../src/db/datastore/ipv4_nat_entry.rs | 32 +++- nexus/src/internal_api/http_entrypoints.rs | 79 ++++++++++ openapi/nexus-internal.json | 144 ++++++++++++++++++ 5 files changed, 312 insertions(+), 9 deletions(-) diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 1214484ff0f..79f6811e9fa 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -2369,6 +2369,10 @@ CREATE TABLE omicron.public.ipv4_nat_entry ( time_deleted TIMESTAMPTZ ); +CREATE UNIQUE INDEX ON omicron.public.ipv4_nat_entry ( + external_address, first_port, last_port +); + CREATE UNIQUE INDEX ON omicron.public.ipv4_nat_entry ( gen ) diff --git a/nexus/db-model/src/ipv4_nat_entry.rs b/nexus/db-model/src/ipv4_nat_entry.rs index cd09cfe001a..a2f3d5d3968 100644 --- a/nexus/db-model/src/ipv4_nat_entry.rs +++ b/nexus/db-model/src/ipv4_nat_entry.rs @@ -1,11 +1,22 @@ +use std::net::{Ipv4Addr, Ipv6Addr}; + use super::MacAddr; use crate::{ schema::{ipv4_nat_entry, nat_gen}, SqlU16, SqlU32, Vni, }; use chrono::{DateTime, Utc}; +use omicron_common::api::external; +use schemars::JsonSchema; +use serde::Serialize; use uuid::Uuid; +// TODO correctness +// If we're not going to store ipv4 and ipv6 +// NAT entries in the same table, and we don't +// need any of the special properties of the IpNetwork +// column type, does it make sense to use a different +// column type? /// Database representation of an Ipv4 NAT Entry. #[derive(Insertable, Debug, Clone)] #[diesel(table_name = ipv4_nat_entry)] @@ -19,6 +30,11 @@ pub struct NewIpv4NatEntry { pub mac: MacAddr, } +// TODO correctness +// If we're not going to store ipv4 and ipv6 +// NAT entries in the same table, we should probably +// make the types more restrictive to prevent an +// accidental ipv6 entry from being created. #[derive(Queryable, Debug, Clone, Selectable)] #[diesel(table_name = ipv4_nat_entry)] pub struct Ipv4NatEntry { @@ -40,7 +56,7 @@ impl Ipv4NatEntry { } pub fn last_port(&self) -> u16 { - self.first_port.into() + self.last_port.into() } pub fn gen(&self) -> u32 { @@ -55,3 +71,47 @@ pub struct Ipv4NatGen { pub log_cnt: SqlU32, pub is_called: bool, } + +/// NAT Record +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct Ipv4NatEntryView { + pub external_address: Ipv4Addr, + pub first_port: u16, + pub last_port: u16, + pub sled_address: Ipv6Addr, + pub vni: external::Vni, + pub mac: external::MacAddr, + pub gen: u32, + pub deleted: bool, +} + +impl From for Ipv4NatEntryView { + fn from(value: Ipv4NatEntry) -> Self { + let external_address = match value.external_address.ip() { + std::net::IpAddr::V4(a) => a, + std::net::IpAddr::V6(_) => unreachable!(), + }; + + let sled_address = match value.sled_address.ip() { + std::net::IpAddr::V4(_) => unreachable!(), + std::net::IpAddr::V6(a) => a, + }; + + Self { + external_address, + first_port: value.first_port(), + last_port: value.last_port(), + sled_address, + vni: value.vni.0, + mac: *value.mac, + gen: value.gen(), + deleted: value.time_deleted.is_some(), + } + } +} + +/// NAT Generation +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct Ipv4NatGenView { + pub gen: u32, +} diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs index 137dc2ada1f..97462d763cd 100644 --- a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -7,6 +7,7 @@ use crate::db::model::{Ipv4NatEntry, NewIpv4NatEntry}; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::{DateTime, Utc}; use diesel::prelude::*; +use nexus_db_model::Ipv4NatEntryView; use nexus_db_model::SqlU32; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; @@ -132,6 +133,7 @@ impl DataStore { &self, opctx: &OpContext, gen: u32, + limit: u32, ) -> ListResultVec { use db::schema::ipv4_nat_entry::dsl; let gen: SqlU32 = gen.into(); @@ -141,6 +143,7 @@ impl DataStore { let list = dsl::ipv4_nat_entry .filter(dsl::gen.gt(gen)) .order_by(dsl::gen) + .limit(limit as i64) .select(Ipv4NatEntry::as_select()) .load_async(self.pool_authorized(opctx).await?) .await @@ -151,6 +154,19 @@ impl DataStore { Ok(list) } + pub async fn ipv4_nat_changeset( + &self, + opctx: &OpContext, + gen: u32, + limit: u32, + ) -> ListResultVec { + let nat_entries = + self.ipv4_nat_list_since_gen(opctx, gen, limit).await?; + let nat_entries: Vec = + nat_entries.iter().map(|e| e.clone().into()).collect(); + Ok(nat_entries) + } + pub async fn ipv4_nat_current_gen( &self, opctx: &OpContext, @@ -219,7 +235,7 @@ mod test { // We should not have any NAT entries at this moment let initial_state = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); assert!(initial_state.is_empty()); assert_eq!(datastore.ipv4_nat_current_gen(&opctx).await.unwrap(), 0); @@ -249,7 +265,7 @@ mod test { datastore.ipv4_nat_create(&opctx, nat1).await.unwrap(); let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); // The NAT table has undergone one change. One entry has been added, // none deleted, so we should be at generation 1. @@ -273,7 +289,7 @@ mod test { datastore.ipv4_nat_create(&opctx, nat2).await.unwrap(); let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); // The NAT table has undergone two changes. Two entries have been // added, none deleted, so we should be at generation 2. @@ -284,7 +300,7 @@ mod test { // Delete the first nat entry. It should show up as a later generation number. datastore.ipv4_nat_delete(&opctx, &first_entry).await.unwrap(); let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); // The NAT table has undergone three changes. Two entries have been // added, one deleted, so we should be at generation 3. Since the @@ -299,7 +315,7 @@ mod test { // Cancel deletion of NAT entry datastore.ipv4_nat_cancel_delete(&opctx, nat_entry).await.unwrap(); let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); // The NAT table has undergone four changes. let nat_entry = nat_entries.last().unwrap(); @@ -317,7 +333,7 @@ mod test { // Nothing should have changed (no records currently marked for deletion) let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); assert_eq!(nat_entries.len(), 2); assert_eq!(datastore.ipv4_nat_current_gen(&opctx).await.unwrap(), 4); @@ -338,7 +354,7 @@ mod test { // Both records should still exist (soft deleted record is newer than cutoff // values ) let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); assert_eq!(nat_entries.len(), 2); assert_eq!(datastore.ipv4_nat_current_gen(&opctx).await.unwrap(), 5); @@ -348,7 +364,7 @@ mod test { // Soft deleted NAT entry should be removed from the table let nat_entries = - datastore.ipv4_nat_list_since_gen(&opctx, 0).await.unwrap(); + datastore.ipv4_nat_list_since_gen(&opctx, 0, 10).await.unwrap(); assert_eq!(nat_entries.len(), 1); diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index 47f6899259b..5ae5d451388 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -24,6 +24,8 @@ use dropshot::RequestContext; use dropshot::ResultsPage; use dropshot::TypedBody; use hyper::Body; +use nexus_db_model::Ipv4NatEntryView; +use nexus_db_model::Ipv4NatGenView; use nexus_types::internal_api::params::SwitchPutRequest; use nexus_types::internal_api::params::SwitchPutResponse; use nexus_types::internal_api::views::to_list; @@ -65,6 +67,8 @@ pub fn internal_api() -> NexusApiDescription { api.register(saga_list)?; api.register(saga_view)?; + api.register(ipv4_nat_changeset)?; + api.register(ipv4_nat_gen)?; Ok(()) } @@ -484,3 +488,78 @@ async fn saga_view( }; apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } + +// NAT RPW internal APIs + +/// Path parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatPathParam { + /// which change number to start generating + /// the change set from + from_gen: u32, +} + +/// Query parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatQueryParam { + limit: u32, +} + +/// Fetch NAT ChangeSet +/// +/// Caller provides their generation as `from_gen`, along with a query +/// parameter for the page size (`limit`). Endpoint will return changes +/// that have occured since the caller's generation number up to the latest +/// change or until the `limit` is reached. If there are no changes, an +/// empty vec is returned. +#[endpoint { + method = GET, + path = "/rpw/nat/ipv4/changeset/{from_gen}" +}] +async fn ipv4_nat_changeset( + rqctx: RequestContext>, + path_params: Path, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_internal_api(&rqctx).await; + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let changeset = nexus + .datastore() + .ipv4_nat_changeset(&opctx, path.from_gen, query.limit) + .await?; + Ok(HttpResponseOk(changeset)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +// TODO: Remove (maybe) +// This endpoint may be more trouble than it's worth. The reason this endpoint +// may be a headache is because the underlying SEQUENCE in the DB will advance, +// even on failed transactions. This is not a big deal, but it will lead to +// callers trying to catch up to entries that don't exist if they check this +// endpoint first. As an alternative, we could just page through the +// changesets until we get nothing back, then revert to periodic polling +// of the changeset endpoint. + +/// Fetch NAT generation +#[endpoint { + method = GET, + path = "/rpw/nat/ipv4/gen" +}] +async fn ipv4_nat_gen( + rqctx: RequestContext>, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_internal_api(&rqctx).await; + let nexus = &apictx.nexus; + let gen = nexus.datastore().ipv4_nat_current_gen(&opctx).await?; + let view = Ipv4NatGenView { gen }; + Ok(HttpResponseOk(view)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index db276de5c03..6b43da91709 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -357,6 +357,82 @@ } } }, + "/rpw/nat/ipv4/changeset/{from_gen}": { + "get": { + "summary": "Fetch NAT ChangeSet", + "description": "Caller provides their generation as `from_gen`, along with a query parameter for the page size (`limit`). Endpoint will return changes that have occured since the caller's generation number up to the latest change or until the `limit` is reached. If there are no changes, an empty vec is returned.", + "operationId": "ipv4_nat_changeset", + "parameters": [ + { + "in": "path", + "name": "from_gen", + "description": "which change number to start generating the change set from", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "query", + "name": "limit", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Ipv4NatEntryView", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4NatEntryView" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/rpw/nat/ipv4/gen": { + "get": { + "summary": "Fetch NAT generation", + "operationId": "ipv4_nat_gen", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv4NatGenView" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sagas": { "get": { "summary": "List sagas", @@ -2040,6 +2116,68 @@ } ] }, + "Ipv4NatEntryView": { + "description": "NAT Record", + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "external_address": { + "type": "string", + "format": "ipv4" + }, + "first_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "gen": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "last_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "mac": { + "$ref": "#/components/schemas/MacAddr" + }, + "sled_address": { + "type": "string", + "format": "ipv6" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "deleted", + "external_address", + "first_port", + "gen", + "last_port", + "mac", + "sled_address", + "vni" + ] + }, + "Ipv4NatGenView": { + "description": "NAT Generation", + "type": "object", + "properties": { + "gen": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "gen" + ] + }, "Ipv4Network": { "type": "string", "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" @@ -3284,6 +3422,12 @@ "minLength": 1, "maxLength": 63 }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "ZpoolPutRequest": { "description": "Sent by a sled agent on startup to Nexus to request further instruction", "type": "object",