From 05284564f82d835c06ad9ee822a498a99ab08fe7 Mon Sep 17 00:00:00 2001 From: Michael Zeller Date: Wed, 10 Jan 2024 19:08:21 -0500 Subject: [PATCH] The control plane should use libipcc (#4536) We should use libipcc in omicron rather than make direct calls via `ioctl`. The library will take care of hiding the implementation details from upstream consumers -- this becomes important in the future when communication with the service processor from the host OS physically changes with newer board design. Currently the only consumer in this repo is installinator but the control plane is about to start communicating with the RoT via IPCC as well. This PR introduces new bindings around `libipcc` and removes the old `ioctl` interfaces. --------- Co-authored-by: Andy Fiddaman --- .cargo/config | 45 ++++++- .github/buildomat/build-and-test.sh | 9 ++ Cargo.lock | 7 +- Cargo.toml | 6 +- gateway/Cargo.toml | 2 +- gateway/src/http_entrypoints.rs | 13 +- gateway/src/http_entrypoints/conversions.rs | 2 +- installinator/Cargo.toml | 2 +- installinator/src/artifact.rs | 4 +- ipcc-key-value/src/ioctl.rs | 126 ------------------- ipcc-key-value/src/ioctl_common.rs | 57 --------- ipcc-key-value/src/ioctl_stub.rs | 32 ----- {ipcc-key-value => ipcc}/Cargo.toml | 3 +- ipcc/build.rs | 16 +++ ipcc/src/ffi.rs | 83 +++++++++++++ ipcc/src/handle.rs | 129 ++++++++++++++++++++ ipcc/src/handle_stub.rs | 25 ++++ {ipcc-key-value => ipcc}/src/lib.rs | 116 +++++++++++++++--- openapi/gateway.json | 2 +- 19 files changed, 420 insertions(+), 259 deletions(-) delete mode 100644 ipcc-key-value/src/ioctl.rs delete mode 100644 ipcc-key-value/src/ioctl_common.rs delete mode 100644 ipcc-key-value/src/ioctl_stub.rs rename {ipcc-key-value => ipcc}/Cargo.toml (91%) create mode 100644 ipcc/build.rs create mode 100644 ipcc/src/ffi.rs create mode 100644 ipcc/src/handle.rs create mode 100644 ipcc/src/handle_stub.rs rename {ipcc-key-value => ipcc}/src/lib.rs (75%) diff --git a/.cargo/config b/.cargo/config index 6794e988ad..f658f146c9 100644 --- a/.cargo/config +++ b/.cargo/config @@ -7,13 +7,50 @@ [build] rustdocflags = "--document-private-items" -# On illumos, use `-znocompstrtab` to reduce link time. +# On illumos, use `-znocompstrtab` to reduce link time. We also add the Oxide +# specific platform directory to the RPATH where additional libraries can be +# found such as libipcc. # -# Note that these flags are overridden by a user's environment variable, so -# things critical to correctness probably don't belong here. +# Our reasoning for including `-R/usr/platform/oxide/lib/amd64` here: +# - Oxide specific features - This path contains Oxide specific libraries such +# as libipcc and will likely grow over time to include more functionality. +# - Avoid the rpaths crate - The rpaths crate was built to deal with runtime +# paths that are dynamic such as with libraries like libpq which can live in +# different locations based on OS. This path will only ever be found on +# illumos and will be tied directly to the Oxide platform. +# - Less developer burden - Having something like the ipcc crate emit +# `DEP_IPCC_LIBDIRS` means that we end up littering the repo with Cargo.toml +# and build.rs changes whenever ipcc shows up somewhere in the dependency +# tree. While initially exploring that path we ran into a significant number +# of tests failing due to not being able to find libipcc in the runtime path +# which can be confusing or surprising to someone doing work on omicron. +# +# We could also update Helios so that a symlink is created from +# /usr/platform/oxide/lib/amd64 into /usr/lib/64 but we opted to not take +# this route forward as it meant waiting for another round of updates on +# shared build machines and to buildomat itself. +# +# As documented at: +# https://doc.rust-lang.org/cargo/reference/config.html#buildrustflags +# +# There are four mutually exclusive sources of extra flags. They are checked in +# order, with the first one being used: +# 1. `CARGO_ENCODED_RUSTFLAGS` environment variable. +# 2. `RUSTFLAGS` environment variable. +# 3. All matching target..rustflags and target..rustflags config +# entries joined together. +# 4. build.rustflags config value. +# +# When overriding the defaults in this config by environment variable the user +# may need to manually pass the additional options found below. +# +# Note that other runtime paths should not be added here, but should instead +# reuse the infrastructure found in the `rpaths` crate which can be found in +# this repo. Those paths are usually platform specific and will vary based on a +# variety of things such as host OS. [target.x86_64-unknown-illumos] rustflags = [ - "-C", "link-arg=-Wl,-znocompstrtab" + "-C", "link-arg=-Wl,-znocompstrtab,-R/usr/platform/oxide/lib/amd64" ] # Set up `cargo xtask`. diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index eab64c528c..5cf086b1a3 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -4,6 +4,8 @@ set -o errexit set -o pipefail set -o xtrace +target_os=$1 + # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) @@ -48,6 +50,13 @@ ptime -m bash ./tools/install_builder_prerequisites.sh -y # banner build export RUSTFLAGS="-D warnings" +# When running on illumos we need to pass an additional runpath that is +# usually configured via ".cargo/config" but the `RUSTFLAGS` env variable +# takes precedence. This path contains oxide specific libraries such as +# libipcc. +if [[ $target_os == "illumos" ]]; then + RUSTFLAGS="-D warnings -C link-arg=-R/usr/platform/oxide/lib/amd64" +fi export RUSTDOCFLAGS="-D warnings" export TMPDIR=$TEST_TMPDIR export RUST_BACKTRACE=1 diff --git a/Cargo.lock b/Cargo.lock index a4157829af..7491f30dde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3306,7 +3306,7 @@ dependencies = [ "illumos-utils", "installinator-artifact-client", "installinator-common", - "ipcc-key-value", + "ipcc", "itertools 0.12.0", "libc", "omicron-common", @@ -3459,9 +3459,10 @@ dependencies = [ ] [[package]] -name = "ipcc-key-value" +name = "ipcc" version = "0.1.0" dependencies = [ + "cfg-if", "ciborium", "libc", "omicron-common", @@ -4712,7 +4713,7 @@ dependencies = [ "http", "hyper", "illumos-utils", - "ipcc-key-value", + "ipcc", "omicron-common", "omicron-test-utils", "omicron-workspace-hack", diff --git a/Cargo.toml b/Cargo.toml index b2d0e406da..fbef04d3c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "installinator", "internal-dns-cli", "internal-dns", - "ipcc-key-value", + "ipcc", "key-manager", "nexus", "nexus/authz-macros", @@ -105,7 +105,7 @@ default-members = [ "installinator", "internal-dns-cli", "internal-dns", - "ipcc-key-value", + "ipcc", "key-manager", "nexus", "nexus/authz-macros", @@ -225,7 +225,7 @@ installinator-artifactd = { path = "installinator-artifactd" } installinator-artifact-client = { path = "clients/installinator-artifact-client" } installinator-common = { path = "installinator-common" } internal-dns = { path = "internal-dns" } -ipcc-key-value = { path = "ipcc-key-value" } +ipcc = { path = "ipcc" } ipnetwork = { version = "0.20", features = ["schemars"] } itertools = "0.12.0" key-manager = { path = "key-manager" } diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index f2e5f83a8a..450c4b445e 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -17,7 +17,7 @@ hex.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true -ipcc-key-value.workspace = true +ipcc.workspace = true omicron-common.workspace = true once_cell.workspace = true schemars.workspace = true diff --git a/gateway/src/http_entrypoints.rs b/gateway/src/http_entrypoints.rs index e33e8dd4a6..b5a765a8a8 100644 --- a/gateway/src/http_entrypoints.rs +++ b/gateway/src/http_entrypoints.rs @@ -443,11 +443,11 @@ pub struct ImageVersion { pub version: u32, } -// This type is a duplicate of the type in `ipcc-key-value`, and we provide a +// This type is a duplicate of the type in `ipcc`, and we provide a // `From<_>` impl to convert to it. We keep these types distinct to allow us to // choose different representations for MGS's HTTP API (this type) and the wire // format passed through the SP to installinator -// (`ipcc_key_value::InstallinatorImageId`), although _currently_ they happen to +// (`ipcc::InstallinatorImageId`), although _currently_ they happen to // be defined identically. #[derive( Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema, @@ -1292,7 +1292,7 @@ async fn sp_power_state_set( /// Set the installinator image ID the sled should use for recovery. /// -/// This value can be read by the host via IPCC; see the `ipcc-key-value` crate. +/// This value can be read by the host via IPCC; see the `ipcc` crate. #[endpoint { method = PUT, path = "/sp/{type}/{slot}/ipcc/installinator-image-id", @@ -1302,14 +1302,13 @@ async fn sp_installinator_image_id_set( path: Path, body: TypedBody, ) -> Result { - use ipcc_key_value::Key; + use ipcc::Key; let apictx = rqctx.context(); let sp_id = path.into_inner().sp.into(); let sp = apictx.mgmt_switch.sp(sp_id)?; - let image_id = - ipcc_key_value::InstallinatorImageId::from(body.into_inner()); + let image_id = ipcc::InstallinatorImageId::from(body.into_inner()); sp.set_ipcc_key_lookup_value( Key::InstallinatorImageId as u8, @@ -1330,7 +1329,7 @@ async fn sp_installinator_image_id_delete( rqctx: RequestContext>, path: Path, ) -> Result { - use ipcc_key_value::Key; + use ipcc::Key; let apictx = rqctx.context(); let sp_id = path.into_inner().sp.into(); diff --git a/gateway/src/http_entrypoints/conversions.rs b/gateway/src/http_entrypoints/conversions.rs index 1182163bcc..a4aef7425e 100644 --- a/gateway/src/http_entrypoints/conversions.rs +++ b/gateway/src/http_entrypoints/conversions.rs @@ -397,7 +397,7 @@ impl From for HostStartupOptions { } } -impl From for ipcc_key_value::InstallinatorImageId { +impl From for ipcc::InstallinatorImageId { fn from(id: InstallinatorImageId) -> Self { Self { update_id: id.update_id, diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index d489e73ec1..43966d1202 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -20,7 +20,7 @@ http.workspace = true illumos-utils.workspace = true installinator-artifact-client.workspace = true installinator-common.workspace = true -ipcc-key-value.workspace = true +ipcc.workspace = true itertools.workspace = true libc.workspace = true omicron-common.workspace = true diff --git a/installinator/src/artifact.rs b/installinator/src/artifact.rs index f74d7b7f06..734759a2c2 100644 --- a/installinator/src/artifact.rs +++ b/installinator/src/artifact.rs @@ -9,7 +9,7 @@ use clap::Args; use futures::StreamExt; use installinator_artifact_client::ClientError; use installinator_common::EventReport; -use ipcc_key_value::{InstallinatorImageId, Ipcc}; +use ipcc::{InstallinatorImageId, Ipcc}; use omicron_common::update::{ArtifactHash, ArtifactHashId}; use tokio::sync::mpsc; use uuid::Uuid; @@ -47,7 +47,7 @@ pub(crate) struct ArtifactIdOpts { impl ArtifactIdOpts { pub(crate) fn resolve(&self) -> Result { if self.from_ipcc { - let ipcc = Ipcc::open().context("error opening IPCC")?; + let ipcc = Ipcc::new().context("error opening IPCC")?; ipcc.installinator_image_id() .context("error retrieving installinator image ID") } else { diff --git a/ipcc-key-value/src/ioctl.rs b/ipcc-key-value/src/ioctl.rs deleted file mode 100644 index b0524e973f..0000000000 --- a/ipcc-key-value/src/ioctl.rs +++ /dev/null @@ -1,126 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -//! IPCC `ioctl` interface. -//! -//! This module is tightly-coupled to the host OS image, and should only be used -//! by services bundled with it (e.g., sled-agent and installinator). - -use crate::InstallinatorImageId; -use crate::InstallinatorImageIdError; -use crate::IpccKey; -use crate::IpccKeyLookupError; -use crate::PingError; -use crate::Pong; -use libc::c_int; -use std::fs::File; -use std::io; -use std::os::unix::prelude::AsRawFd; - -pub struct Ipcc { - file: File, -} - -impl Ipcc { - pub fn open() -> io::Result { - let file = File::options().read(true).write(true).open(IPCC_DEV)?; - Ok(Self { file }) - } - - pub fn ping(&self) -> Result { - const EXPECTED_REPLY: &[u8] = b"pong"; - - let mut buf = [0; EXPECTED_REPLY.len()]; - let n = self.key_lookup(IpccKey::Ping, &mut buf)?; - let buf = &buf[..n]; - - if buf == EXPECTED_REPLY { - Ok(Pong) - } else { - Err(PingError::UnexpectedReply(buf.to_vec())) - } - } - - pub fn installinator_image_id( - &self, - ) -> Result { - let mut buf = [0; InstallinatorImageId::CBOR_SERIALIZED_SIZE]; - let n = self.key_lookup(IpccKey::InstallinatorImageId, &mut buf)?; - let id = InstallinatorImageId::deserialize(&buf[..n]) - .map_err(InstallinatorImageIdError::DeserializationFailed)?; - Ok(id) - } - - fn key_lookup( - &self, - key: IpccKey, - buf: &mut [u8], - ) -> Result { - let mut kl = IpccKeyLookup { - key: key as u8, - buflen: u16::try_from(buf.len()).unwrap_or(u16::MAX), - result: 0, - datalen: 0, - buf: buf.as_mut_ptr(), - }; - - let result = unsafe { - libc::ioctl( - self.file.as_raw_fd(), - IPCC_KEYLOOKUP, - &mut kl as *mut IpccKeyLookup, - ) - }; - - if result != 0 { - let error = io::Error::last_os_error(); - return Err(IpccKeyLookupError::IoctlFailed { key, error }); - } - - match kl.result { - IPCC_KEYLOOKUP_SUCCESS => Ok(usize::from(kl.datalen)), - IPCC_KEYLOOKUP_UNKNOWN_KEY => { - Err(IpccKeyLookupError::UnknownKey { key }) - } - IPCC_KEYLOOKUP_NO_VALUE => { - Err(IpccKeyLookupError::NoValueForKey { key }) - } - IPCC_KEYLOOKUP_BUFFER_TOO_SMALL => { - Err(IpccKeyLookupError::BufferTooSmallForValue { key }) - } - _ => Err(IpccKeyLookupError::UnknownResultValue { - key, - result: kl.result, - }), - } - } -} - -// -------------------------------------------------------------------- -// Constants and structures from stlouis `usr/src/uts/oxide/sys/ipcc.h` -// -------------------------------------------------------------------- - -const IPCC_DEV: &str = "/dev/ipcc"; - -const IPCC_IOC: c_int = - ((b'i' as c_int) << 24) | ((b'c' as c_int) << 16) | ((b'c' as c_int) << 8); - -const IPCC_KEYLOOKUP: c_int = IPCC_IOC | 4; - -const IPCC_KEYLOOKUP_SUCCESS: u8 = 0; -const IPCC_KEYLOOKUP_UNKNOWN_KEY: u8 = 1; -const IPCC_KEYLOOKUP_NO_VALUE: u8 = 2; -const IPCC_KEYLOOKUP_BUFFER_TOO_SMALL: u8 = 3; - -#[derive(Debug, Clone, Copy)] -#[repr(C)] -struct IpccKeyLookup { - key: u8, - buflen: u16, - result: u8, - datalen: u16, - buf: *mut u8, -} diff --git a/ipcc-key-value/src/ioctl_common.rs b/ipcc-key-value/src/ioctl_common.rs deleted file mode 100644 index 670cc7bff2..0000000000 --- a/ipcc-key-value/src/ioctl_common.rs +++ /dev/null @@ -1,57 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -//! This module contains types shared between the real (illumos-only) -//! `crate::ioctl` module and the generic `crate::ioctl_stub` module. - -use std::io; -use thiserror::Error; - -/// IPCC keys; the source of truth for these is RFD 316 + the -/// `host-sp-messages` crate in hubris. -#[derive(Debug, Clone, Copy)] -#[repr(u8)] -pub enum IpccKey { - Ping = 0, - InstallinatorImageId = 1, -} - -#[derive(Debug, Error)] -pub enum IpccKeyLookupError { - #[error("IPCC key lookup ioctl failed for key {key:?}: {error}")] - IoctlFailed { key: IpccKey, error: io::Error }, - #[error("IPCC key lookup failed for key {key:?}: unknown key")] - UnknownKey { key: IpccKey }, - #[error("IPCC key lookup failed for key {key:?}: no value for key")] - NoValueForKey { key: IpccKey }, - #[error( - "IPCC key lookup failed for key {key:?}: buffer too small for value" - )] - BufferTooSmallForValue { key: IpccKey }, - #[error( - "IPCC key lookup failed for key {key:?}: unknown result value {result}" - )] - UnknownResultValue { key: IpccKey, result: u8 }, -} - -#[derive(Debug, Error)] -pub enum InstallinatorImageIdError { - #[error(transparent)] - IpccKeyLookupError(#[from] IpccKeyLookupError), - #[error("deserializing installinator image ID failed: {0}")] - DeserializationFailed(String), -} - -#[derive(Debug, Error)] -pub enum PingError { - #[error(transparent)] - IpccKeyLookupError(#[from] IpccKeyLookupError), - #[error("unexpected reply from SP (expected `pong`: {0:?})")] - UnexpectedReply(Vec), -} - -#[derive(Debug, Clone, Copy)] -pub struct Pong; diff --git a/ipcc-key-value/src/ioctl_stub.rs b/ipcc-key-value/src/ioctl_stub.rs deleted file mode 100644 index cbf54b3eb4..0000000000 --- a/ipcc-key-value/src/ioctl_stub.rs +++ /dev/null @@ -1,32 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -//! Stub definition of the `Ipcc` type for compiling (but not running) on -//! non-Oxide systems. - -use crate::InstallinatorImageId; -use crate::InstallinatorImageIdError; -use crate::PingError; -use crate::Pong; -use std::io; - -pub struct Ipcc {} - -impl Ipcc { - pub fn open() -> io::Result { - panic!("ipcc unavailable on this platform") - } - - pub fn ping(&self) -> Result { - panic!("ipcc unavailable on this platform") - } - - pub fn installinator_image_id( - &self, - ) -> Result { - panic!("ipcc unavailable on this platform") - } -} diff --git a/ipcc-key-value/Cargo.toml b/ipcc/Cargo.toml similarity index 91% rename from ipcc-key-value/Cargo.toml rename to ipcc/Cargo.toml index 04aea9f939..98a781ab86 100644 --- a/ipcc-key-value/Cargo.toml +++ b/ipcc/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ipcc-key-value" +name = "ipcc" version = "0.1.0" edition = "2021" license = "MPL-2.0" @@ -12,6 +12,7 @@ serde.workspace = true thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true +cfg-if.workspace = true [dev-dependencies] omicron-common = { workspace = true, features = ["testing"] } diff --git a/ipcc/build.rs b/ipcc/build.rs new file mode 100644 index 0000000000..a64133dac2 --- /dev/null +++ b/ipcc/build.rs @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/// This path is where Oxide specific libraries live on helios systems. +#[cfg(target_os = "illumos")] +static OXIDE_PLATFORM: &str = "/usr/platform/oxide/lib/amd64/"; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + #[cfg(target_os = "illumos")] + { + println!("cargo:rustc-link-arg=-Wl,-R{}", OXIDE_PLATFORM); + println!("cargo:rustc-link-search={}", OXIDE_PLATFORM); + } +} diff --git a/ipcc/src/ffi.rs b/ipcc/src/ffi.rs new file mode 100644 index 0000000000..420c1ddcde --- /dev/null +++ b/ipcc/src/ffi.rs @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +use std::ffi::{c_char, c_int, c_uint}; + +/// Opaque libipcc handle +#[repr(C)] +pub(crate) struct libipcc_handle_t { + _data: [u8; 0], + _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, +} + +/// Indicates that there was no error. Used as the initialized value when +/// calling into libipcc. +pub(crate) const LIBIPCC_ERR_OK: libipcc_err_t = 0; + +/// Indicates that there was a memory allocation error. The system error +/// contains the specific errno. +pub(crate) const LIBIPCC_ERR_NO_MEM: libipcc_err_t = 1; + +/// One of the function parameters does not pass validation. There will be more +/// detail available via libipcc_errmsg(). +pub(crate) const LIBIPCC_ERR_INVALID_PARAM: libipcc_err_t = 2; + +/// An internal error occurred. There will be more detail available via +/// libipcc_errmsg() and libipcc_syserr(). +pub(crate) const LIBIPCC_ERR_INTERNAL: libipcc_err_t = 3; + +/// The requested lookup key was not known to the SP. +pub(crate) const LIBIPCC_ERR_KEY_UNKNOWN: libipcc_err_t = 4; + +/// The value for the requested lookup key was too large for the +/// supplied buffer. +pub(crate) const LIBIPCC_ERR_KEY_BUFTOOSMALL: libipcc_err_t = 5; + +/// An attempt to write to a key failed because the key is read-only. +pub(crate) const LIBIPCC_ERR_KEY_READONLY: libipcc_err_t = 6; + +/// An attempt to write to a key failed because the passed value is too +/// long. +pub(crate) const LIBIPCC_ERR_KEY_VALTOOLONG: libipcc_err_t = 7; + +/// Compression or decompression failed. If appropriate, libipcc_syserr() will +/// return the Z_ error from zlib. +pub(crate) const LIBIPCC_ERR_KEY_ZERR: libipcc_err_t = 8; +pub(crate) type libipcc_err_t = c_uint; + +/// Maxium length of an error message retrieved by libipcc_errmsg(). +pub(crate) const LIBIPCC_ERR_LEN: usize = 1024; + +/// Flags that can be passed to libipcc when looking up a key. Today this is +/// used for looking up a compressed key, however nothing in the public API of +/// this crate takes advantage of this. +pub(crate) type libipcc_key_flag_t = ::std::os::raw::c_uint; + +#[link(name = "ipcc")] +extern "C" { + pub(crate) fn libipcc_init( + lihp: *mut *mut libipcc_handle_t, + libipcc_errp: *mut libipcc_err_t, + syserrp: *mut c_int, + errmsg: *const c_char, + errlen: usize, + ) -> bool; + pub(crate) fn libipcc_fini(lih: *mut libipcc_handle_t); + pub(crate) fn libipcc_err(lih: *mut libipcc_handle_t) -> libipcc_err_t; + pub(crate) fn libipcc_syserr(lih: *mut libipcc_handle_t) -> c_int; + pub(crate) fn libipcc_errmsg(lih: *mut libipcc_handle_t) -> *const c_char; + pub(crate) fn libipcc_keylookup( + lih: *mut libipcc_handle_t, + key: u8, + bufp: *mut *mut u8, + lenp: *mut usize, + flags: libipcc_key_flag_t, + ) -> bool; +} diff --git a/ipcc/src/handle.rs b/ipcc/src/handle.rs new file mode 100644 index 0000000000..91b71a6ce3 --- /dev/null +++ b/ipcc/src/handle.rs @@ -0,0 +1,129 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +use std::{ + ffi::{c_int, CStr, CString}, + ptr, +}; + +use crate::IpccError; +use crate::{ffi::*, IpccErrorInner}; + +pub struct IpccHandle(*mut libipcc_handle_t); + +impl Drop for IpccHandle { + fn drop(&mut self) { + unsafe { + libipcc_fini(self.0); + } + } +} +fn ipcc_fatal_error>( + context: C, + lerr: libipcc_err_t, + syserr: c_int, + errmsg: CString, +) -> IpccError { + let context = context.into(); + let syserr = if syserr == 0 { + "no system errno".to_string() + } else { + std::io::Error::from_raw_os_error(syserr).to_string() + }; + let inner = IpccErrorInner { + context, + errmsg: errmsg.to_string_lossy().into_owned(), + syserr, + }; + match lerr { + LIBIPCC_ERR_OK => panic!("called fatal on LIBIPCC_ERR_OK"), + LIBIPCC_ERR_NO_MEM => IpccError::NoMem(inner), + LIBIPCC_ERR_INVALID_PARAM => IpccError::InvalidParam(inner), + LIBIPCC_ERR_INTERNAL => IpccError::Internal(inner), + LIBIPCC_ERR_KEY_UNKNOWN => IpccError::KeyUnknown(inner), + LIBIPCC_ERR_KEY_BUFTOOSMALL => IpccError::KeyBufTooSmall(inner), + LIBIPCC_ERR_KEY_READONLY => IpccError::KeyReadonly(inner), + LIBIPCC_ERR_KEY_VALTOOLONG => IpccError::KeyValTooLong(inner), + LIBIPCC_ERR_KEY_ZERR => IpccError::KeyZerr(inner), + _ => IpccError::UnknownErr(inner), + } +} + +impl IpccHandle { + pub fn new() -> Result { + let mut ipcc_handle: *mut libipcc_handle_t = ptr::null_mut(); + // We subtract 1 from the length of the inital vector since CString::new + // will append a nul for us. + // Safety: Unwrapped because we guarantee that the supplied bytes + // contain no 0 bytes up front. + let errmsg = CString::new(vec![1; LIBIPCC_ERR_LEN - 1]).unwrap(); + let errmsg_len = errmsg.as_bytes().len(); + let errmsg_ptr = errmsg.into_raw(); + let mut lerr = LIBIPCC_ERR_OK; + let mut syserr = 0; + if !unsafe { + libipcc_init( + &mut ipcc_handle, + &mut lerr, + &mut syserr, + errmsg_ptr, + errmsg_len, + ) + } { + // Safety: CString::from_raw retakes ownership of a CString + // transferred to C via CString::into_raw. We are calling into_raw() + // above so it is safe to turn this back into it's owned variant. + let errmsg = unsafe { CString::from_raw(errmsg_ptr) }; + return Err(ipcc_fatal_error( + "Could not init libipcc handle", + lerr, + syserr, + errmsg, + )); + } + + Ok(IpccHandle(ipcc_handle)) + } + + fn fatal>(&self, context: C) -> IpccError { + let lerr = unsafe { libipcc_err(self.0) }; + let errmsg = unsafe { libipcc_errmsg(self.0) }; + // Safety: CStr::from_ptr is documented as safe if: + // 1. The pointer contains a valid null terminator at the end of + // the string + // 2. The pointer is valid for reads of bytes up to and including + // the null terminator + // 3. The memory referenced by the return CStr is not mutated for + // the duration of lifetime 'a + // + // (1) is true because this crate initializes space for an error message + // via CString::new which adds a terminator on our behalf. + // (2) should be guaranteed by libipcc itself since it is writing error + // messages into the CString backed buffer that we gave it. + // (3) We aren't currently mutating the memory referenced by the + // CStr, and we are creating an owned copy of the data immediately so + // that it can outlive the lifetime of the libipcc handle if needed. + let errmsg = unsafe { CStr::from_ptr(errmsg) }.to_owned(); + let syserr = unsafe { libipcc_syserr(self.0) }; + ipcc_fatal_error(context, lerr, syserr, errmsg) + } + + pub(crate) fn key_lookup( + &self, + key: u8, + buf: &mut [u8], + ) -> Result { + let mut lenp = buf.len(); + + if !unsafe { + libipcc_keylookup(self.0, key, &mut buf.as_mut_ptr(), &mut lenp, 0) + } { + return Err(self.fatal(format!("lookup of key {key} failed"))); + } + + Ok(lenp) + } +} diff --git a/ipcc/src/handle_stub.rs b/ipcc/src/handle_stub.rs new file mode 100644 index 0000000000..bc4b84b7fe --- /dev/null +++ b/ipcc/src/handle_stub.rs @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +use crate::IpccError; + +/// This stub and it's implementation are used for non-illumos platforms which +/// lack libipcc. +pub struct IpccHandle; + +impl IpccHandle { + pub fn new() -> Result { + panic!("ipcc unavailable on this platform") + } + + pub(crate) fn key_lookup( + &self, + _key: u8, + _buf: &mut [u8], + ) -> Result { + panic!("ipcc unavailable on this platform") + } +} diff --git a/ipcc-key-value/src/lib.rs b/ipcc/src/lib.rs similarity index 75% rename from ipcc-key-value/src/lib.rs rename to ipcc/src/lib.rs index 16e6685018..e997c51230 100644 --- a/ipcc-key-value/src/lib.rs +++ b/ipcc/src/lib.rs @@ -4,26 +4,28 @@ // Copyright 2023 Oxide Computer Company -//! Utilities for key/value pairs passed from the control plane to the SP -//! (through MGS) to the host (through the host/SP uart) via IPCC. +//! An interface to libipcc (inter-processor communications channel) which +//! currently supports looking up values stored in the SP by key. These +//! values are variously static, passed from the control plane to the SP +//! (through MGS) or set from userland via libipcc. +use cfg_if::cfg_if; use omicron_common::update::ArtifactHash; use serde::Deserialize; use serde::Serialize; +use thiserror::Error; use uuid::Uuid; -mod ioctl_common; -pub use ioctl_common::*; - -#[cfg(target_os = "illumos")] -mod ioctl; -#[cfg(target_os = "illumos")] -pub use ioctl::Ipcc; - -#[cfg(not(target_os = "illumos"))] -mod ioctl_stub; -#[cfg(not(target_os = "illumos"))] -pub use ioctl_stub::Ipcc; +cfg_if! { + if #[cfg(target_os = "illumos")] { + mod ffi; + mod handle; + use handle::IpccHandle; + } else { + mod handle_stub; + use handle_stub::IpccHandle; + } +} #[cfg(test)] use proptest::arbitrary::any; @@ -38,9 +40,9 @@ use proptest::strategy::Strategy; #[repr(u8)] pub enum Key { /// Always responds `"pong"`. - Ping = 0, + Ping = IpccKey::Ping as u8, /// The value should be an encoded [`InstallinatorImageId`]. - InstallinatorImageId = 1, + InstallinatorImageId = IpccKey::InstallinatorImageId as u8, } /// Description of the images `installinator` needs to fetch from a peer on the @@ -135,10 +137,84 @@ impl InstallinatorImageId { } } -// TODO Add ioctl wrappers? `installinator` is the only client for -// `Key::InstallinatorImageId`, but we might grow other keys for other clients, -// at which point we probably want all the ioctl wrapping to happen in one -// place. +#[derive(Debug, Error)] +pub enum InstallinatorImageIdError { + #[error(transparent)] + Ipcc(#[from] IpccError), + #[error("deserializing installinator image ID failed: {0}")] + DeserializationFailed(String), +} + +#[derive(Error, Debug)] +pub enum IpccError { + #[error("Memory allocation error")] + NoMem(#[source] IpccErrorInner), + #[error("Invalid parameter")] + InvalidParam(#[source] IpccErrorInner), + #[error("Internal error occurred")] + Internal(#[source] IpccErrorInner), + #[error("Requested lookup key was not known to the SP")] + KeyUnknown(#[source] IpccErrorInner), + #[error("Value for the requested lookup key was too large for the supplied buffer")] + KeyBufTooSmall(#[source] IpccErrorInner), + #[error("Attempted to write to read-only key")] + KeyReadonly(#[source] IpccErrorInner), + #[error("Attempted write to key failed because the value is too long")] + KeyValTooLong(#[source] IpccErrorInner), + #[error("Compression or decompression failed")] + KeyZerr(#[source] IpccErrorInner), + #[error("Unknown libipcc error")] + UnknownErr(#[source] IpccErrorInner), +} + +#[derive(Error, Debug)] +#[error("{context}: {errmsg} ({syserr})")] +pub struct IpccErrorInner { + pub context: String, + pub errmsg: String, + pub syserr: String, +} + +/// These are the IPCC keys we can look up. +/// NB: These keys match the definitions found in libipcc (RFD 316) and should +/// match the values in `[ipcc::Key]` one-to-one. +#[derive(Debug, Clone, Copy)] +#[allow(dead_code)] +#[repr(u8)] +enum IpccKey { + Ping = 0, + InstallinatorImageId = 1, + Inventory = 2, + System = 3, + Dtrace = 4, +} + +/// Interface to the inter-processor communications channel. +/// For more information see rfd 316. +pub struct Ipcc { + handle: IpccHandle, +} + +impl Ipcc { + /// Creates a new `Ipcc` instance. + pub fn new() -> Result { + let handle = IpccHandle::new()?; + Ok(Self { handle }) + } + + /// Returns the current `InstallinatorImageId`. + pub fn installinator_image_id( + &self, + ) -> Result { + let mut buf = [0; InstallinatorImageId::CBOR_SERIALIZED_SIZE]; + let n = self + .handle + .key_lookup(IpccKey::InstallinatorImageId as u8, &mut buf)?; + let id = InstallinatorImageId::deserialize(&buf[..n]) + .map_err(InstallinatorImageIdError::DeserializationFailed)?; + Ok(id) + } +} #[cfg(test)] mod tests { diff --git a/openapi/gateway.json b/openapi/gateway.json index 9eacbe122d..5961b670ed 100644 --- a/openapi/gateway.json +++ b/openapi/gateway.json @@ -1129,7 +1129,7 @@ "/sp/{type}/{slot}/ipcc/installinator-image-id": { "put": { "summary": "Set the installinator image ID the sled should use for recovery.", - "description": "This value can be read by the host via IPCC; see the `ipcc-key-value` crate.", + "description": "This value can be read by the host via IPCC; see the `ipcc` crate.", "operationId": "sp_installinator_image_id_set", "parameters": [ {