diff --git a/dev-tools/ls-apis/src/api_metadata.rs b/dev-tools/ls-apis/src/api_metadata.rs new file mode 100644 index 0000000000..b9b95bb75a --- /dev/null +++ b/dev-tools/ls-apis/src/api_metadata.rs @@ -0,0 +1,60 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Developer-maintained API metadata + +use crate::ClientPackageName; +use crate::DeploymentUnit; +use crate::ServerComponent; +use crate::ServerPackageName; +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AllApiMetadata { + apis: Vec, +} + +impl AllApiMetadata { + pub fn apis(&self) -> impl Iterator { + self.apis.iter() + } + + pub fn client_pkgnames(&self) -> impl Iterator { + self.apis().map(|api| &api.client_package_name) + } + + pub fn server_components(&self) -> impl Iterator { + self.apis().map(|api| &api.server_component) + } + + pub fn client_pkgname_lookup(&self, pkgname: &str) -> Option<&ApiMetadata> { + // XXX-dap this is worth optimizing but it would require a separate type + // -- this one would be the "raw" type. + self.apis.iter().find(|api| *api.client_package_name == pkgname) + } +} + +#[derive(Deserialize)] +pub struct ApiMetadata { + /// primary key for APIs is the client package name + pub client_package_name: ClientPackageName, + /// human-readable label for the API + pub label: String, + /// package name that the corresponding API lives in + // XXX-dap unused right now + pub server_package_name: ServerPackageName, + /// package name that the corresponding server lives in + pub server_component: ServerComponent, + /// name of the unit of deployment + group: Option, +} + +impl ApiMetadata { + pub fn group(&self) -> DeploymentUnit { + self.group + .clone() + .unwrap_or_else(|| (*self.server_component).clone().into()) + } +} diff --git a/dev-tools/ls-apis/src/cargo.rs b/dev-tools/ls-apis/src/cargo.rs new file mode 100644 index 0000000000..9c44cf2fd1 --- /dev/null +++ b/dev-tools/ls-apis/src/cargo.rs @@ -0,0 +1,275 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Extract API metadata from Cargo metadata + +use crate::ClientPackageName; +use anyhow::{anyhow, ensure, Context, Result}; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use cargo_metadata::DependencyKind; +use cargo_metadata::Metadata; +use cargo_metadata::Package; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::fmt::Display; +use url::Url; + +pub struct Workspace { + name: String, + all_packages: BTreeMap, // XXX-dap memory usage + progenitor_clients: BTreeSet, +} + +impl Workspace { + pub fn load(name: &str, extra_repos: Option<&Utf8Path>) -> Result { + eprintln!("loading metadata for workspace {name}"); + + let mut cmd = cargo_metadata::MetadataCommand::new(); + if let Some(extra_repos) = extra_repos { + cmd.manifest_path(extra_repos.join(name).join("Cargo.toml")); + } + let metadata = cmd.exec().context("loading metadata")?; + + // Find all packages with a direct non-dev, non-build dependency on + // "progenitor". These generally ought to be suffixed with "-client". + let progenitor_clients = direct_dependents(&metadata, "progenitor")? + .into_iter() + .filter_map(|mypkg| { + if mypkg.name.ends_with("-client") { + Some(ClientPackageName::from(mypkg.name)) + } else { + eprintln!("ignoring apparent non-client: {}", mypkg.name); + None + } + }) + .collect(); + + let all_packages = metadata + .packages + .iter() + .map(|p| (p.name.clone(), p.clone())) + .collect(); + + Ok(Workspace { + name: name.to_owned(), + all_packages, + progenitor_clients, + }) + } + + pub fn client_packages(&self) -> impl Iterator { + self.progenitor_clients.iter() + } + + pub fn find_package(&self, pkgname: &str) -> Option<&Package> { + self.all_packages.get(pkgname) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn walk_required_deps_recursively( + &self, + root: &Package, + func: &mut dyn FnMut(&Package, &Package), + ) -> Result<()> { + let mut remaining = vec![root]; + let mut seen: BTreeSet = BTreeSet::new(); + + while let Some(next) = remaining.pop() { + for d in &next.dependencies { + if seen.contains(&d.name) { + continue; + } + + seen.insert(d.name.clone()); + + if d.optional { + continue; + } + + if !matches!( + d.kind, + DependencyKind::Normal | DependencyKind::Build + ) { + continue; + } + + let pkg = self.find_package(&d.name).ok_or_else(|| { + anyhow!( + "package {:?} has dependency {:?} not in workspace \ + metadata", + next.name, + d.name + ) + })?; + func(next, pkg); + remaining.push(pkg); + } + } + + Ok(()) + } +} + +pub struct MyPackage { + name: String, + location: MyPackageLocation, +} + +impl MyPackage { + fn new(workspace: &Metadata, pkg: &Package) -> Result { + // Figure out where this thing is. It's generally one of two places: + // (1) In a remote repository. In that case, it will have a "source" + // property that's the URL to a package. + // (2) Inside this workspace. In that case, it will have no "source", + // but it will have a manifest_path that's inside this workspace. + let location = if let Some(source) = &pkg.source { + let source_repo_str = &source.repr; + let repo_name = + source_repo_name(source_repo_str).with_context(|| { + format!("parsing source {:?}", source_repo_str) + })?; + + // Figuring out where in that repo the package lives is trickier. + // Here we encode some knowledge of where Cargo would have checked + // out the repo. + let cargo_home = std::env::var("CARGO_HOME") + .context("looking up CARGO_HOME in environment")?; + let cargo_path = + Utf8PathBuf::from(cargo_home).join("git").join("checkouts"); + let path = + pkg.manifest_path.strip_prefix(&cargo_path).map_err(|_| { + anyhow!( + "expected non-local package manifest path ({:?}) to be \ + under {:?}", + pkg.manifest_path, + cargo_path, + ) + })?; + + // There should be two extra leading directory components here. + // Remove them. We've gone too far if the file name isn't right + // after that. + let tail: Utf8PathBuf = path.components().skip(2).collect(); + ensure!( + tail.file_name() == Some("Cargo.toml"), + "unexpected non-local package manifest path: {:?}", + pkg.manifest_path + ); + + let path = tail + .parent() + .ok_or_else(|| { + anyhow!( + "unexpected non-local package manifest path: {:?}", + pkg.manifest_path + ) + })? + .to_owned(); + MyPackageLocation::RemoteRepo { oxide_github_repo: repo_name, path } + } else { + let manifest_path = &pkg.manifest_path; + let relative_path = manifest_path + .strip_prefix(&workspace.workspace_root) + .map_err(|_| { + anyhow!( + "no \"source\", so assuming this package is inside this \ + repo, but its manifest path ({:?}) is not under the \ + workspace root ({:?})", + manifest_path, + &workspace.workspace_root + ) + })?; + // XXX-dap commonize with above + ensure!( + relative_path.file_name() == Some("Cargo.toml"), + "unexpected manifest path for local package: {:?}", + manifest_path + ); + let path = relative_path + .parent() + .ok_or_else(|| { + anyhow!( + "unexpected manifest path for local package: {:?}", + manifest_path + ) + })? + .to_owned(); + + MyPackageLocation::Omicron { path } + }; + + Ok(MyPackage { name: pkg.name.clone(), location }) + } +} + +pub enum MyPackageLocation { + Omicron { path: Utf8PathBuf }, + RemoteRepo { oxide_github_repo: String, path: Utf8PathBuf }, +} + +impl Display for MyPackageLocation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MyPackageLocation::Omicron { path } => { + write!(f, "omicron:{}", path) + } + MyPackageLocation::RemoteRepo { oxide_github_repo, path } => { + write!(f, "{}:{}", oxide_github_repo, path) + } + } + } +} + +fn source_repo_name(raw: &str) -> Result { + let repo_url = + Url::parse(raw).with_context(|| format!("parsing {:?}", raw))?; + ensure!(repo_url.scheme() == "git+https", "unsupported URL scheme",); + ensure!( + matches!(repo_url.host_str(), Some(h) if h == "github.com"), + "unexpected URL host (expected \"github.com\")", + ); + let path_segments: Vec<_> = repo_url + .path_segments() + .ok_or_else(|| anyhow!("expected URL to contain path segments"))? + .collect(); + ensure!( + path_segments.len() == 2, + "expected exactly two path segments in URL", + ); + ensure!( + path_segments[0] == "oxidecomputer", + "expected repo under Oxide's GitHub organization", + ); + + Ok(path_segments[1].to_string()) +} + +fn direct_dependents( + workspace: &Metadata, + pkg_name: &str, +) -> Result> { + workspace + .packages + .iter() + .filter_map(|pkg| { + if pkg.dependencies.iter().any(|dep| { + matches!( + dep.kind, + DependencyKind::Normal | DependencyKind::Build + ) && dep.name == pkg_name + }) { + Some( + MyPackage::new(workspace, pkg) + .with_context(|| format!("package {:?}", pkg.name)), + ) + } else { + None + } + }) + .collect() +} diff --git a/dev-tools/ls-apis/src/lib.rs b/dev-tools/ls-apis/src/lib.rs new file mode 100644 index 0000000000..2b949f7e71 --- /dev/null +++ b/dev-tools/ls-apis/src/lib.rs @@ -0,0 +1,53 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Compile information about Progenitor-based APIs + +use serde::Deserialize; + +mod api_metadata; +mod cargo; + +pub use api_metadata::AllApiMetadata; +pub use api_metadata::ApiMetadata; +pub use cargo::Workspace; + +#[macro_use] +extern crate newtype_derive; + +#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] +#[serde(transparent)] +pub struct ClientPackageName(String); +NewtypeDebug! { () pub struct ClientPackageName(String); } +NewtypeDeref! { () pub struct ClientPackageName(String); } +NewtypeDerefMut! { () pub struct ClientPackageName(String); } +NewtypeDisplay! { () pub struct ClientPackageName(String); } +NewtypeFrom! { () pub struct ClientPackageName(String); } + +#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] +#[serde(transparent)] +pub struct DeploymentUnit(String); +NewtypeDebug! { () pub struct DeploymentUnit(String); } +NewtypeDeref! { () pub struct DeploymentUnit(String); } +NewtypeDerefMut! { () pub struct DeploymentUnit(String); } +NewtypeDisplay! { () pub struct DeploymentUnit(String); } +NewtypeFrom! { () pub struct DeploymentUnit(String); } + +#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] +#[serde(transparent)] +pub struct ServerPackageName(String); +NewtypeDebug! { () pub struct ServerPackageName(String); } +NewtypeDeref! { () pub struct ServerPackageName(String); } +NewtypeDerefMut! { () pub struct ServerPackageName(String); } +NewtypeDisplay! { () pub struct ServerPackageName(String); } +NewtypeFrom! { () pub struct ServerPackageName(String); } + +#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] +#[serde(transparent)] +pub struct ServerComponent(String); +NewtypeDebug! { () pub struct ServerComponent(String); } +NewtypeDeref! { () pub struct ServerComponent(String); } +NewtypeDerefMut! { () pub struct ServerComponent(String); } +NewtypeDisplay! { () pub struct ServerComponent(String); } +NewtypeFrom! { () pub struct ServerComponent(String); } diff --git a/dev-tools/ls-apis/src/main.rs b/dev-tools/ls-apis/src/main.rs index cb9577bb3d..d061274dd1 100644 --- a/dev-tools/ls-apis/src/main.rs +++ b/dev-tools/ls-apis/src/main.rs @@ -14,20 +14,17 @@ use anyhow::{anyhow, ensure, Context, Result}; use camino::Utf8Path; use camino::Utf8PathBuf; -use cargo_metadata::DependencyKind; -use cargo_metadata::Metadata; use cargo_metadata::Package; use clap::{Args, Parser, Subcommand}; +use omicron_ls_apis::AllApiMetadata; +use omicron_ls_apis::ClientPackageName; +use omicron_ls_apis::DeploymentUnit; +use omicron_ls_apis::ServerComponent; +use omicron_ls_apis::Workspace; use petgraph::dot::Dot; use serde::de::DeserializeOwned; -use serde::Deserialize; use std::collections::BTreeMap; use std::collections::BTreeSet; -use std::fmt::Display; -use url::Url; - -#[macro_use] -extern crate newtype_derive; #[derive(Parser)] #[command( @@ -89,7 +86,6 @@ impl TryFrom<&LsApis> for LoadArgs { let api_manifest_path = args.api_manifest.clone().unwrap_or_else(|| { - // XXX TODO-cleanup can these be done in one join call? self_manifest_dir .join("..") .join("..") @@ -108,42 +104,6 @@ impl TryFrom<&LsApis> for LoadArgs { } } -#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] -#[serde(transparent)] -pub struct ClientPackageName(String); -NewtypeDebug! { () pub struct ClientPackageName(String); } -NewtypeDeref! { () pub struct ClientPackageName(String); } -NewtypeDerefMut! { () pub struct ClientPackageName(String); } -NewtypeDisplay! { () pub struct ClientPackageName(String); } -NewtypeFrom! { () pub struct ClientPackageName(String); } - -#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] -#[serde(transparent)] -pub struct DeploymentUnit(String); -NewtypeDebug! { () pub struct DeploymentUnit(String); } -NewtypeDeref! { () pub struct DeploymentUnit(String); } -NewtypeDerefMut! { () pub struct DeploymentUnit(String); } -NewtypeDisplay! { () pub struct DeploymentUnit(String); } -NewtypeFrom! { () pub struct DeploymentUnit(String); } - -#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] -#[serde(transparent)] -pub struct ServerPackageName(String); -NewtypeDebug! { () pub struct ServerPackageName(String); } -NewtypeDeref! { () pub struct ServerPackageName(String); } -NewtypeDerefMut! { () pub struct ServerPackageName(String); } -NewtypeDisplay! { () pub struct ServerPackageName(String); } -NewtypeFrom! { () pub struct ServerPackageName(String); } - -#[derive(Clone, Deserialize, Ord, PartialOrd, Eq, PartialEq)] -#[serde(transparent)] -pub struct ServerComponent(String); -NewtypeDebug! { () pub struct ServerComponent(String); } -NewtypeDeref! { () pub struct ServerComponent(String); } -NewtypeDerefMut! { () pub struct ServerComponent(String); } -NewtypeDisplay! { () pub struct ServerComponent(String); } -NewtypeFrom! { () pub struct ServerComponent(String); } - struct Apis { server_component_units: BTreeMap, unit_server_components: BTreeMap>, @@ -369,7 +329,7 @@ impl ApisHelper { api_metadata.client_pkgnames().collect(); let mut errors = Vec::new(); for (_, workspace) in &workspaces { - for client_pkgname in &workspace.progenitor_clients { + for client_pkgname in workspace.client_packages() { if api_metadata.client_pkgname_lookup(client_pkgname).is_some() { // It's possible that we will find multiple references @@ -423,309 +383,6 @@ impl ApisHelper { } } -struct MyPackage { - name: String, - location: MyPackageLocation, -} - -impl MyPackage { - fn new(workspace: &Metadata, pkg: &Package) -> Result { - // Figure out where this thing is. It's generally one of two places: - // (1) In a remote repository. In that case, it will have a "source" - // property that's the URL to a package. - // (2) Inside this workspace. In that case, it will have no "source", - // but it will have a manifest_path that's inside this workspace. - let location = if let Some(source) = &pkg.source { - let source_repo_str = &source.repr; - let repo_name = - source_repo_name(source_repo_str).with_context(|| { - format!("parsing source {:?}", source_repo_str) - })?; - - // Figuring out where in that repo the package lives is trickier. - // Here we encode some knowledge of where Cargo would have checked - // out the repo. - let cargo_home = std::env::var("CARGO_HOME") - .context("looking up CARGO_HOME in environment")?; - let cargo_path = - Utf8PathBuf::from(cargo_home).join("git").join("checkouts"); - let path = - pkg.manifest_path.strip_prefix(&cargo_path).map_err(|_| { - anyhow!( - "expected non-local package manifest path ({:?}) to be \ - under {:?}", - pkg.manifest_path, - cargo_path, - ) - })?; - - // There should be two extra leading directory components here. - // Remove them. We've gone too far if the file name isn't right - // after that. - let tail: Utf8PathBuf = path.components().skip(2).collect(); - ensure!( - tail.file_name() == Some("Cargo.toml"), - "unexpected non-local package manifest path: {:?}", - pkg.manifest_path - ); - - let path = tail - .parent() - .ok_or_else(|| { - anyhow!( - "unexpected non-local package manifest path: {:?}", - pkg.manifest_path - ) - })? - .to_owned(); - MyPackageLocation::RemoteRepo { oxide_github_repo: repo_name, path } - } else { - let manifest_path = &pkg.manifest_path; - let relative_path = manifest_path - .strip_prefix(&workspace.workspace_root) - .map_err(|_| { - anyhow!( - "no \"source\", so assuming this package is inside this \ - repo, but its manifest path ({:?}) is not under the \ - workspace root ({:?})", - manifest_path, - &workspace.workspace_root - ) - })?; - // XXX-dap commonize with above - ensure!( - relative_path.file_name() == Some("Cargo.toml"), - "unexpected manifest path for local package: {:?}", - manifest_path - ); - let path = relative_path - .parent() - .ok_or_else(|| { - anyhow!( - "unexpected manifest path for local package: {:?}", - manifest_path - ) - })? - .to_owned(); - - MyPackageLocation::Omicron { path } - }; - - Ok(MyPackage { name: pkg.name.clone(), location }) - } -} - -enum MyPackageLocation { - Omicron { path: Utf8PathBuf }, - RemoteRepo { oxide_github_repo: String, path: Utf8PathBuf }, -} - -impl Display for MyPackageLocation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - MyPackageLocation::Omicron { path } => { - write!(f, "omicron:{}", path) - } - MyPackageLocation::RemoteRepo { oxide_github_repo, path } => { - write!(f, "{}:{}", oxide_github_repo, path) - } - } - } -} - -fn source_repo_name(raw: &str) -> Result { - let repo_url = - Url::parse(raw).with_context(|| format!("parsing {:?}", raw))?; - ensure!(repo_url.scheme() == "git+https", "unsupported URL scheme",); - ensure!( - matches!(repo_url.host_str(), Some(h) if h == "github.com"), - "unexpected URL host (expected \"github.com\")", - ); - let path_segments: Vec<_> = repo_url - .path_segments() - .ok_or_else(|| anyhow!("expected URL to contain path segments"))? - .collect(); - ensure!( - path_segments.len() == 2, - "expected exactly two path segments in URL", - ); - ensure!( - path_segments[0] == "oxidecomputer", - "expected repo under Oxide's GitHub organization", - ); - - Ok(path_segments[1].to_string()) -} - -fn direct_dependents( - workspace: &Metadata, - pkg_name: &str, -) -> Result> { - workspace - .packages - .iter() - .filter_map(|pkg| { - if pkg.dependencies.iter().any(|dep| { - matches!( - dep.kind, - DependencyKind::Normal | DependencyKind::Build - ) && dep.name == pkg_name - }) { - Some( - MyPackage::new(workspace, pkg) - .with_context(|| format!("package {:?}", pkg.name)), - ) - } else { - None - } - }) - .collect() -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct AllApiMetadata { - apis: Vec, -} - -impl AllApiMetadata { - fn apis(&self) -> impl Iterator { - self.apis.iter() - } - - fn client_pkgnames(&self) -> impl Iterator { - self.apis().map(|api| &api.client_package_name) - } - - fn server_components(&self) -> impl Iterator { - self.apis().map(|api| &api.server_component) - } - - fn client_pkgname_lookup(&self, pkgname: &str) -> Option<&ApiMetadata> { - // XXX-dap this is worth optimizing but it would require a separate type - // -- this one would be the "raw" type. - self.apis.iter().find(|api| *api.client_package_name == pkgname) - } -} - -#[derive(Deserialize)] -struct ApiMetadata { - /// primary key for APIs is the client package name - client_package_name: ClientPackageName, - /// human-readable label for the API - label: String, - /// package name that the corresponding API lives in - // XXX-dap unused right now - server_package_name: ServerPackageName, - /// package name that the corresponding server lives in - server_component: ServerComponent, - /// name of the unit of deployment - group: Option, -} - -impl ApiMetadata { - fn group(&self) -> DeploymentUnit { - self.group - .clone() - .unwrap_or_else(|| (*self.server_component).clone().into()) - } -} - -struct Workspace { - name: String, - all_packages: BTreeMap, // XXX-dap memory usage - progenitor_clients: BTreeSet, -} - -impl Workspace { - pub fn load(name: &str, extra_repos: Option<&Utf8Path>) -> Result { - eprintln!("loading metadata for workspace {name}"); - - let mut cmd = cargo_metadata::MetadataCommand::new(); - if let Some(extra_repos) = extra_repos { - cmd.manifest_path(extra_repos.join(name).join("Cargo.toml")); - } - let metadata = cmd.exec().context("loading metadata")?; - - // Find all packages with a direct non-dev, non-build dependency on - // "progenitor". These generally ought to be suffixed with "-client". - let progenitor_clients = direct_dependents(&metadata, "progenitor")? - .into_iter() - .filter_map(|mypkg| { - if mypkg.name.ends_with("-client") { - Some(ClientPackageName::from(mypkg.name)) - } else { - eprintln!("ignoring apparent non-client: {}", mypkg.name); - None - } - }) - .collect(); - - let all_packages = metadata - .packages - .iter() - .map(|p| (p.name.clone(), p.clone())) - .collect(); - - Ok(Workspace { - name: name.to_owned(), - all_packages, - progenitor_clients, - }) - } - - pub fn find_package(&self, pkgname: &str) -> Option<&Package> { - self.all_packages.get(pkgname) - } - - pub fn name(&self) -> &str { - &self.name - } - - fn walk_required_deps_recursively( - &self, - root: &Package, - func: &mut dyn FnMut(&Package, &Package), - ) -> Result<()> { - let mut remaining = vec![root]; - let mut seen: BTreeSet = BTreeSet::new(); - - while let Some(next) = remaining.pop() { - for d in &next.dependencies { - if seen.contains(&d.name) { - continue; - } - - seen.insert(d.name.clone()); - - if d.optional { - continue; - } - - if !matches!( - d.kind, - DependencyKind::Normal | DependencyKind::Build - ) { - continue; - } - - let pkg = self.find_package(&d.name).ok_or_else(|| { - anyhow!( - "package {:?} has dependency {:?} not in workspace \ - metadata", - next.name, - d.name - ) - })?; - func(next, pkg); - remaining.push(pkg); - } - } - - Ok(()) - } -} - fn parse_toml_file(path: &Utf8Path) -> Result { let s = std::fs::read_to_string(path) .with_context(|| format!("read {:?}", path))?;