diff --git a/.github/workflows/crates-io.yml b/.github/workflows/crates-io.yml index c899782ddb9..eb7c0fd1b0f 100644 --- a/.github/workflows/crates-io.yml +++ b/.github/workflows/crates-io.yml @@ -7,6 +7,9 @@ on: description: "If publish packages" type: boolean default: false + version: + description: "Workspace version to publish" + type: string pull_request: branches: [master] @@ -29,8 +32,8 @@ jobs: uses: dsherret/rust-toolchain-file@v1 - name: "Check packages" - run: cargo run --release -p crates-io-manager check + run: cargo run --release -p crates-io check - name: "Publish packages" if: ${{ github.event_name == 'workflow_dispatch' && inputs.publish }} - run: cargo run --release -p crates-io-manager publish + run: cargo run --release -p crates-io publish -v ${{ inputs.version }} diff --git a/Cargo.lock b/Cargo.lock index eed45194824..25cf17816a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1040,16 +1040,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "cargo_toml" -version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599aa35200ffff8f04c1925aa1acc92fa2e08874379ef42e210a80e527e60838" -dependencies = [ - "serde", - "toml 0.7.8", -] - [[package]] name = "cc" version = "1.0.83" @@ -1611,16 +1601,15 @@ dependencies = [ ] [[package]] -name = "crates-io-manager" +name = "crates-io" version = "1.0.3" dependencies = [ "anyhow", "cargo_metadata 0.18.1", - "cargo_toml", "clap 4.4.11", "reqwest", "serde", - "toml 0.7.8", + "toml_edit 0.21.0", ] [[package]] @@ -13228,14 +13217,14 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.15", ] [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -13253,6 +13242,17 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/Cargo.toml b/Cargo.toml index 2fd458ec51e..3b0ba9fda32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -445,9 +445,7 @@ fail = "0.5" # gear scale-value = "^0.12" # gsdk heck = "0.4.1" # gsdk-api-gen etc = "0.1.16" # gcli -cargo_toml = "0.15.3" # crates-io -crates-io = "0.37.0" # crates-io -curl = "0.4.44" # crates-io +toml_edit = "0.21.0" # crates-io scale-decode = "0.9.0" # gsdk directories = "5.0.1" # utils/key-finder num-traits = { version = "0.2", default-features = false } # gear-core diff --git a/utils/crates-io/Cargo.toml b/utils/crates-io/Cargo.toml index cdac3d49a29..6f5c850fce7 100644 --- a/utils/crates-io/Cargo.toml +++ b/utils/crates-io/Cargo.toml @@ -1,13 +1,12 @@ [package] -name = "crates-io-manager" +name = "crates-io" version.workspace = true edition.workspace = true [dependencies] anyhow.workspace = true cargo_metadata.workspace = true -cargo_toml.workspace = true clap = { workspace = true, features = ["derive"] } -reqwest = { workspace = true, features = ["blocking", "json", "default-tls"] } serde = { workspace = true, features = ["derive"] } -toml.workspace = true +reqwest = { workspace = true, features = ["blocking", "json", "default-tls"] } +toml_edit.workspace = true diff --git a/utils/crates-io/src/lib.rs b/utils/crates-io/src/lib.rs index 209e0d5d711..454b60615c0 100644 --- a/utils/crates-io/src/lib.rs +++ b/utils/crates-io/src/lib.rs @@ -21,10 +21,9 @@ mod manifest; mod publisher; -pub mod rename; mod version; -pub use self::{manifest::ManifestWithPath, publisher::Publisher, version::verify}; +pub use self::{manifest::Manifest, publisher::Publisher, version::verify}; use anyhow::Result; use std::process::{Command, ExitStatus}; diff --git a/utils/crates-io/src/main.rs b/utils/crates-io/src/main.rs index 1cf6443480e..15fc0b0fe85 100644 --- a/utils/crates-io/src/main.rs +++ b/utils/crates-io/src/main.rs @@ -20,15 +20,21 @@ use anyhow::Result; use clap::Parser; -use crates_io_manager::Publisher; +use crates_io::Publisher; /// The command to run. #[derive(Clone, Debug, Parser)] enum Command { + /// Build manifests for packages that to be published. + Build, /// Check packages that to be published. Check, /// Publish packages. - Publish, + Publish { + /// The version to publish. + #[clap(long, short)] + version: Option, + }, } /// Gear crates-io manager command line interface @@ -44,9 +50,13 @@ pub struct Opt { fn main() -> Result<()> { let Opt { command } = Opt::parse(); - let publisher = Publisher::new()?.build()?; + let publisher = Publisher::new()?; match command { - Command::Check => publisher.check(), - Command::Publish => publisher.publish(), + Command::Check => publisher.build(None)?.check(), + Command::Publish { version } => publisher.build(version)?.publish(), + Command::Build => { + publisher.build(None)?; + Ok(()) + } } } diff --git a/utils/crates-io/src/manifest.rs b/utils/crates-io/src/manifest.rs index ca1d62f68e5..942a43118cc 100644 --- a/utils/crates-io/src/manifest.rs +++ b/utils/crates-io/src/manifest.rs @@ -18,24 +18,26 @@ //! Manifest utils for crates-io-manager -use anyhow::Result; -use cargo_toml::{Manifest, Value}; -use std::{ - fs, - path::{Path, PathBuf}, -}; +use anyhow::{anyhow, Result}; +use cargo_metadata::Package; +use std::{fs, path::PathBuf}; +use toml_edit::Document; + +use crate::version; + +const WORKSPACE_NAME: &str = "__gear_workspace"; /// Cargo manifest with path -pub struct ManifestWithPath { +pub struct Manifest { /// Crate name pub name: String, /// Cargo manifest - pub manifest: Manifest, + pub manifest: Document, /// Path of the manifest pub path: PathBuf, } -impl ManifestWithPath { +impl Manifest { /// Get the workspace manifest pub fn workspace() -> Result { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -46,23 +48,141 @@ impl ManifestWithPath { .canonicalize()?; Ok(Self { - name: "__gear_workspace".into(), - manifest: Manifest::from_path(&path)?, + name: WORKSPACE_NAME.to_string(), + manifest: fs::read_to_string(&path)?.parse()?, path, }) } /// Complete the manifest of the specified crate from - /// the current manifest - pub fn manifest(&self, path: impl AsRef) -> Result { - let mut manifest = Manifest::::from_slice_with_metadata(&fs::read(&path)?)?; - manifest - .complete_from_path_and_workspace(path.as_ref(), Some((&self.manifest, &self.path)))?; + /// the workspace manifest + pub fn manifest(&self, pkg: &Package) -> Result { + self.ensure_workspace()?; + + // Complete documentation as from + let mut manifest: Document = fs::read_to_string(&pkg.manifest_path)?.parse()?; + let name = pkg.name.clone(); + manifest["package"]["documentation"] = toml_edit::value(format!("https://docs.rs/{name}")); Ok(Self { - name: manifest.package.clone().map(|p| p.name).unwrap_or_default(), + name, manifest, - path: path.as_ref().to_path_buf(), + path: pkg.manifest_path.clone().into(), }) } + + /// complete the versions of the specified crates + pub fn complete_versions(&mut self, index: &[&str]) -> Result<()> { + self.ensure_workspace()?; + + let version = self.manifest["workspace"]["package"]["version"] + .clone() + .as_str() + .ok_or_else(|| anyhow!("Could not find version in workspace manifest"))? + .to_string(); + + let Some(deps) = self.manifest["workspace"]["dependencies"].as_table_mut() else { + return Err(anyhow!( + "Failed to parse dependencies from workspace {}", + self.path.display() + )); + }; + + for (key, dep) in deps.iter_mut() { + let name = key.get(); + if !index.contains(&name) { + continue; + } + + dep["version"] = toml_edit::value(version.clone()); + } + + self.rename_deps()?; + Ok(()) + } + + /// Set version for the workspace. + pub fn with_version(mut self, version: Option) -> Result { + self.ensure_workspace()?; + + let version = if let Some(version) = version { + version + } else { + self.version()? + "-" + &version::hash()? + }; + + self.manifest["workspace"]["package"]["version"] = toml_edit::value(version); + + Ok(self) + } + + /// Get version from the current manifest. + pub fn version(&self) -> Result { + self.ensure_workspace()?; + + Ok(self.manifest["workspace"]["package"]["version"] + .as_str() + .ok_or_else(|| { + anyhow!( + "Could not find version in workspace manifest: {}", + self.path.display() + ) + })? + .to_string()) + } + + /// Write manifest to disk. + pub fn write(&self) -> Result<()> { + fs::write(&self.path, self.manifest.to_string()).map_err(Into::into) + } + + /// Rename dependencies + fn rename_deps(&mut self) -> Result<()> { + self.ensure_workspace()?; + + let Some(deps) = self.manifest["workspace"]["dependencies"].as_table_like_mut() else { + return Ok(()); + }; + + for (name, dep) in deps.iter_mut() { + let name = name.get(); + if !name.starts_with("sp-") { + continue; + } + + // Format dotted values into inline table. + if let Some(table) = dep.as_table_mut() { + table.remove("branch"); + table.remove("git"); + table.remove("workspace"); + + if name == "sp-arithmetic" { + // NOTE: the required version of sp-arithmetic is 6.0.0 in + // git repo, but 7.0.0 in crates.io, so we need to fix it. + table.insert("version", toml_edit::value("7.0.0")); + } + + // Force the dep to be inline table in case of losing + // documentation. + let mut inline = table.clone().into_inline_table(); + inline.fmt(); + *dep = toml_edit::value(inline); + }; + } + + Ok(()) + } + + /// Ensure the current function is called on the workspace manifest + /// + /// TODO: remove this interface after #3565 + fn ensure_workspace(&self) -> Result<()> { + if self.name != WORKSPACE_NAME { + return Err(anyhow!( + "This method can only be called on the workspace manifest" + )); + } + + Ok(()) + } } diff --git a/utils/crates-io/src/publisher.rs b/utils/crates-io/src/publisher.rs index 11211fd1215..0155eea94c8 100644 --- a/utils/crates-io/src/publisher.rs +++ b/utils/crates-io/src/publisher.rs @@ -18,18 +18,15 @@ //! Packages publisher -use crate::{rename, ManifestWithPath, PACKAGES, SAFE_DEPENDENCIES, STACKED_DEPENDENCIES}; +use crate::{Manifest, PACKAGES, SAFE_DEPENDENCIES, STACKED_DEPENDENCIES}; use anyhow::Result; -use cargo_metadata::{Metadata, MetadataCommand}; -use std::{ - collections::{BTreeMap, HashMap}, - fs, -}; +use cargo_metadata::{Metadata, MetadataCommand, Package}; +use std::collections::{BTreeMap, HashMap}; /// crates-io packages publisher. pub struct Publisher { metadata: Metadata, - graph: BTreeMap, ManifestWithPath>, + graph: BTreeMap, Manifest>, index: HashMap, } @@ -62,28 +59,35 @@ impl Publisher { /// 1. Replace git dependencies to crates-io dependencies. /// 2. Rename version of all local packages /// 3. Patch dependencies if needed - pub fn build(mut self) -> Result { - let workspace = ManifestWithPath::workspace()?; - for p in &self.metadata.packages { - if !self.index.contains_key(&p.name) { + pub fn build(mut self, version: Option) -> Result { + let index = self.index.keys().map(|s| s.as_ref()).collect::>(); + let mut workspace = Manifest::workspace()?.with_version(version)?; + let version = workspace.version()?; + + for pkg @ Package { name, .. } in &self.metadata.packages { + if !index.contains(&name.as_ref()) { continue; } - let version = p.version.to_string(); - println!("Verifying {}@{} ...", &p.name, &version); - if crate::verify(&p.name, &version)? { - println!("Package {}@{} already published .", &p.name, &version); + println!("Verifying {}@{} ...", &name, &version); + if crate::verify(name, &version)? { + println!("Package {}@{} already published .", &name, &version); continue; } - let mut manifest = workspace.manifest(&p.manifest_path)?; - rename::package(p, &mut manifest.manifest)?; - rename::deps(&mut manifest.manifest, self.index.keys().collect(), version)?; - self.graph - .insert(self.index.get(&p.name).cloned(), manifest); + .insert(self.index.get(name).cloned(), workspace.manifest(pkg)?); } + // Flush new manifests to disk + workspace.complete_versions(&index)?; + let mut manifests = self.graph.values().collect::>(); + manifests.push(&workspace); + manifests + .iter() + .map(|m| m.write()) + .collect::>>()?; + Ok(self) } @@ -91,10 +95,8 @@ impl Publisher { /// /// TODO: Complete the check process (#3565) pub fn check(&self) -> Result<()> { - self.flush()?; - let mut failed = Vec::new(); - for ManifestWithPath { path, name, .. } in self.graph.values() { + for Manifest { path, name, .. } in self.graph.values() { if !PACKAGES.contains(&name.as_str()) { continue; } @@ -115,9 +117,7 @@ impl Publisher { /// Publish packages pub fn publish(&self) -> Result<()> { - self.flush()?; - - for ManifestWithPath { path, .. } in self.graph.values() { + for Manifest { path, .. } in self.graph.values() { println!("Publishing {path:?}"); let status = crate::publish(&path.to_string_lossy())?; if !status.success() { @@ -127,13 +127,4 @@ impl Publisher { Ok(()) } - - /// Flush new manifests to disk - fn flush(&self) -> Result<()> { - for ManifestWithPath { path, manifest, .. } in self.graph.values() { - fs::write(path, toml::to_string_pretty(&manifest)?)?; - } - - Ok(()) - } } diff --git a/utils/crates-io/src/rename.rs b/utils/crates-io/src/rename.rs deleted file mode 100644 index 46eadc9a3fd..00000000000 --- a/utils/crates-io/src/rename.rs +++ /dev/null @@ -1,73 +0,0 @@ -// This file is part of Gear. - -// Copyright (C) 2021-2023 Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Renaming handler - -use anyhow::Result; -use cargo_metadata::Package; -use cargo_toml::{Dependency, Manifest}; - -/// Rename a package -pub fn package(pkg: &Package, manifest: &mut Manifest) -> Result<()> { - // NOTE: This is a bug inside of crate cargo_toml, it should - // not append crate-type = ["rlib"] to proc-macro crates, fixing - // it by hacking it now. - if pkg.name.ends_with("-codegen") { - if let Some(product) = manifest.lib.as_mut() { - product.crate_type = vec![]; - } - } - - Ok(()) -} - -/// Rename a dependencies -pub fn deps(map: &mut Manifest, index: Vec<&String>, version: String) -> Result<()> { - for (name, dep) in map.dependencies.iter_mut() { - // No need to update dependencies for packages without - // local dependencies. - if !index.contains(&name) && !name.starts_with("sp-") { - continue; - } - - let mut detail = if let Dependency::Detailed(detail) = &dep { - detail.clone() - } else { - continue; - }; - - match name.as_ref() { - // NOTE: the required version of sp-arithmetic is 6.0.0 in - // git repo, but 7.0.0 in crates.io, so we need to fix it. - "sp-arithmetic" => { - detail.version = Some("7.0.0".into()); - detail.branch = None; - detail.git = None; - } - sp if sp.starts_with("sp-") => { - detail.branch = None; - detail.git = None; - } - _ => detail.version = Some(version.clone()), - } - - *dep = Dependency::Detailed(detail); - } - - Ok(()) -} diff --git a/utils/crates-io/src/version.rs b/utils/crates-io/src/version.rs index d22924818ef..4986ef927fc 100644 --- a/utils/crates-io/src/version.rs +++ b/utils/crates-io/src/version.rs @@ -18,8 +18,9 @@ //! Crate verifier -use anyhow::Result; +use anyhow::{anyhow, Result}; use serde::Deserialize; +use std::process::Command; #[derive(Debug, Deserialize)] struct Resp { @@ -44,3 +45,15 @@ pub fn verify(name: &str, version: &str) -> Result { Ok(resp.versions.into_iter().any(|v| v.num == version)) } + +/// Get the short hash of the current commit. +pub fn hash() -> Result { + Ok(Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .map_err(|e| anyhow!("failed to execute command git, {e}"))? + .stdout + .iter() + .filter_map(|&c| (!c.is_ascii_whitespace()).then_some(c as char)) + .collect()) +}