From 8aaedaf8544b6de21a9b239bb51ba521e74c013b Mon Sep 17 00:00:00 2001 From: clearloop <26088946+clearloop@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:42:17 +0800 Subject: [PATCH] feat(crates-io): introduce crates-io manager (#3103) --- .github/workflows/crates-io.yml | 25 ++++ Cargo.lock | 88 +++++++++++++- Cargo.toml | 14 ++- common/src/lib.rs | 3 +- gcli/Cargo.toml | 2 - gcli/src/cmd/program.rs | 1 - gcli/src/meta/executor.rs | 6 - gmeta/Cargo.toml | 2 +- gsdk/codegen/Cargo.toml | 2 +- sandbox/env/Cargo.toml | 12 +- sandbox/host/Cargo.toml | 20 ++-- utils/actor-system-error/Cargo.toml | 9 +- utils/crates-io/Cargo.toml | 16 +++ utils/crates-io/publish.rs | 171 ++++++++++++++++++++++++++++ utils/utils/Cargo.toml | 7 +- utils/wasm-builder/Cargo.toml | 2 +- 16 files changed, 339 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/crates-io.yml create mode 100644 utils/crates-io/Cargo.toml create mode 100644 utils/crates-io/publish.rs diff --git a/.github/workflows/crates-io.yml b/.github/workflows/crates-io.yml new file mode 100644 index 00000000000..f04744968fd --- /dev/null +++ b/.github/workflows/crates-io.yml @@ -0,0 +1,25 @@ +name: Crates IO + +on: + workflow_dispatch: + +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + RUST_BACKTRACE: short + TERM: xterm-256color + +jobs: + publish: + runs-on: ubuntu-latest + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + steps: + - name: "ACTIONS: Checkout" + uses: actions/checkout@v3 + + - name: "Install: Rust toolchain" + uses: dsherret/rust-toolchain-file@v1 + + - name: "Publish packages" + run: cargo run --release -p crates-io-manager diff --git a/Cargo.lock b/Cargo.lock index 8362f8093d5..06d86e1eefb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ [[package]] name = "actor-system-error" -version = "0.1.0" +version = "1.0.2" dependencies = [ "derive_more", ] @@ -1025,6 +1025,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cargo_toml" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "599aa35200ffff8f04c1925aa1acc92fa2e08874379ef42e210a80e527e60838" +dependencies = [ + "serde", + "toml 0.7.8", +] + [[package]] name = "cc" version = "1.0.83" @@ -1586,6 +1596,32 @@ dependencies = [ "wasmtime-types", ] +[[package]] +name = "crates-io" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876aa69b4afca5f2eb5e23daa3445930faf829bcb67075a20ffa884f11f8c57c" +dependencies = [ + "anyhow", + "curl", + "percent-encoding", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "crates-io-manager" +version = "1.0.2" +dependencies = [ + "anyhow", + "cargo_metadata", + "cargo_toml", + "crates-io", + "curl", + "toml 0.7.8", +] + [[package]] name = "crc" version = "3.0.1" @@ -1730,6 +1766,36 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "curl" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2 0.4.9", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.68+curl-8.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "windows-sys 0.48.0", +] + [[package]] name = "curve25519-dalek" version = "2.1.3" @@ -3818,8 +3884,6 @@ dependencies = [ "etc", "gear-core", "gear-core-errors", - "gear-core-processor", - "gear-lazy-pages-interface", "gmeta", "gsdk", "hex", @@ -4322,7 +4386,7 @@ dependencies = [ [[package]] name = "gear-sandbox-env" -version = "0.1.0" +version = "1.0.2" dependencies = [ "parity-scale-codec", "sp-core", @@ -4332,7 +4396,7 @@ dependencies = [ [[package]] name = "gear-sandbox-host" -version = "0.1.0" +version = "1.0.2" dependencies = [ "environmental", "gear-sandbox-env", @@ -4427,7 +4491,7 @@ dependencies = [ [[package]] name = "gear-utils" -version = "0.1.0" +version = "1.0.2" dependencies = [ "env_logger", "gear-core", @@ -7244,6 +7308,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index b1410017c88..7ef743886fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ members = [ "gclient", "gcore", "gmeta", + "gmeta/codegen", "gsdk", "gsdk/codegen", "gsdk/api-gen", @@ -98,6 +99,8 @@ base64 = "0.21.5" byteorder = { version = "1.5.0", default-features = false } blake2-rfc = { version = "0.2.18", default-features = false } bs58 = { version = "0.5.0", default-features = false } +# TODO: upgrade this package ( issue #2694 ) +cargo_metadata = "=0.15.3" clap = { version = "4.4.8" } codec = { package = "parity-scale-codec", version = "3.6.4", default-features = false } color-eyre = "0.6.2" @@ -109,6 +112,7 @@ dlmalloc = { git = "https://github.com/gear-tech/dlmalloc-rust.git" } dyn-clonable = "0.9.0" enum-iterator = "1.4.0" env_logger = "0.10" +environmental = "1.1.3" futures = { version = "0.3", default-features = false } futures-timer = "3.0.2" futures-util = "0.3.29" @@ -163,11 +167,15 @@ tokio = { version = "1.34.0" } url = "2.4.1" wat = "1.0.79" wabt = "0.10.0" +wasmer = "2.2" +wasmer-cache = "2.2.1" +wasmer-types = "2.2" wasmi = { version = "0.14.0", default-features = false } wasmparser = { package = "wasmparser-nostd", version = "0.100.1", default-features = false } which = "4.4.0" winapi = "0.3.9" paste = "1.0" +tempfile = "3.5.0" # Published deps # @@ -194,6 +202,7 @@ gstd = { path = "gstd" } gsys = { path = "gsys" } gtest = { path = "gtest" } gmeta = { path = "gmeta" } +gmeta-codegen = { path = "gmeta/codegen" } gear-authorship = { path = "node/authorship" } gear-core-backend = { path = "core-backend", default-features = false } gear-call-gen = { path = "utils/call-gen" } @@ -422,8 +431,6 @@ demo-wat = { path = "examples/wat" } cfg-if = "1.0.0" # gear-lazy-pages errno = "0.3" # gear-lazy-pages nix = "0.26.4" # gear-lazy-pages -# TODO: upgrade this package ( issue #2694 ) -cargo_metadata = "=0.15.3" # utils/wasm-builder impl-trait-for-tuples = "0.2.2" # pallets/staking-rewards indexmap = "2.1.0" # utils/weight-diff indicatif = "*" # utils/wasm-gen @@ -453,6 +460,9 @@ fail = "0.5" # gear scale-value = "^0.12" # gsdk heck = "0.4.1" # gsdk-api-gen etc = "0.1.16" # gcli +cargo_toml = "0.15.3" # crates-io +crates-io = "0.37.0" # crates-io +curl = "0.4.44" # crates-io scale-decode = "0.9.0" # gsdk directories = "5.0.1" # utils/key-finder num-traits = { version = "0.2", default-features = false } # gear-core diff --git a/common/src/lib.rs b/common/src/lib.rs index 14c182b5dad..e8fdff55497 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -66,7 +66,6 @@ use gear_core::{ }; use primitive_types::H256; use sp_arithmetic::traits::{BaseArithmetic, One, Saturating, UniqueSaturatedInto, Unsigned}; -use sp_core::crypto::UncheckedFrom; use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, prelude::*, @@ -108,7 +107,7 @@ impl Origin for sp_runtime::AccountId32 { } fn from_origin(v: H256) -> Self { - sp_runtime::AccountId32::unchecked_from(v) + Self::new(v.0) } } diff --git a/gcli/Cargo.toml b/gcli/Cargo.toml index 46d21f695c0..a562d8cefff 100644 --- a/gcli/Cargo.toml +++ b/gcli/Cargo.toml @@ -40,8 +40,6 @@ clap = { workspace = true, features = ["derive"] } thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } whoami.workspace = true -core-processor = { workspace = true, features = [ "std" ] } -gear-lazy-pages-interface = { workspace = true, features = [ "std" ] } reqwest = { workspace = true, default-features = false, features = [ "json", "rustls-tls" ] } etc.workspace = true sp-io = { workspace = true, features = [ "std" ] } diff --git a/gcli/src/cmd/program.rs b/gcli/src/cmd/program.rs index b3ccfdd9b0e..80db325eb2b 100644 --- a/gcli/src/cmd/program.rs +++ b/gcli/src/cmd/program.rs @@ -105,7 +105,6 @@ impl Program { Ok(()) } - /// Display meta. fn meta(path: &PathBuf, name: &Option) -> Result<()> { let ext = path .extension() diff --git a/gcli/src/meta/executor.rs b/gcli/src/meta/executor.rs index be76fd537c4..42381ae2328 100644 --- a/gcli/src/meta/executor.rs +++ b/gcli/src/meta/executor.rs @@ -23,8 +23,6 @@ use anyhow::{anyhow, Result}; use wasmi::{AsContextMut, Engine, Extern, Linker, Memory, MemoryType, Module, Store}; -const PAGE_STORAGE_PREFIX: [u8; 32] = *b"gcligcligcligcligcligcligcligcli"; - /// HostState for the WASM executor #[derive(Default)] pub struct HostState { @@ -39,10 +37,6 @@ pub fn call_metadata(wasm: &[u8]) -> Result> { /// Executes the WASM code. fn execute(wasm: &[u8], method: &str) -> Result> { - assert!(gear_lazy_pages_interface::try_to_enable_lazy_pages( - PAGE_STORAGE_PREFIX - )); - let engine = Engine::default(); let module = Module::new(&engine, wasm).unwrap(); diff --git a/gmeta/Cargo.toml b/gmeta/Cargo.toml index 7f79855f0cf..a2f56a365ab 100644 --- a/gmeta/Cargo.toml +++ b/gmeta/Cargo.toml @@ -14,7 +14,7 @@ repository.workspace = true scale-info.workspace = true blake2-rfc.workspace = true hex = { workspace = true, features = ["alloc"] } -gmeta-codegen = { path = "codegen", optional = true } +gmeta-codegen = { workspace = true, optional = true } derive_more.workspace = true [dev-dependencies] diff --git a/gsdk/codegen/Cargo.toml b/gsdk/codegen/Cargo.toml index 5d6fda229d1..ae54b23b204 100644 --- a/gsdk/codegen/Cargo.toml +++ b/gsdk/codegen/Cargo.toml @@ -14,6 +14,6 @@ repository.workspace = true proc-macro = true [dependencies] -syn = { workspace = true, features = ["full"] } +syn = { workspace = true, features = ["default", "full"] } quote.workspace = true proc-macro2.workspace = true diff --git a/sandbox/env/Cargo.toml b/sandbox/env/Cargo.toml index 844eaa9bc90..9c4a8891467 100644 --- a/sandbox/env/Cargo.toml +++ b/sandbox/env/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "gear-sandbox-env" -version = "0.1.0" -authors = ["Gear Technologies"] -edition = "2021" -license = "GPL-3.0" -homepage = "https://gear-tech.io" -repository = "https://github.com/gear-tech/gear" description = "This crate provides means to instantiate and execute wasm modules." +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +version.workspace = true [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/sandbox/host/Cargo.toml b/sandbox/host/Cargo.toml index 984f6140cf6..d2cdb363668 100644 --- a/sandbox/host/Cargo.toml +++ b/sandbox/host/Cargo.toml @@ -1,20 +1,20 @@ [package] name = "gear-sandbox-host" -version = "0.1.0" -authors = ["Gear Technologies"] -edition = "2021" -license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://gear-tech.io" -repository = "https://github.com/gear-tech/gear" description = "A set of common definitions that are needed for defining execution engines." readme = "README.md" +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +version.workspace = true [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { workspace = true, features = ["std"] } -environmental = "1.1.3" +environmental.workspace = true thiserror.workspace = true log = { workspace = true, features = ["std"] } sandbox-wasmer.workspace = true @@ -23,9 +23,9 @@ wasmi = { git = "https://github.com/gear-tech/wasmi", branch = "v0.13.2-sign-ext sp-allocator = { workspace = true, features = ["std"] } sp-wasm-interface = { workspace = true, features = ["std"] } gear-sandbox-env = { workspace = true, features = ["std"] } -wasmer-cache = { version = "2.2.1", optional = true } -once_cell = "1.17.1" -tempfile = "3.5.0" +wasmer-cache = { workspace = true, optional = true } +once_cell.workspace = true +tempfile.workspace = true [features] default = ["wasmer-cache"] diff --git a/utils/actor-system-error/Cargo.toml b/utils/actor-system-error/Cargo.toml index 503a0e8bbb2..9e90a629252 100644 --- a/utils/actor-system-error/Cargo.toml +++ b/utils/actor-system-error/Cargo.toml @@ -1,7 +1,12 @@ [package] name = "actor-system-error" -version = "0.1.0" -edition = "2021" +description = "Helper crate for implementation of errors on gear backend" +edition.workspace = true +version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true [dependencies] derive_more.workspace = true diff --git a/utils/crates-io/Cargo.toml b/utils/crates-io/Cargo.toml new file mode 100644 index 00000000000..834d1e80a86 --- /dev/null +++ b/utils/crates-io/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "crates-io-manager" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "publish" +path = "publish.rs" + +[dependencies] +anyhow.workspace = true +cargo_metadata.workspace = true +cargo_toml.workspace = true +crates-io.workspace = true +curl.workspace = true +toml.workspace = true diff --git a/utils/crates-io/publish.rs b/utils/crates-io/publish.rs new file mode 100644 index 00000000000..f8e121abd08 --- /dev/null +++ b/utils/crates-io/publish.rs @@ -0,0 +1,171 @@ +//! mini-program for publishing packages to crates.io. +use anyhow::Result; +use cargo_metadata::MetadataCommand; +use cargo_toml::{Dependency, Manifest, Value}; +use crates_io::Registry; +use curl::easy::Easy; +use std::{ + collections::{BTreeMap, HashMap}, + env, fs, + path::PathBuf, + process::{Command, ExitStatus}, + thread, + time::Duration, +}; + +/// Packages need to be published. +const PACKAGES: [&str; 14] = [ + // Packages without local dependencies. + "actor-system-error", + "gear-common-codegen", + "gear-core-errors", + "gear-wasm-instrument", + "gmeta-codegen", + "gsdk-codegen", + "gsys", + // The packages below have local dependencies, + // and should be published in order. + "gmeta", + "gear-core", + "gear-utils", + "gear-common", + "gsdk", + "gcli", + "gclient", +]; + +/// Packages need to be patched in dependencies. +const PATCHED_PACKAGES: [&str; 1] = ["sp-arithmetic"]; + +struct CratesIo { + registry: Registry, +} + +impl CratesIo { + /// Create a new instance of `CratesIo`. + pub fn new() -> Result { + let mut handle = Easy::new(); + handle.useragent("gear-crates-io-manager")?; + + Ok(Self { + registry: Registry::new_handle("https://crates.io".into(), None, handle, false), + }) + } + + /// Verify if the package is published to crates.io. + pub fn verify(&mut self, package: &str, version: &str) -> Result { + // Here using limit = 1 since we are searching explicit + // packages here. + let (crates, _total) = self.registry.search(package, 1)?; + if crates.len() != 1 { + return Ok(false); + } + + Ok(crates[0].max_version == version) + } +} + +fn main() -> Result<()> { + let mut validator = CratesIo::new()?; + let metadata = MetadataCommand::new().no_deps().exec()?; + let mut graph = BTreeMap::new(); + let index = HashMap::::from_iter( + PACKAGES.into_iter().enumerate().map(|(i, p)| (p.into(), i)), + ); + + let workspace_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../Cargo.toml") + .canonicalize()?; + let workspace = Manifest::from_path(&workspace_path)?; + + for p in metadata.packages.into_iter() { + if !index.contains_key(&p.name) { + continue; + } + + let version = p.version.to_string(); + if validator.verify(&p.name, &version)? { + println!("Package {}@{} already published.", &p.name, &version); + continue; + } + + let path = p.manifest_path.into_std_path_buf(); + let mut manifest = Manifest::::from_slice_with_metadata(&fs::read(&path)?)?; + manifest.complete_from_path_and_workspace(&path, Some((&workspace, &workspace_path)))?; + + // NOTE: This is a bug inside of crate cargo_toml, it should + // not append crate-type = ["rlib"] to proc-macro crates, fixing + // it by hacking it now. + if p.name.ends_with("-codegen") { + if let Some(mut product) = manifest.lib { + product.crate_type = vec![]; + manifest.lib = Some(product); + } + } + + for (name, dep) in manifest.dependencies.iter_mut() { + // No need to update dependencies for packages without + // local dependencies. + if !index.contains_key(name) && !PATCHED_PACKAGES.contains(&name.as_str()) { + continue; + } + + let mut detail = if let Dependency::Detailed(detail) = &dep { + detail.clone() + } else { + continue; + }; + + match name.as_ref() { + // NOTE: the required version of sp-arithmetic is 6.0.0 in + // git repo, but 7.0.0 in crates.io, so we need to fix it. + "sp-arithmetic" => { + detail.version = Some("7.0.0".into()); + detail.branch = None; + detail.git = None; + } + _ => detail.version = Some(version.to_string()), + } + + *dep = Dependency::Detailed(detail); + } + + graph.insert(index.get(&p.name), (path, manifest)); + } + + for (path, manifest) in graph.values() { + println!("Publishing {:?}", path); + fs::write(path, toml::to_string_pretty(manifest)?)?; + + let path = path.to_string_lossy(); + let status = publish(&path)?; + if !status.success() { + println!( + "Failed to publish package {}...\nRetry after 11 mins...", + &path + ); + // The most likely reason for failure is that + // we have reached the rate limit of crates.io. + // + // Need to wait for 10 mins and try again. here + // we use 11 mins to be safe. + // + // Only retry for once, if it still fails, we + // will just give up. + thread::sleep(Duration::from_secs(660)); + publish(&path)?; + } + } + + Ok(()) +} + +fn publish(manifest: &str) -> Result { + Command::new("cargo") + .arg("publish") + .arg("--manifest-path") + .arg(manifest) + .arg("--allow-dirty") + .status() + .map_err(Into::into) +} diff --git a/utils/utils/Cargo.toml b/utils/utils/Cargo.toml index 779b44942ad..f91cf7d64d6 100644 --- a/utils/utils/Cargo.toml +++ b/utils/utils/Cargo.toml @@ -1,7 +1,12 @@ [package] name = "gear-utils" -version = "0.1.0" +description = "Utils of gear network" +version.workspace = true +authors.workspace = true edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true [dependencies] nonempty.workspace = true diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index c876488ce9e..79bad58298c 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -30,7 +30,7 @@ regex.workspace = true [dev-dependencies] wabt.workspace = true -wasmi = {workspace = true, features = ["std"]} +wasmi = { workspace = true, features = ["std"] } [features] metawasm = ["gmeta/codegen"]