From 57f35c91ca4808c1c996f7b50d3eedcd20e534a2 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Thu, 8 Aug 2024 20:24:18 +0800 Subject: [PATCH 01/16] Initial draft to implement downloading rust toolchain during sdist Initial draft to implement downloading rust toolchain during sdist --- src/build_context.rs | 31 ++++++++++++++++++++++++++++++- src/source_distribution.rs | 36 ++++++++++++++++++++++++++++++++++++ src/upload.rs | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/build_context.rs b/src/build_context.rs index 2e498537d..a6e337688 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -8,7 +8,7 @@ use crate::module_writer::{ }; use crate::project_layout::ProjectLayout; use crate::python_interpreter::InterpreterKind; -use crate::source_distribution::source_distribution; +use crate::source_distribution::{download_and_execute_rustup, source_distribution}; use crate::target::{Arch, Os}; use crate::{ compile, pyproject_toml::Format, BuildArtifact, Metadata23, ModuleWriter, PyProjectToml, @@ -273,6 +273,22 @@ impl BuildContext { let sdist_path = source_distribution(self, pyproject, self.excludes(Format::Sdist)?) .context("Failed to build source distribution")?; + if self.require_rust_toolchain() { + let mut target_path = sdist_path.clone().to_path_buf(); + target_path.pop(); + target_path.pop(); + target_path.push("bin"); + + fs::create_dir_all(&target_path) + .context("Fail to create directory for installing rust toolchain")?; + + let target_path_str = target_path + .to_str() + .context("Fail to construct target path for installing rust toolchain")?; + + download_and_execute_rustup(target_path_str, target_path_str) + .context("Failed to download rust toolchain")?; + } Ok(Some((sdist_path, "source".to_string()))) } None => Ok(None), @@ -1132,6 +1148,19 @@ impl BuildContext { } Ok(wheels) } + /// Check if user requires to install rust toolchain + /// + /// Loop over `build-system.requires` defined in pyproject.toml and see if `rust-toolchain` is provided + pub fn require_rust_toolchain(&self) -> bool { + match &self.pyproject_toml { + Some(pyproject_toml) => pyproject_toml + .build_system + .requires + .iter() + .any(|req| req.name.as_ref() == "rust-toolchain"), + None => false, + } + } } /// Calculate the sha256 of a file diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 6f8e8ded0..df1afa383 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,5 +1,6 @@ use crate::module_writer::{add_data, ModuleWriter}; use crate::pyproject_toml::SdistGenerator; +use crate::upload::http_agent; use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::{Metadata, MetadataCommand, PackageId}; @@ -8,9 +9,11 @@ use ignore::overrides::Override; use normpath::PathExt as _; use path_slash::PathExt as _; use std::collections::HashMap; +use std::io::copy; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use tempfile::NamedTempFile; use tracing::debug; /// Path dependency information. @@ -735,3 +738,36 @@ where None } } + +/// Downloads the rustup installer script and executes it to install rustup +/// +/// Inspired by https://github.com/chriskuehl/rustenv +pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { + let mut tf = NamedTempFile::new()?; + let agent = http_agent()?; + let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; + + copy(&mut response.as_bytes(), &mut tf)?; + + #[cfg(unix)] + { + Command::new("sh") + .arg(tf.path()) + .arg("-y") + .arg("--no-modify-path") + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + #[cfg(windows)] + { + Command::new("cmd") + .args(&["/C", tf.path(), "-y", "--no-modify-path"]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + Ok(()) +} diff --git a/src/upload.rs b/src/upload.rs index 5560a62e1..9971833a9 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -382,7 +382,7 @@ fn http_agent() -> Result { #[cfg(feature = "rustls")] #[allow(clippy::result_large_err)] -fn http_agent() -> Result { +pub fn http_agent() -> Result { use std::sync::Arc; let builder = ureq::builder().try_proxy_from_env(true); From 1ec7982c6188327b52449555c05ae43f505586cf Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 09:08:55 +0800 Subject: [PATCH 02/16] Enhance pep517 write-dist-info command to check for rust toolchain --- maturin/__init__.py | 17 ------- src/build_context.rs | 99 +++++++++++++++++++++++++++----------- src/lib.rs | 4 +- src/main.rs | 16 +++++- src/source_distribution.rs | 36 -------------- 5 files changed, 88 insertions(+), 84 deletions(-) diff --git a/maturin/__init__.py b/maturin/__init__.py index 3b69ba647..495d50e02 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -165,23 +165,6 @@ def get_requires_for_build_sdist(config_settings: Optional[Mapping[str, Any]] = def prepare_metadata_for_build_wheel( metadata_directory: str, config_settings: Optional[Mapping[str, Any]] = None ) -> str: - print("Checking for Rust toolchain....") - is_cargo_installed = False - try: - output = subprocess.check_output(["cargo", "--version"]).decode("utf-8", "ignore") - if "cargo" in output: - is_cargo_installed = True - except (FileNotFoundError, SubprocessError): - pass - - if not is_cargo_installed: - sys.stderr.write( - "\nCargo, the Rust package manager, is not installed or is not on PATH.\n" - "This package requires Rust and Cargo to compile extensions. Install it through\n" - "the system's package manager or via https://rustup.rs/\n\n" - ) - sys.exit(1) - command = [ "maturin", "pep517", diff --git a/src/build_context.rs b/src/build_context.rs index a6e337688..e8cf1f66c 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -8,8 +8,10 @@ use crate::module_writer::{ }; use crate::project_layout::ProjectLayout; use crate::python_interpreter::InterpreterKind; -use crate::source_distribution::{download_and_execute_rustup, source_distribution}; +use crate::source_distribution::source_distribution; use crate::target::{Arch, Os}; +#[cfg(feature = "upload")] +use crate::upload::http_agent; use crate::{ compile, pyproject_toml::Format, BuildArtifact, Metadata23, ModuleWriter, PyProjectToml, PythonInterpreter, Target, @@ -28,8 +30,13 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::fmt::{Display, Formatter}; use std::io; +#[cfg(feature = "rustls")] +use std::io::copy; use std::path::{Path, PathBuf}; +use std::process::Command; use std::str::FromStr; +#[cfg(feature = "rustls")] +use tempfile::NamedTempFile; use tracing::instrument; /// The way the rust code is used in the wheel @@ -273,22 +280,6 @@ impl BuildContext { let sdist_path = source_distribution(self, pyproject, self.excludes(Format::Sdist)?) .context("Failed to build source distribution")?; - if self.require_rust_toolchain() { - let mut target_path = sdist_path.clone().to_path_buf(); - target_path.pop(); - target_path.pop(); - target_path.push("bin"); - - fs::create_dir_all(&target_path) - .context("Fail to create directory for installing rust toolchain")?; - - let target_path_str = target_path - .to_str() - .context("Fail to construct target path for installing rust toolchain")?; - - download_and_execute_rustup(target_path_str, target_path_str) - .context("Failed to download rust toolchain")?; - } Ok(Some((sdist_path, "source".to_string()))) } None => Ok(None), @@ -1148,18 +1139,72 @@ impl BuildContext { } Ok(wheels) } - /// Check if user requires to install rust toolchain + + /// Check if Rust toolchain is installed + pub fn is_toolchain_installed() -> bool { + return Command::new("cargo").arg("--version").output().is_ok(); + } + + /// Downloads the rustup installer script and executes it to install rustup /// - /// Loop over `build-system.requires` defined in pyproject.toml and see if `rust-toolchain` is provided - pub fn require_rust_toolchain(&self) -> bool { - match &self.pyproject_toml { - Some(pyproject_toml) => pyproject_toml - .build_system - .requires - .iter() - .any(|req| req.name.as_ref() == "rust-toolchain"), - None => false, + /// Inspired by https://github.com/chriskuehl/rustenv + #[cfg(feature = "rustls")] + pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { + let mut tf = NamedTempFile::new()?; + let agent = http_agent()?; + let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; + + copy(&mut response.as_bytes(), &mut tf)?; + + #[cfg(unix)] + { + Command::new("sh") + .arg(tf.path()) + .arg("-y") + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; } + + #[cfg(windows)] + { + let cargo_env_path = cargo_env_path.replace("/", "\\"); + + Command::new("cmd") + .args(&["/C", "CALL", &cargo_env_path]) + .status()?; + } + + Ok(()) + } + + /// Refresh the current shell to include path for rust toolchain + pub fn add_cargo_to_path(cargo_home: &str) -> Result<()> { + let cargo_bin_path = Path::new(cargo_home).join("bin"); + + #[cfg(unix)] + { + let current_path = env::var("PATH").unwrap_or_default(); + let new_path = format!("{}:{}", cargo_bin_path.display(), current_path); + unsafe {env::set_var("PATH", &new_path)}; + Command::new(cargo_bin_path.join("rustup")) + .arg("default") + .arg("stable") + .output() + .context("Failed to set default Rust toolchain using rustup")?; + } + + /// FIXME: Test the following command + #[cfg(windows)] + { + Command::new("cmd") + .args(&["/C", tf.path(), "-y", "--no-modify-path"]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 176d3006b..3acccc5e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::pyproject_toml::PyProjectToml; pub use crate::python_interpreter::PythonInterpreter; pub use crate::target::Target; #[cfg(feature = "upload")] -pub use crate::upload::{upload, upload_ui, PublishOpt, Registry, UploadError}; +pub use crate::upload::{upload_ui, PublishOpt, Registry, UploadError}; pub use auditwheel::PlatformTag; mod auditwheel; @@ -64,4 +64,4 @@ mod python_interpreter; mod source_distribution; mod target; #[cfg(feature = "upload")] -mod upload; +mod upload; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index d5d3ef978..11ba0c441 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,12 +10,16 @@ use cargo_zigbuild::Zig; #[cfg(feature = "cli-completion")] use clap::CommandFactory; use clap::{Parser, Subcommand}; +#[cfg(feature = "rustls")] +use dirs::home_dir; #[cfg(feature = "scaffolding")] use maturin::{ci::GenerateCI, init_project, new_project, GenerateProjectOptions}; use maturin::{ - develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, DevelopOptions, PathWriter, - PlatformTag, PythonInterpreter, Target, + develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, + DevelopOptions, PathWriter, PlatformTag, PythonInterpreter, Target, }; +#[cfg(feature = "rustls")] +use maturin::BuildContext; #[cfg(feature = "schemars")] use maturin::{generate_json_schema, GenerateJsonSchemaOptions}; #[cfg(feature = "upload")] @@ -273,6 +277,14 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { strip, } => { assert_eq!(build_options.interpreter.len(), 1); + #[cfg(feature = "rustls")] + if !BuildContext::is_toolchain_installed() { + let home_dir = home_dir().context("Unabel to get user home directory")?; + let home_dir_str = home_dir.to_str().context("Unable to convert home directory string")?; + BuildContext::download_and_execute_rustup(home_dir_str, home_dir_str) + .context("Unable to install & execute rustup")?; + BuildContext::add_cargo_to_path(home_dir_str).context("Unable to add cargo path")?; + } let context = build_options.into_build_context(true, strip, false)?; // Since afaik all other PEP 517 backends also return linux tagged wheels, we do so too diff --git a/src/source_distribution.rs b/src/source_distribution.rs index df1afa383..6f8e8ded0 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,6 +1,5 @@ use crate::module_writer::{add_data, ModuleWriter}; use crate::pyproject_toml::SdistGenerator; -use crate::upload::http_agent; use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::{Metadata, MetadataCommand, PackageId}; @@ -9,11 +8,9 @@ use ignore::overrides::Override; use normpath::PathExt as _; use path_slash::PathExt as _; use std::collections::HashMap; -use std::io::copy; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; -use tempfile::NamedTempFile; use tracing::debug; /// Path dependency information. @@ -738,36 +735,3 @@ where None } } - -/// Downloads the rustup installer script and executes it to install rustup -/// -/// Inspired by https://github.com/chriskuehl/rustenv -pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { - let mut tf = NamedTempFile::new()?; - let agent = http_agent()?; - let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; - - copy(&mut response.as_bytes(), &mut tf)?; - - #[cfg(unix)] - { - Command::new("sh") - .arg(tf.path()) - .arg("-y") - .arg("--no-modify-path") - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; - } - - #[cfg(windows)] - { - Command::new("cmd") - .args(&["/C", tf.path(), "-y", "--no-modify-path"]) - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; - } - - Ok(()) -} From 6db3fdfc49c2085774bb1b9342064eaa9f538efa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 01:09:09 +0000 Subject: [PATCH 03/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 3acccc5e7..0ca61dbd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -64,4 +64,4 @@ mod python_interpreter; mod source_distribution; mod target; #[cfg(feature = "upload")] -mod upload; \ No newline at end of file +mod upload; From 2a721a783090d4852755c65422859b807227151d Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 16:32:49 +0800 Subject: [PATCH 04/16] Fix window command --- src/build_context.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/build_context.rs b/src/build_context.rs index e8cf1f66c..d959d0a28 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1168,11 +1168,12 @@ impl BuildContext { #[cfg(windows)] { - let cargo_env_path = cargo_env_path.replace("/", "\\"); - Command::new("cmd") - .args(&["/C", "CALL", &cargo_env_path]) - .status()?; + .args(&["/C", tf.path().to_str().unwrap()]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status() + .context("Failed to execute rustup script on Windows")?; } Ok(()) @@ -1194,14 +1195,16 @@ impl BuildContext { .context("Failed to set default Rust toolchain using rustup")?; } - /// FIXME: Test the following command #[cfg(windows)] { - Command::new("cmd") - .args(&["/C", tf.path(), "-y", "--no-modify-path"]) - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; + let current_path = env::var("PATH").unwrap_or_default(); + let new_path = format!("{};{}", cargo_bin_path.display(), current_path); + env::set_var("PATH", &new_path); + Command::new(cargo_bin_path.join("rustup.exe")) + .arg("default") + .arg("stable") + .output() + .context("Failed to set default Rust toolchain using rustup")?; } Ok(()) From 136fee269fa5779279128d89175ff3ed12e57f4c Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 16:45:42 +0800 Subject: [PATCH 05/16] Fix failing cargo fmt and ruff CI jobs --- maturin/__init__.py | 1 - src/build_context.rs | 2 +- src/main.rs | 15 +++++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/maturin/__init__.py b/maturin/__init__.py index 495d50e02..c17ff1272 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -18,7 +18,6 @@ import struct import subprocess import sys -from subprocess import SubprocessError from typing import Any, Dict, Mapping, List, Optional try: diff --git a/src/build_context.rs b/src/build_context.rs index d959d0a28..e5274e544 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1187,7 +1187,7 @@ impl BuildContext { { let current_path = env::var("PATH").unwrap_or_default(); let new_path = format!("{}:{}", cargo_bin_path.display(), current_path); - unsafe {env::set_var("PATH", &new_path)}; + unsafe { env::set_var("PATH", &new_path) }; Command::new(cargo_bin_path.join("rustup")) .arg("default") .arg("stable") diff --git a/src/main.rs b/src/main.rs index 11ba0c441..ac276e167 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,14 +12,14 @@ use clap::CommandFactory; use clap::{Parser, Subcommand}; #[cfg(feature = "rustls")] use dirs::home_dir; +#[cfg(feature = "rustls")] +use maturin::BuildContext; #[cfg(feature = "scaffolding")] use maturin::{ci::GenerateCI, init_project, new_project, GenerateProjectOptions}; use maturin::{ - develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, - DevelopOptions, PathWriter, PlatformTag, PythonInterpreter, Target, + develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, DevelopOptions, PathWriter, + PlatformTag, PythonInterpreter, Target, }; -#[cfg(feature = "rustls")] -use maturin::BuildContext; #[cfg(feature = "schemars")] use maturin::{generate_json_schema, GenerateJsonSchemaOptions}; #[cfg(feature = "upload")] @@ -280,10 +280,13 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { #[cfg(feature = "rustls")] if !BuildContext::is_toolchain_installed() { let home_dir = home_dir().context("Unabel to get user home directory")?; - let home_dir_str = home_dir.to_str().context("Unable to convert home directory string")?; + let home_dir_str = home_dir + .to_str() + .context("Unable to convert home directory string")?; BuildContext::download_and_execute_rustup(home_dir_str, home_dir_str) .context("Unable to install & execute rustup")?; - BuildContext::add_cargo_to_path(home_dir_str).context("Unable to add cargo path")?; + BuildContext::add_cargo_to_path(home_dir_str) + .context("Unable to add cargo path")?; } let context = build_options.into_build_context(true, strip, false)?; From 908bec04670b6de410ead55d18b74280a2f4630c Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 17:11:58 +0800 Subject: [PATCH 06/16] Fix failing codespell CI jobs --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ac276e167..f66df2ec8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -279,7 +279,7 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { assert_eq!(build_options.interpreter.len(), 1); #[cfg(feature = "rustls")] if !BuildContext::is_toolchain_installed() { - let home_dir = home_dir().context("Unabel to get user home directory")?; + let home_dir = home_dir().context("Unable to get user home directory")?; let home_dir_str = home_dir .to_str() .context("Unable to convert home directory string")?; From c6c7d72a3100cf433f9b5ba2e340ac0693dcd7ed Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Thu, 8 Aug 2024 20:24:18 +0800 Subject: [PATCH 07/16] Initial draft to implement downloading rust toolchain during sdist Initial draft to implement downloading rust toolchain during sdist --- src/build_context.rs | 31 ++++++++++++++++++++++++++++++- src/source_distribution.rs | 36 ++++++++++++++++++++++++++++++++++++ src/upload.rs | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/build_context.rs b/src/build_context.rs index a98d23360..169482d3a 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -8,7 +8,7 @@ use crate::module_writer::{ }; use crate::project_layout::ProjectLayout; use crate::python_interpreter::InterpreterKind; -use crate::source_distribution::source_distribution; +use crate::source_distribution::{download_and_execute_rustup, source_distribution}; use crate::target::{Arch, Os}; use crate::{ compile, pyproject_toml::Format, BuildArtifact, Metadata23, ModuleWriter, PyProjectToml, @@ -273,6 +273,22 @@ impl BuildContext { let sdist_path = source_distribution(self, pyproject, self.excludes(Format::Sdist)?) .context("Failed to build source distribution")?; + if self.require_rust_toolchain() { + let mut target_path = sdist_path.clone().to_path_buf(); + target_path.pop(); + target_path.pop(); + target_path.push("bin"); + + fs::create_dir_all(&target_path) + .context("Fail to create directory for installing rust toolchain")?; + + let target_path_str = target_path + .to_str() + .context("Fail to construct target path for installing rust toolchain")?; + + download_and_execute_rustup(target_path_str, target_path_str) + .context("Failed to download rust toolchain")?; + } Ok(Some((sdist_path, "source".to_string()))) } None => Ok(None), @@ -1136,6 +1152,19 @@ impl BuildContext { } Ok(wheels) } + /// Check if user requires to install rust toolchain + /// + /// Loop over `build-system.requires` defined in pyproject.toml and see if `rust-toolchain` is provided + pub fn require_rust_toolchain(&self) -> bool { + match &self.pyproject_toml { + Some(pyproject_toml) => pyproject_toml + .build_system + .requires + .iter() + .any(|req| req.name.as_ref() == "rust-toolchain"), + None => false, + } + } } /// Calculate the sha256 of a file diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 2b255ea13..c1ebb3644 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,5 +1,6 @@ use crate::module_writer::{add_data, ModuleWriter}; use crate::pyproject_toml::SdistGenerator; +use crate::upload::http_agent; use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::camino::Utf8Path; @@ -11,9 +12,11 @@ use path_slash::PathExt as _; use std::collections::HashMap; use std::ffi::OsStr; use std::io::Write; +use std::io::copy; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use tempfile::NamedTempFile; use toml_edit::DocumentMut; use tracing::debug; @@ -830,3 +833,36 @@ where None } } + +/// Downloads the rustup installer script and executes it to install rustup +/// +/// Inspired by https://github.com/chriskuehl/rustenv +pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { + let mut tf = NamedTempFile::new()?; + let agent = http_agent()?; + let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; + + copy(&mut response.as_bytes(), &mut tf)?; + + #[cfg(unix)] + { + Command::new("sh") + .arg(tf.path()) + .arg("-y") + .arg("--no-modify-path") + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + #[cfg(windows)] + { + Command::new("cmd") + .args(&["/C", tf.path(), "-y", "--no-modify-path"]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + Ok(()) +} diff --git a/src/upload.rs b/src/upload.rs index 5b9abd82f..c9ae7abc5 100644 --- a/src/upload.rs +++ b/src/upload.rs @@ -396,7 +396,7 @@ fn http_agent() -> Result { #[cfg(feature = "rustls")] #[allow(clippy::result_large_err)] -fn http_agent() -> Result { +pub fn http_agent() -> Result { use std::sync::Arc; let builder = ureq::builder().try_proxy_from_env(true); From d45b609e1744527802d4ac1e428bf19548d92f22 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 09:08:55 +0800 Subject: [PATCH 08/16] Enhance pep517 write-dist-info command to check for rust toolchain --- maturin/__init__.py | 17 ------- src/build_context.rs | 99 +++++++++++++++++++++++++++----------- src/lib.rs | 4 +- src/main.rs | 16 +++++- src/source_distribution.rs | 34 ------------- 5 files changed, 88 insertions(+), 82 deletions(-) diff --git a/maturin/__init__.py b/maturin/__init__.py index 3b69ba647..495d50e02 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -165,23 +165,6 @@ def get_requires_for_build_sdist(config_settings: Optional[Mapping[str, Any]] = def prepare_metadata_for_build_wheel( metadata_directory: str, config_settings: Optional[Mapping[str, Any]] = None ) -> str: - print("Checking for Rust toolchain....") - is_cargo_installed = False - try: - output = subprocess.check_output(["cargo", "--version"]).decode("utf-8", "ignore") - if "cargo" in output: - is_cargo_installed = True - except (FileNotFoundError, SubprocessError): - pass - - if not is_cargo_installed: - sys.stderr.write( - "\nCargo, the Rust package manager, is not installed or is not on PATH.\n" - "This package requires Rust and Cargo to compile extensions. Install it through\n" - "the system's package manager or via https://rustup.rs/\n\n" - ) - sys.exit(1) - command = [ "maturin", "pep517", diff --git a/src/build_context.rs b/src/build_context.rs index 169482d3a..55bc2cb62 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -8,8 +8,10 @@ use crate::module_writer::{ }; use crate::project_layout::ProjectLayout; use crate::python_interpreter::InterpreterKind; -use crate::source_distribution::{download_and_execute_rustup, source_distribution}; +use crate::source_distribution::source_distribution; use crate::target::{Arch, Os}; +#[cfg(feature = "upload")] +use crate::upload::http_agent; use crate::{ compile, pyproject_toml::Format, BuildArtifact, Metadata23, ModuleWriter, PyProjectToml, PythonInterpreter, Target, @@ -28,8 +30,13 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::fmt::{Display, Formatter}; use std::io; +#[cfg(feature = "rustls")] +use std::io::copy; use std::path::{Path, PathBuf}; +use std::process::Command; use std::str::FromStr; +#[cfg(feature = "rustls")] +use tempfile::NamedTempFile; use tracing::instrument; /// The way the rust code is used in the wheel @@ -273,22 +280,6 @@ impl BuildContext { let sdist_path = source_distribution(self, pyproject, self.excludes(Format::Sdist)?) .context("Failed to build source distribution")?; - if self.require_rust_toolchain() { - let mut target_path = sdist_path.clone().to_path_buf(); - target_path.pop(); - target_path.pop(); - target_path.push("bin"); - - fs::create_dir_all(&target_path) - .context("Fail to create directory for installing rust toolchain")?; - - let target_path_str = target_path - .to_str() - .context("Fail to construct target path for installing rust toolchain")?; - - download_and_execute_rustup(target_path_str, target_path_str) - .context("Failed to download rust toolchain")?; - } Ok(Some((sdist_path, "source".to_string()))) } None => Ok(None), @@ -1152,18 +1143,72 @@ impl BuildContext { } Ok(wheels) } - /// Check if user requires to install rust toolchain + + /// Check if Rust toolchain is installed + pub fn is_toolchain_installed() -> bool { + return Command::new("cargo").arg("--version").output().is_ok(); + } + + /// Downloads the rustup installer script and executes it to install rustup /// - /// Loop over `build-system.requires` defined in pyproject.toml and see if `rust-toolchain` is provided - pub fn require_rust_toolchain(&self) -> bool { - match &self.pyproject_toml { - Some(pyproject_toml) => pyproject_toml - .build_system - .requires - .iter() - .any(|req| req.name.as_ref() == "rust-toolchain"), - None => false, + /// Inspired by https://github.com/chriskuehl/rustenv + #[cfg(feature = "rustls")] + pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { + let mut tf = NamedTempFile::new()?; + let agent = http_agent()?; + let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; + + copy(&mut response.as_bytes(), &mut tf)?; + + #[cfg(unix)] + { + Command::new("sh") + .arg(tf.path()) + .arg("-y") + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; } + + #[cfg(windows)] + { + let cargo_env_path = cargo_env_path.replace("/", "\\"); + + Command::new("cmd") + .args(&["/C", "CALL", &cargo_env_path]) + .status()?; + } + + Ok(()) + } + + /// Refresh the current shell to include path for rust toolchain + pub fn add_cargo_to_path(cargo_home: &str) -> Result<()> { + let cargo_bin_path = Path::new(cargo_home).join("bin"); + + #[cfg(unix)] + { + let current_path = env::var("PATH").unwrap_or_default(); + let new_path = format!("{}:{}", cargo_bin_path.display(), current_path); + unsafe {env::set_var("PATH", &new_path)}; + Command::new(cargo_bin_path.join("rustup")) + .arg("default") + .arg("stable") + .output() + .context("Failed to set default Rust toolchain using rustup")?; + } + + /// FIXME: Test the following command + #[cfg(windows)] + { + Command::new("cmd") + .args(&["/C", tf.path(), "-y", "--no-modify-path"]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status()?; + } + + Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 176d3006b..3acccc5e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::pyproject_toml::PyProjectToml; pub use crate::python_interpreter::PythonInterpreter; pub use crate::target::Target; #[cfg(feature = "upload")] -pub use crate::upload::{upload, upload_ui, PublishOpt, Registry, UploadError}; +pub use crate::upload::{upload_ui, PublishOpt, Registry, UploadError}; pub use auditwheel::PlatformTag; mod auditwheel; @@ -64,4 +64,4 @@ mod python_interpreter; mod source_distribution; mod target; #[cfg(feature = "upload")] -mod upload; +mod upload; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 8836ef554..6321a1399 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,12 +10,16 @@ use cargo_zigbuild::Zig; #[cfg(feature = "cli-completion")] use clap::CommandFactory; use clap::{Parser, Subcommand}; +#[cfg(feature = "rustls")] +use dirs::home_dir; #[cfg(feature = "scaffolding")] use maturin::{ci::GenerateCI, init_project, new_project, GenerateProjectOptions}; use maturin::{ - develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, DevelopOptions, PathWriter, - PlatformTag, PythonInterpreter, Target, + develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, + DevelopOptions, PathWriter, PlatformTag, PythonInterpreter, Target, }; +#[cfg(feature = "rustls")] +use maturin::BuildContext; #[cfg(feature = "schemars")] use maturin::{generate_json_schema, GenerateJsonSchemaOptions}; #[cfg(feature = "upload")] @@ -273,6 +277,14 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { strip, } => { assert_eq!(build_options.interpreter.len(), 1); + #[cfg(feature = "rustls")] + if !BuildContext::is_toolchain_installed() { + let home_dir = home_dir().context("Unabel to get user home directory")?; + let home_dir_str = home_dir.to_str().context("Unable to convert home directory string")?; + BuildContext::download_and_execute_rustup(home_dir_str, home_dir_str) + .context("Unable to install & execute rustup")?; + BuildContext::add_cargo_to_path(home_dir_str).context("Unable to add cargo path")?; + } let context = build_options.into_build_context(true, strip, false)?; // Since afaik all other PEP 517 backends also return linux tagged wheels, we do so too diff --git a/src/source_distribution.rs b/src/source_distribution.rs index c1ebb3644..ee8d66b06 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,6 +1,5 @@ use crate::module_writer::{add_data, ModuleWriter}; use crate::pyproject_toml::SdistGenerator; -use crate::upload::http_agent; use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::camino::Utf8Path; @@ -833,36 +832,3 @@ where None } } - -/// Downloads the rustup installer script and executes it to install rustup -/// -/// Inspired by https://github.com/chriskuehl/rustenv -pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { - let mut tf = NamedTempFile::new()?; - let agent = http_agent()?; - let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; - - copy(&mut response.as_bytes(), &mut tf)?; - - #[cfg(unix)] - { - Command::new("sh") - .arg(tf.path()) - .arg("-y") - .arg("--no-modify-path") - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; - } - - #[cfg(windows)] - { - Command::new("cmd") - .args(&["/C", tf.path(), "-y", "--no-modify-path"]) - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; - } - - Ok(()) -} From 978677cab3f071409f87d0fc1295f4a957d50a22 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 16:32:49 +0800 Subject: [PATCH 09/16] Fix window command --- src/build_context.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/build_context.rs b/src/build_context.rs index 55bc2cb62..f5ac72f01 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1172,11 +1172,12 @@ impl BuildContext { #[cfg(windows)] { - let cargo_env_path = cargo_env_path.replace("/", "\\"); - Command::new("cmd") - .args(&["/C", "CALL", &cargo_env_path]) - .status()?; + .args(&["/C", tf.path().to_str().unwrap()]) + .env("RUSTUP_HOME", rustup_home) + .env("CARGO_HOME", cargo_home) + .status() + .context("Failed to execute rustup script on Windows")?; } Ok(()) @@ -1198,14 +1199,16 @@ impl BuildContext { .context("Failed to set default Rust toolchain using rustup")?; } - /// FIXME: Test the following command #[cfg(windows)] { - Command::new("cmd") - .args(&["/C", tf.path(), "-y", "--no-modify-path"]) - .env("RUSTUP_HOME", rustup_home) - .env("CARGO_HOME", cargo_home) - .status()?; + let current_path = env::var("PATH").unwrap_or_default(); + let new_path = format!("{};{}", cargo_bin_path.display(), current_path); + env::set_var("PATH", &new_path); + Command::new(cargo_bin_path.join("rustup.exe")) + .arg("default") + .arg("stable") + .output() + .context("Failed to set default Rust toolchain using rustup")?; } Ok(()) From 620c8f324c8b415ad93d8c37e4ccb6919d183771 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 01:09:09 +0000 Subject: [PATCH 10/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 3acccc5e7..0ca61dbd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -64,4 +64,4 @@ mod python_interpreter; mod source_distribution; mod target; #[cfg(feature = "upload")] -mod upload; \ No newline at end of file +mod upload; From 22f58854e14431b7fb3a8b60108cc710bc07c579 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 16:45:42 +0800 Subject: [PATCH 11/16] Fix failing cargo fmt and ruff CI jobs --- maturin/__init__.py | 1 - src/build_context.rs | 2 +- src/main.rs | 15 +++++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/maturin/__init__.py b/maturin/__init__.py index 495d50e02..c17ff1272 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -18,7 +18,6 @@ import struct import subprocess import sys -from subprocess import SubprocessError from typing import Any, Dict, Mapping, List, Optional try: diff --git a/src/build_context.rs b/src/build_context.rs index f5ac72f01..fa6a1dcaa 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1191,7 +1191,7 @@ impl BuildContext { { let current_path = env::var("PATH").unwrap_or_default(); let new_path = format!("{}:{}", cargo_bin_path.display(), current_path); - unsafe {env::set_var("PATH", &new_path)}; + unsafe { env::set_var("PATH", &new_path) }; Command::new(cargo_bin_path.join("rustup")) .arg("default") .arg("stable") diff --git a/src/main.rs b/src/main.rs index 6321a1399..080875334 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,14 +12,14 @@ use clap::CommandFactory; use clap::{Parser, Subcommand}; #[cfg(feature = "rustls")] use dirs::home_dir; +#[cfg(feature = "rustls")] +use maturin::BuildContext; #[cfg(feature = "scaffolding")] use maturin::{ci::GenerateCI, init_project, new_project, GenerateProjectOptions}; use maturin::{ - develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, - DevelopOptions, PathWriter, PlatformTag, PythonInterpreter, Target, + develop, write_dist_info, BridgeModel, BuildOptions, CargoOptions, DevelopOptions, PathWriter, + PlatformTag, PythonInterpreter, Target, }; -#[cfg(feature = "rustls")] -use maturin::BuildContext; #[cfg(feature = "schemars")] use maturin::{generate_json_schema, GenerateJsonSchemaOptions}; #[cfg(feature = "upload")] @@ -280,10 +280,13 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { #[cfg(feature = "rustls")] if !BuildContext::is_toolchain_installed() { let home_dir = home_dir().context("Unabel to get user home directory")?; - let home_dir_str = home_dir.to_str().context("Unable to convert home directory string")?; + let home_dir_str = home_dir + .to_str() + .context("Unable to convert home directory string")?; BuildContext::download_and_execute_rustup(home_dir_str, home_dir_str) .context("Unable to install & execute rustup")?; - BuildContext::add_cargo_to_path(home_dir_str).context("Unable to add cargo path")?; + BuildContext::add_cargo_to_path(home_dir_str) + .context("Unable to add cargo path")?; } let context = build_options.into_build_context(true, strip, false)?; From e06953f787f32af29a3f0b09de09e987f90da120 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Tue, 13 Aug 2024 17:11:58 +0800 Subject: [PATCH 12/16] Fix failing codespell CI jobs --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 080875334..a76360878 100644 --- a/src/main.rs +++ b/src/main.rs @@ -279,7 +279,7 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { assert_eq!(build_options.interpreter.len(), 1); #[cfg(feature = "rustls")] if !BuildContext::is_toolchain_installed() { - let home_dir = home_dir().context("Unabel to get user home directory")?; + let home_dir = home_dir().context("Unable to get user home directory")?; let home_dir_str = home_dir .to_str() .context("Unable to convert home directory string")?; From cc62930fdc571a0b7796d8c8ae22678d70a398a2 Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Mon, 21 Oct 2024 00:39:00 +0800 Subject: [PATCH 13/16] Add CI test to test pip install without toolchain --- .github/workflows/test.yml | 28 +++++++++++++++++++++++ maturin/__init__.py | 2 +- src/build_context.rs | 46 ++++++++++++++++++++++++-------------- src/main.rs | 10 ++++----- src/source_distribution.rs | 2 -- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d93c33e3..2a31cc540 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -530,6 +530,34 @@ jobs: - repository: "oxigraph/oxigraph" manifest-dir: "python" + test-pip-install-without-toolchain: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: dtolnay/rust-toolchain@stable + id: rustup + - name: Install python build tools + run: pip install setuptools wheel>=0.36.2 setuptools-rust>=1.4.0 tomli>=1.1.0 + - name: Build maturin wheel + run: | + export MATURIN_SETUP_ARGS="--features rustls" + python setup.py bdist_wheel + - name: install maturin wheel + run: pip install --force-reinstall ./dist/*.whl + - name: test + run: maturin sdist --manifest-path ./test-crates/hello-world/Cargo.toml -o ./target/hello-world/sdist + - name: Remove toolchain + run: rustup self uninstall -y + - name: pip install + run: | + export MATURIN_PEP517_ARGS="--verbose" + export RUST_LOG="debug" + pip install --no-build-isolation --verbose ./target/hello-world/sdist/hello_world-0.1.0.tar.gz + check: name: Check ${{ matrix.platform.target }} if: github.event_name != 'pull_request' diff --git a/maturin/__init__.py b/maturin/__init__.py index c17ff1272..bac59a43c 100644 --- a/maturin/__init__.py +++ b/maturin/__init__.py @@ -182,7 +182,7 @@ def prepare_metadata_for_build_wheel( print("Running `{}`".format(" ".join(command))) try: - _output = subprocess.check_output(command) + _output = subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: sys.stderr.write(f"Error running maturin: {e}\n") sys.exit(1) diff --git a/src/build_context.rs b/src/build_context.rs index fa6a1dcaa..72d5430d0 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1146,14 +1146,18 @@ impl BuildContext { /// Check if Rust toolchain is installed pub fn is_toolchain_installed() -> bool { - return Command::new("cargo").arg("--version").output().is_ok(); + let output = Command::new("cargo").arg("--version").output(); + match output { + Ok(out) => out.status.success(), + Err(_) => false, + } } /// Downloads the rustup installer script and executes it to install rustup /// /// Inspired by https://github.com/chriskuehl/rustenv #[cfg(feature = "rustls")] - pub fn download_and_execute_rustup(rustup_home: &str, cargo_home: &str) -> Result<()> { + pub fn install_installer(rustup_home: &str, cargo_home: &str) -> Result<()> { let mut tf = NamedTempFile::new()?; let agent = http_agent()?; let response = agent.get("https://sh.rustup.rs").call()?.into_string()?; @@ -1165,6 +1169,8 @@ impl BuildContext { Command::new("sh") .arg(tf.path()) .arg("-y") + .arg("--default-toolchain") + .arg("none") .env("RUSTUP_HOME", rustup_home) .env("CARGO_HOME", cargo_home) .status()?; @@ -1184,31 +1190,37 @@ impl BuildContext { } /// Refresh the current shell to include path for rust toolchain - pub fn add_cargo_to_path(cargo_home: &str) -> Result<()> { - let cargo_bin_path = Path::new(cargo_home).join("bin"); + pub fn install_toolchain(cargo_home: &str) -> Result<()> { + let current_path = env::var("PATH").unwrap_or_else(|_| String::from("")); #[cfg(unix)] { - let current_path = env::var("PATH").unwrap_or_default(); - let new_path = format!("{}:{}", cargo_bin_path.display(), current_path); - unsafe { env::set_var("PATH", &new_path) }; - Command::new(cargo_bin_path.join("rustup")) + let cargo_bin_path = format!("{}/bin", cargo_home); + let new_path = format!("{}:{}", cargo_bin_path, current_path); + env::set_var("PATH", &new_path); + + let cargo_env_path = format!("{}/env", cargo_home); + Command::new("sh") + .arg("-c") + .arg(format!(". {}", cargo_env_path)) + .status()?; // Execute and get the status + + let rustup_command = format!("{}/bin/rustup", cargo_home); + Command::new(rustup_command) .arg("default") .arg("stable") - .output() - .context("Failed to set default Rust toolchain using rustup")?; + .status()?; } #[cfg(windows)] { - let current_path = env::var("PATH").unwrap_or_default(); - let new_path = format!("{};{}", cargo_bin_path.display(), current_path); + let cargo_bin_path = format!("{}\\bin", cargo_home); + let new_path = format!("{};{}", cargo_bin_path, current_path); env::set_var("PATH", &new_path); - Command::new(cargo_bin_path.join("rustup.exe")) - .arg("default") - .arg("stable") - .output() - .context("Failed to set default Rust toolchain using rustup")?; + Command::new("cmd") + .args(&["/C", "rustup default stable"]) + .status() + .context("Failed to set rustup default stable on Windows"); } Ok(()) diff --git a/src/main.rs b/src/main.rs index a76360878..fd71fadb5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -277,16 +277,16 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { strip, } => { assert_eq!(build_options.interpreter.len(), 1); - #[cfg(feature = "rustls")] + #[cfg(feature = "rustls")] { if !BuildContext::is_toolchain_installed() { let home_dir = home_dir().context("Unable to get user home directory")?; let home_dir_str = home_dir .to_str() .context("Unable to convert home directory string")?; - BuildContext::download_and_execute_rustup(home_dir_str, home_dir_str) - .context("Unable to install & execute rustup")?; - BuildContext::add_cargo_to_path(home_dir_str) - .context("Unable to add cargo path")?; + let _ = BuildContext::install_installer(home_dir_str, home_dir_str); + let _ = BuildContext::install_toolchain(home_dir_str) + .context("Unable to install rust toolchain")?; + } } let context = build_options.into_build_context(true, strip, false)?; diff --git a/src/source_distribution.rs b/src/source_distribution.rs index ee8d66b06..2b255ea13 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -11,11 +11,9 @@ use path_slash::PathExt as _; use std::collections::HashMap; use std::ffi::OsStr; use std::io::Write; -use std::io::copy; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; -use tempfile::NamedTempFile; use toml_edit::DocumentMut; use tracing::debug; From bbea38e2042b758c31bfa931ca8878eb08dded8e Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Mon, 21 Oct 2024 21:37:48 +0800 Subject: [PATCH 14/16] Fix fmt & clippy CI job. Use upload & download artifact github action to test pip install without toolchain --- .github/workflows/test.yml | 42 +++++++++++++++++++++++++++++++++++--- src/build_context.rs | 11 +++++----- src/main.rs | 20 ++++++++++-------- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0853e6574..dd3f261d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -531,7 +531,7 @@ jobs: - repository: "oxigraph/oxigraph" manifest-dir: "python" - test-pip-install-without-toolchain: + build-maturin-wheel: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -551,8 +551,44 @@ jobs: run: pip install --force-reinstall ./dist/*.whl - name: test run: maturin sdist --manifest-path ./test-crates/hello-world/Cargo.toml -o ./target/hello-world/sdist - - name: Remove toolchain - run: rustup self uninstall -y + - name: Upload sdist + id: upload-sdist + uses: actions/upload-artifact@v3 + env: + ACTIONS_RUNTIME_TOKEN: abc + with: + name: hello-world-sdist + path: ./target/hello-world/sdist/*.tar.gz + - name: Upload wheels + id: upload-wheel + uses: actions/upload-artifact@v3 + env: + ACTIONS_RUNTIME_TOKEN: abc + with: + name: maturin-wheel + path: ./dist/*.whl + + test-pip-install-without-toolchain: + runs-on: ubuntu-latest + needs: [build-maturin-wheel] + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Download sdist from previous job + uses: actions/download-artifact@v3 + with: + name: hello-world-sdist + path: ./target/hello-world/sdist + - name: Download wheel from previous job + uses: actions/download-artifact@v3 + with: + name: maturin-wheel + path: ./dist/*.whl + - name: Install maturin whl + run: pip install --force-reinstall ./dist/*.whl - name: pip install run: | export MATURIN_PEP517_ARGS="--verbose" diff --git a/src/build_context.rs b/src/build_context.rs index 75df7ce08..1fb102393 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1180,10 +1180,12 @@ impl BuildContext { { Command::new("cmd") .args(&["/C", tf.path().to_str().unwrap()]) + .args("-y") + .arg("--default-toolchain") + .arg("none") .env("RUSTUP_HOME", rustup_home) .env("CARGO_HOME", cargo_home) - .status() - .context("Failed to execute rustup script on Windows")?; + .status()?; } Ok(()) @@ -1203,7 +1205,7 @@ impl BuildContext { Command::new("sh") .arg("-c") .arg(format!(". {}", cargo_env_path)) - .status()?; // Execute and get the status + .status()?; let rustup_command = format!("{}/bin/rustup", cargo_home); Command::new(rustup_command) @@ -1219,8 +1221,7 @@ impl BuildContext { env::set_var("PATH", &new_path); Command::new("cmd") .args(&["/C", "rustup default stable"]) - .status() - .context("Failed to set rustup default stable on Windows"); + .status(); } Ok(()) diff --git a/src/main.rs b/src/main.rs index fd71fadb5..1a3f211ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -277,15 +277,17 @@ fn pep517(subcommand: Pep517Command) -> Result<()> { strip, } => { assert_eq!(build_options.interpreter.len(), 1); - #[cfg(feature = "rustls")] { - if !BuildContext::is_toolchain_installed() { - let home_dir = home_dir().context("Unable to get user home directory")?; - let home_dir_str = home_dir - .to_str() - .context("Unable to convert home directory string")?; - let _ = BuildContext::install_installer(home_dir_str, home_dir_str); - let _ = BuildContext::install_toolchain(home_dir_str) - .context("Unable to install rust toolchain")?; + #[cfg(feature = "rustls")] + { + if !BuildContext::is_toolchain_installed() { + let home_dir = home_dir().context("Unable to get user home directory")?; + let home_dir_str = home_dir + .to_str() + .context("Unable to convert home directory string")?; + BuildContext::install_installer(home_dir_str, home_dir_str) + .context("Unable to install installer")?; + BuildContext::install_toolchain(home_dir_str) + .context("Unable to install rust toolchain")?; } } let context = build_options.into_build_context(true, strip, false)?; From 4b67e92c73074f16a1d87b3aa9b879c928bcfa0c Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Mon, 21 Oct 2024 22:28:10 +0800 Subject: [PATCH 15/16] Fix pip install *.whl --- .github/workflows/test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dd3f261d6..7359ad8e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -554,16 +554,12 @@ jobs: - name: Upload sdist id: upload-sdist uses: actions/upload-artifact@v3 - env: - ACTIONS_RUNTIME_TOKEN: abc with: name: hello-world-sdist path: ./target/hello-world/sdist/*.tar.gz - name: Upload wheels id: upload-wheel uses: actions/upload-artifact@v3 - env: - ACTIONS_RUNTIME_TOKEN: abc with: name: maturin-wheel path: ./dist/*.whl @@ -588,7 +584,9 @@ jobs: name: maturin-wheel path: ./dist/*.whl - name: Install maturin whl - run: pip install --force-reinstall ./dist/*.whl + run: | + WHEEL_FILE=$(ls ./dist/*.whl) + pip install --force-reinstall "$WHEEL_FILE" - name: pip install run: | export MATURIN_PEP517_ARGS="--verbose" From 0be8c05509dde6ed47fb0b4d42450c9cbaf6ba73 Mon Sep 17 00:00:00 2001 From: owenlch Date: Mon, 21 Oct 2024 22:57:27 +0800 Subject: [PATCH 16/16] Fix windows build --- src/build_context.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/build_context.rs b/src/build_context.rs index 1fb102393..a66398acc 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1179,10 +1179,8 @@ impl BuildContext { #[cfg(windows)] { Command::new("cmd") - .args(&["/C", tf.path().to_str().unwrap()]) - .args("-y") - .arg("--default-toolchain") - .arg("none") + .args(["/C", tf.path().to_str().unwrap()]) + .args(vec!["-y", "--default-toolchain", "none"]) .env("RUSTUP_HOME", rustup_home) .env("CARGO_HOME", cargo_home) .status()?; @@ -1220,8 +1218,8 @@ impl BuildContext { let new_path = format!("{};{}", cargo_bin_path, current_path); env::set_var("PATH", &new_path); Command::new("cmd") - .args(&["/C", "rustup default stable"]) - .status(); + .args(vec!["/C", "rustup default stable"]) + .status()?; } Ok(())