diff --git a/Cargo.lock b/Cargo.lock index 1f6140d..30bbc92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,7 +46,7 @@ dependencies = [ [[package]] name = "anysnake2" -version = "1.11.0" +version = "1.12.0" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index c0903c9..9f6cda8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "anysnake2" -version = "1.11.1" +version = "1.12.0" authors = ["Florian Finkernagel "] edition = "2021" diff --git a/examples/full/anysnake2.toml b/examples/full/anysnake2.toml index 613e5fc..093915c 100644 --- a/examples/full/anysnake2.toml +++ b/examples/full/anysnake2.toml @@ -81,6 +81,15 @@ plotnine = {method = "fetchFromGitHub", owner = "has2k1", repo = "plotnine", rev lvr = {method = "fetchhg", url="https://hg.sr.ht/~bwe/lvr", rev="db6f0a3254fbd3939d6b6b8c6d1711e7129faba1", hash_db6f0a3254fbd3939d6b6b8c6d1711e7129faba1 = "sha256-r2yDQ4JuOAZ7oWfjat2R/5OcMi0q7BY1QCK/Z9hyeyY=" } # pandas="<1.0" +# if you have packages that depend on other packages that you define using the 'method' method +# you unfortunatly must tell anysnake2 about it by providing a list in overrides +# like it's done for 'testrepo' (which depends on testrepo2) here. +testrepo={method="fetchFromGitHub", owner="TyberiusPrime", repo="_anysnake2_test_repo", overrides = ["testrepo2"], rev = "97d57e17c1bd4a5f547fa1c1be57c2f0a1d2ec6f", hash_97d57e17c1bd4a5f547fa1c1be57c2f0a1d2ec6f = "sha256-mZw37fLouWrA2L+49UOfUsF1MDy/q5pJImw+zczE4wU=" } +testrepo2={method="fetchFromGitHub", owner="TyberiusPrime", repo="_anysnake2_test_repo2", rev = "a42420f8ba0a6bc9bda0425cd665515fb92dc2b4", hash_a42420f8ba0a6bc9bda0425cd665515fb92dc2b4 = "sha256-tLz9vDTxQqFZPKkkBOZmmNNEhtf6JK2nwWiBKNH6od8="} + + + + [clones.code] # target directory # seperate from python packages so you can clone other stuff as well dppd="@gh/TyberiusPrime" # one /-> github.com/TyberiusPrime/dppd diff --git a/examples/python_buildPackage_interdependency_with_overrides/anysnake2.toml b/examples/python_buildPackage_interdependency_with_overrides/anysnake2.toml new file mode 100644 index 0000000..af1fef5 --- /dev/null +++ b/examples/python_buildPackage_interdependency_with_overrides/anysnake2.toml @@ -0,0 +1,38 @@ +# basic anysnake2.toml example +# package settings +[anysnake2] +rev = "dev" + +[outside_nixpkgs] +rev = "22.05" # the nixpgks version or github hash + +[nixpkgs] +# the nixpkgs used inside the container +rev = "22.05" # the nixpgks version or github hash +packages = ["which"] + + +[python] # python section is optional +version="3.10" # does not go down to 3.x.x. That's implicit in the nixpkgs (for now) +ecosystem_date="2022-11-23" # you get whatever packages the solver would have produced on that day + +# additional_mkpython_arguments = """ +# """ # must be verbatim nix code + +[clones.code] +# example-cli-python="git+https://github.com/ojixzzz/example-cli-python" + +[python.packages] +# you can use standard python requirements.txt version specification syntax +# i.e. version specifiers from https://www.python.org/dev/peps/pep-0440/#id53 +# you can refer to the repos you cloned +testrepo={method="fetchFromGitHub", owner="TyberiusPrime", repo="_anysnake2_test_repo", overrides = ["testrepo2"], rev = "97d57e17c1bd4a5f547fa1c1be57c2f0a1d2ec6f", hash_97d57e17c1bd4a5f547fa1c1be57c2f0a1d2ec6f = "sha256-mZw37fLouWrA2L+49UOfUsF1MDy/q5pJImw+zczE4wU=" } +testrepo2={method="fetchFromGitHub", owner="TyberiusPrime", repo="_anysnake2_test_repo2", rev = "a42420f8ba0a6bc9bda0425cd665515fb92dc2b4", hash_a42420f8ba0a6bc9bda0425cd665515fb92dc2b4 = "sha256-tLz9vDTxQqFZPKkkBOZmmNNEhtf6JK2nwWiBKNH6od8="} + + +# and you can fetch from github, git and mercurial (any nix fetcher actually, see +# https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers) +# if using fetchFromGitHub, the necessary hash will be added to this file +# on a trust-on-first-use-basis + + diff --git a/src/config.rs b/src/config.rs index 85585d5..42395ec 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use itertools::Itertools; use serde::de::Deserializer; use serde::Deserialize; use std::collections::HashMap; @@ -45,7 +46,6 @@ pub struct ConfigToml { //todo: refactor - impl ConfigToml { pub fn from_str(raw_config: &str) -> Result { let mut res: ConfigToml = toml::from_str(&raw_config)?; @@ -64,10 +64,9 @@ impl ConfigToml { fs::canonicalize(config_file).context("Could not find config file")?; let raw_config = fs::read_to_string(&abs_config_path).context("Could not read config file")?; - let mut parsed_config: ConfigToml = Self::from_str(&raw_config) - .with_context(|| { - crate::ErrorWithExitCode::new(65, format!("Failure parsing {:?}", &abs_config_path)) - })?; + let mut parsed_config: ConfigToml = Self::from_str(&raw_config).with_context(|| { + crate::ErrorWithExitCode::new(65, format!("Failure parsing {:?}", &abs_config_path)) + })?; parsed_config.anysnake2_toml_path = Some(abs_config_path); Ok(parsed_config) } @@ -92,8 +91,8 @@ impl MinimalConfigToml { fs::canonicalize(config_file).context("Could not find config file")?; let raw_config = fs::read_to_string(&abs_config_path).context("Could not read config file")?; - let mut parsed_config: MinimalConfigToml = Self::from_str(&raw_config) - .with_context(|| { + let mut parsed_config: MinimalConfigToml = + Self::from_str(&raw_config).with_context(|| { crate::ErrorWithExitCode::new(65, format!("Failure parsing {:?}", &abs_config_path)) })?; parsed_config.anysnake2_toml_path = Some(abs_config_path); @@ -227,9 +226,49 @@ impl WithDefaultFlakeSource for Rust { #[derive(Deserialize, Debug)] #[serde(untagged)] +pub enum ParsedPythonPackageDefinition { + Requirement(String), + BuildPythonPackage(HashMap), +} + +#[derive(Debug, Clone)] +pub struct BuildPythonPackageInfo { + options: HashMap, + pub overrides: Option>, +} + +impl BuildPythonPackageInfo { + pub fn get(&self, key: &str) -> Option<&String> { + self.options.get(key) + } + pub fn contains_key(&self, key: &str) -> bool { + self.options.contains_key(key) + } + pub fn insert(&mut self, key: String, value: String) -> Option { + self.options.insert(key, value) + } + pub fn retain(&mut self, f: F) + where + F: FnMut(&String, &mut String) -> bool, + { + self.options.retain(f); + } + + pub fn src_to_nix(&self) -> String { + let mut res = Vec::new(); + for (k, v) in self.options.iter().sorted_by_key(|x| x.0) { + if k != "method" && k != "buildInputs" { + res.push(format!("\"{}\" = \"{}\";", k, v)); + } + } + res.join("\n") + } +} + +#[derive(Debug, Clone)] pub enum PythonPackageDefinition { Requirement(String), - BuildPythonPackage(HashMap), + BuildPythonPackage(BuildPythonPackageInfo), } //todo: trust on first use, or at least complain if never seen before rev and @@ -241,43 +280,110 @@ fn de_python_package_definition<'de, D>( where D: Deserializer<'de>, { - let res: HashMap = + let parsed: HashMap = crate::maps_duplicate_key_is_error::deserialize(deserializer)?; - for (k, v) in res.iter() { - match v { - PythonPackageDefinition::Requirement(_) => {} - PythonPackageDefinition::BuildPythonPackage(def) => { + let res: Result, D::Error> = parsed + .into_iter() + .map(|(pkg_name, v)| match v { + ParsedPythonPackageDefinition::Requirement(x) => { + Ok((pkg_name, PythonPackageDefinition::Requirement(x))) + } + ParsedPythonPackageDefinition::BuildPythonPackage(def) => { let mut errors: Vec<&str> = Vec::new(); - match def.get("method") { - Some(method) => match &method[..] { - "fetchFromGitHub" => { - if !def.contains_key("owner") { - errors.push("Was missing 'owner' key.") - } - if !def.contains_key("repo") { - errors.push("Was missing 'repo' key.") - } + let method = def + .get("method") + .ok_or_else(|| { + serde::de::Error::custom(format!( + "Missing method on python package {}", + pkg_name + )) + })? + .as_str() + .ok_or_else(|| { + serde::de::Error::custom(format!( + "method must be a string on python package {}", + pkg_name + )) + })?; + match &method[..] { + "fetchFromGitHub" => { + if !def.contains_key("owner") { + errors.push("Was missing 'owner' key.") } - "fetchGit" | "fetchhg" => { - if !def.contains_key("url") { - errors.push("Was missing 'url' key.") - } + if !def.contains_key("repo") { + errors.push("Was missing 'repo' key.") } - _ => {} - }, - None => errors.push("Was missing 'method' value, e.g. fetchFromGitHub"), - }; + } + "fetchGit" | "fetchhg" => { + if !def.contains_key("url") { + errors.push("Was missing 'url' key.") + } + } + _ => {} + } if !errors.is_empty() { return Err(serde::de::Error::custom(format!( "Python.packages.{}: {}", - k, + pkg_name, errors.join("\n") ))); } + let overrides = match def.get("overrides") { + None => None, + Some(toml::Value::Array(input)) => { + let mut output: Vec = Vec::new(); + for ov in input.into_iter() { + output.push( + ov.as_str() + .ok_or(serde::de::Error::custom(format!( + "Overrides must be an array of strings. Python package {}", + pkg_name, + )))? + .to_string(), + ); + } + Some(output) + } + Some(_) => { + return Err(serde::de::Error::custom(format!( + "Overrides must be an array of strings. Python package {}", + pkg_name, + ))); + } + }; + let string_defs: Result, D::Error> = def + .into_iter() + .filter_map(|(k, v)| match v { + toml::Value::String(v) => Some(Ok((k, v))), + toml::Value::Array(_) => { + if k != "overrides" { + return Some(Err(serde::de::Error::custom(format!( + "Field {} on python package {} must be a string ", + k, pkg_name + )))); + } else { + None + } + } + _ => { + return Some(Err(serde::de::Error::custom(format!( + "Field {} on python package {} must be a string ", + k, pkg_name + )))); + } + }) + .collect(); + Ok(( + pkg_name, + PythonPackageDefinition::BuildPythonPackage(BuildPythonPackageInfo { + options: string_defs?, + overrides: overrides, + }), + )) } - } - } - Ok(res) + }) + .collect(); + res } #[derive(Deserialize, Debug)] diff --git a/src/flake_template.nix b/src/flake_template.nix index 76239b2..236ab65 100644 --- a/src/flake_template.nix +++ b/src/flake_template.nix @@ -206,9 +206,10 @@ ''; patches = []; }; + } #%PYTHON_BUILD_PACKAGES% #%PYTHON_ADDITIONAL_MKPYTHON_ARGUMENTS% - }; + ; }; in rec { defaultPackage = buildSymlinkImage _args; diff --git a/src/flake_writer.rs b/src/flake_writer.rs index b70649c..d08819d 100644 --- a/src/flake_writer.rs +++ b/src/flake_writer.rs @@ -1,4 +1,4 @@ -use crate::config; +use crate::config::{self, BuildPythonPackageInfo}; use anyhow::{anyhow, bail, Context, Result}; use chrono::{NaiveDate, NaiveDateTime}; use ex::fs; @@ -70,7 +70,7 @@ pub fn write_flake( flake_dir: impl AsRef, parsed_config: &mut config::ConfigToml, python_packages: &[(String, String)], - python_build_packages: &HashMap>, // those end up as buildPythonPackages + python_build_packages: &HashMap, // those end up as buildPythonPackages use_generated_file_instead: bool, ) -> Result { let template = std::include_str!("flake_template.nix"); @@ -177,7 +177,7 @@ pub fn write_flake( .replace("#%PYTHON_BUILD_PACKAGES%", &out_python_build_packages) .replace( "#%PYTHON_ADDITIONAL_MKPYTHON_ARGUMENTS%", - &&out_additional_mkpython_arguments, + &format!("// {{{}}}", out_additional_mkpython_arguments), ) .replace("%PYTHON_MAJOR_DOT_MINOR%", &python_major_dot_minor) .replace("%PYPI_DEPS_DB_REV%", &pypi_debs_db_rev) @@ -290,7 +290,7 @@ pub fn write_flake( if let Some(r) = &parsed_config.r { if r.ecosystem_tag.is_some() { bail!("[R]ecosystem_tag is no longer in use. We're using nixR now and you need to specify a 'date' instead"); - } + } // install R kernel if r.packages.contains(&"IRkernel".to_string()) && jupyter_included { jupyter_kernels.push_str( @@ -496,7 +496,7 @@ fn format_input_defs(inputs: &[InputFlake]) -> String { fn extract_non_editable_python_packages( input: &[(String, String)], - build_packages: &HashMap>, + build_packages: &HashMap, flakes_config: &Option>, ) -> Result> { let mut res = Vec::new(); @@ -544,18 +544,8 @@ fn extract_non_editable_python_packages( Ok(res) } -fn src_to_nix(src: &HashMap) -> String { - let mut res = Vec::new(); - for (k, v) in src.iter().sorted_by_key(|x| x.0) { - if k != "method" && k != "buildInputs" { - res.push(format!("\"{}\" = \"{}\";", k, v)); - } - } - res.join("\n") -} - fn python_version_from_spec( - spec: &HashMap, + spec: &BuildPythonPackageInfo, override_version: Option<&str>, ) -> String { format!( @@ -583,13 +573,26 @@ fn get_flake_rev( } fn format_python_build_packages( - input: &HashMap>, + input: &HashMap, flakes_config: &Option>, flakes_used_for_python_packages: &mut HashSet, ) -> Result { - let mut res: String = "packagesExtra = [".into(); + let mut res: String = "".into(); let mut providers: String = "".into(); + let mut packages_extra: Vec = Vec::new(); for (key, spec) in input.iter().sorted_by_key(|x| x.0) { + let overrides = match &spec.overrides { + Some(ov_packages) => { + let mut ov = "overridesPre = [ (self: super: { ".to_string(); + for p in ov_packages { + ov.push_str(&format!("{} = {}_pkg;\n", p, p)); + } + + ov.push_str(" } ) ];"); + ov + } + None => "".to_string(), + }; match spec .get("method") .expect("no method in package definition") @@ -600,35 +603,56 @@ fn format_python_build_packages( let flake_rev = get_flake_rev(flake_name, flakes_config) .with_context(|| format!("python.packages.{}", key))?; res.push_str(&format!( - "({}.mach-nix-build-python-package pkgs mach-nix_ \"{}\")\n", + "{}_pkg = ({}.mach-nix-build-python-package pkgs mach-nix_ \"{}\");\n", + //todo: shohorn the overrides into this?! + flake_name, flake_name, python_version_from_spec(spec, Some(&flake_rev)) )); flakes_used_for_python_packages.insert(flake_name.to_string()); + packages_extra.push(flake_name.to_string()); } _ => { res.push_str(&format!( - " - (mach-nix_.buildPythonPackage {{ + "{}_pkg = (mach-nix_.buildPythonPackage {{ version=\"{}\"; src = pkgs.{} {{ # {} {} }}; - }})", + {} + }});\n", + key, python_version_from_spec(&spec, None), spec.get("method") .expect("Missing 'method' on python build package definition"), key, - src_to_nix(spec), + spec.src_to_nix(), + overrides )); + packages_extra.push(key.to_string()); } } providers.push_str(&format!("providers.{} = \"nixpkgs\";\n", key)); } - res.push_str("];"); - res.push_str("\n"); - res.push_str(&providers); - Ok(res) + + let mut out: String = "// (let ".into(); + out.push_str(&res); + out.push_str("machnix_overrides = (self: super: {"); + for pkg in packages_extra.iter() { + out.push_str(&format!("{} = {}_pkg;\n", pkg, pkg)); + } + out.push_str("} );\n"); + out.push_str("in { packagesExtra = ["); + for pkg in packages_extra.iter() { + out.push_str(pkg); + out.push_str("_pkg "); + } + out.push_str("]"); + out.push_str("\n; overridesPre = [ machnix_overrides ];\n"); + out.push_str("\n"); + out.push_str(&providers); + out.push_str("})\n"); + Ok(out) } fn pypi_deps_date_to_rev(date: NaiveDate, flake_dir: impl AsRef) -> Result { diff --git a/src/main.rs b/src/main.rs index 63cb7b5..f356809 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ extern crate clap; use anyhow::{anyhow, bail, Context, Result}; use clap::{value_t, App, AppSettings, Arg, ArgMatches, SubCommand}; -use config::PythonPackageDefinition; +use config::{BuildPythonPackageInfo, PythonPackageDefinition}; use ex::fs; use indoc::indoc; use lazy_static::lazy_static; @@ -286,12 +286,12 @@ fn collect_python_packages( parsed_config: &mut config::ConfigToml, ) -> Result<( Vec<(String, String)>, - HashMap>, + HashMap, )> { Ok(match &mut parsed_config.python { Some(python) => { let mut requirement_packages: Vec<(String, String)> = Vec::new(); - let mut build_packages: HashMap> = HashMap::new(); + let mut build_packages: HashMap = HashMap::new(); for (name, pp) in python.packages.drain() { match pp { PythonPackageDefinition::Requirement(str_package_definition) => { @@ -1664,7 +1664,7 @@ enum PrefetchHashResult { /// if no rev is set, discover it as well fn apply_trust_on_first_use( config: &config::ConfigToml, - python_build_packages: &mut HashMap>, + python_build_packages: &mut HashMap, outside_nixpkgs_url: &str, ) -> Result<()> { if !python_build_packages.is_empty() { @@ -1819,7 +1819,7 @@ fn apply_trust_on_first_use( /// helper for apply_trust_on_first_use fn store_hash( - spec: &mut HashMap, + spec: &mut BuildPythonPackageInfo, doc: &mut toml_edit::Document, key: String, hash_key: &str, @@ -1831,7 +1831,7 @@ fn store_hash( /// helper for discover_rev_on_first_use fn store_rev( - spec: &mut HashMap, + spec: &mut BuildPythonPackageInfo, doc: &mut toml_edit::Document, key: String, // teh package rev: &String, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 3ae78a2..e2586f4 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -401,3 +401,21 @@ fn test_python_310_nixpkgs_2205() { assert!(code == 0); assert!(stdout.contains("3.5.5")); } + +#[test] +fn test_python_buildpackage_interdependency_with_overrides() { + let (code, stdout, _stderr) = run_test( + "examples/python_buildPackage_interdependency_with_overrides//", + &[ + "run", + "--", + "python", + "-c", + "'import testrepo; print(testrepo.__version__); print(testrepo.testrepo2.__version__)'", + ], + ); + assert!(code == 0); + assert!(stdout.contains("0.66")); + assert!(stdout.contains("0.33")); + +}