diff --git a/Cargo.lock b/Cargo.lock index 152aff0..cca4269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,13 +49,17 @@ name = "anysnake2" version = "1.1.0" dependencies = [ "anyhow", + "base32", + "base64", "chrono", "clap", "ctrlc", "ex", + "hex", "lazy_static", "log", "named-lock", + "nix-base32", "regex", "serde", "serde_json", @@ -64,6 +68,7 @@ dependencies = [ "tempdir", "terminal_size", "toml", + "toml_edit", "ureq", "url", "whoami", @@ -101,6 +106,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base32" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ce669cd6c8588f79e15cf450314f9638f967fc5770ff1c7c1deb0925ea7cfa" + [[package]] name = "base64" version = "0.13.0" @@ -128,6 +139,12 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "cc" version = "1.0.71" @@ -189,6 +206,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "combine" +version = "4.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b727aacc797f9fc28e355d21f34709ac4fc9adecfe470ad07b8f4464f53062" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.1" @@ -217,6 +244,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "ex" version = "0.1.3" @@ -258,6 +291,12 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -284,6 +323,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -299,6 +357,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kstring" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b310ccceade8121d7d77fee406160e457c2f4e7c7982d589da3499bc7ea4526" +dependencies = [ + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -387,6 +454,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix-base32" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548db8274cf1b2b4c093557783f99e9ad64ffdaaa29a6c1af0abc9895c15612" + [[package]] name = "num-integer" version = "0.1.44" @@ -804,6 +877,18 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b80ac5e1b91e3378c63dab121962472b5ca20cf9ab1975e3d588548717807a8" +dependencies = [ + "combine", + "indexmap", + "itertools", + "kstring", +] + [[package]] name = "typenum" version = "1.14.0" diff --git a/Cargo.toml b/Cargo.toml index 01db78c..51742d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,21 +11,25 @@ anyhow = {version="1.0.44", features=["backtrace"]} chrono = "0.4" clap = "2.33" ctrlc = {version="3.2.1", features = ["termination"] } +ex = "0.1.3" +lazy_static = "1.4.0" log = "0.4.14" regex = "1" serde = {version="1.0.130", features = ["derive"]} serde_json = "1.0" -# serde_with = "1.10.0" # broke clippy on darling_marco +sha256 = "1.0.2" stderrlog = "0.5.1" tempdir = "0.3.7" +terminal_size = "0.1.17" toml = "0.5" +toml_edit = "0.13.0" ureq = "2.0" -lazy_static = "1.4.0" -sha256 = "1.0.2" -whoami = "1.1.5" -terminal_size = "0.1.17" -ex = "0.1.3" url = "2.2.2" +whoami = "1.1.5" +base32 = "0.4.0" +base64 = "0.13.0" [dev-dependencies] named-lock = "0.1.1" +hex = "0.4" +nix-base32="0.1" diff --git a/examples/just_python/anysnake2.toml b/examples/just_python/anysnake2.toml index d610c11..aefc0f8 100644 --- a/examples/just_python/anysnake2.toml +++ b/examples/just_python/anysnake2.toml @@ -30,12 +30,13 @@ example-cli-python="editable/code" # https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers) #ugly syntax: -# plotnine = {method = "fetchFromGitHub", owner = "TyberiusPrime", repo = "dppd", rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156", hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="} +plotnine = {method = "fetchFromGitHub", owner = "has2k1", repo = "plotnine", rev = "6c82cdc20d6f81c96772da73fc07a672a0a0a6ef", hash = "sha256-E5nR5xK+sqV3tlxnPDNE0TdTtYtPK47zgwzTG/KmXF0="} # pretty syntax -[python.packages.plotnine] +[python.packages.dppd] method = "fetchFromGitHub" owner = "TyberiusPrime" repo = "dppd" rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156" - hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI=" +hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI=" + #hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI=" # pandas="<1.0" diff --git a/examples/just_python/test.toml b/examples/just_python/test.toml new file mode 100644 index 0000000..fe203c7 --- /dev/null +++ b/examples/just_python/test.toml @@ -0,0 +1,46 @@ +# basic anysnake2.toml example +# package settings +[anysnake2] +rev = "dev" + +[outside_nixpkgs] +rev = "21.05" # the nixpgks version or github hash + +[nixpkgs] +# the nixpkgs used inside the container +rev = "21.05" # the nixpgks version or github hash + + +[python] # python section is optional +version="3.8" # does not go down to 3.8.x. That's implicit in the nixpkgs (for now) +ecosystem_date="2021-08-16" # you get whatever packages the solver would have produced on that day + +[clones.code] +example-cli-python="git+https://github.com/ojixzzz/example-cli-python" + +[python.packages] +# you can use standard python requirements.txt version specification syntax +# i.e. version specifiers from https://www.python.org/dev/peps/pep-0440/#id53 +pandas="1.2" +# you can refer to the repos you cloned + +example-cli-python="editable/code" + +# and you can fetch from github, git and mercurial (any nix fetcher actually, see +# https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers) + +#ugly syntax: +plotnine = {method = "fetchFromGitHub", owner = "has2k1", repo = "plotnine", rev = "6c82cdc20d6f81c96772da73fc07a672a0a0a6ef", hash = """ +sha256-E5nR5xK+sqV3tlxnPDNE0TdTtYtPK47zgwzTG/KmXF0= +"""} +# pretty syntax +[python.packages.dppd] + method = "fetchFromGitHub" + owner = "TyberiusPrime" + repo = "dppd" + rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156" +hash = """ +sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI= +""" + #hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI=" +# pandas="<1.0" diff --git a/src/config.rs b/src/config.rs index 16e3798..b56abb5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -34,6 +34,8 @@ pub struct ConfigToml { pub dev_shell: DevShell, #[serde(rename = "R")] pub r: Option, + #[serde(skip)] + pub source: PathBuf, } impl ConfigToml { diff --git a/src/main.rs b/src/main.rs index 85f3e04..1a4eff8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -233,6 +233,7 @@ fn read_config(matches: &ArgMatches<'static>) -> Result { ErrorWithExitCode::new(65, format!("Failure parsing {:?}", &abs_config_path)) })?; parsed_config.anysnake2_toml_path = Some(abs_config_path); + parsed_config.source = config_file.into(); Ok(parsed_config) } @@ -286,7 +287,10 @@ fn switch_to_configured_version( fn collect_python_packages( parsed_config: &mut config::ConfigToml, -) -> Result<(Vec<(String, String)>, Vec<(String, HashMap)>)> { +) -> Result<( + Vec<(String, String)>, + Vec<(String, HashMap)>, +)> { Ok(match &mut parsed_config.python { Some(python) => { let mut requirement_packages: Vec<(String, String)> = Vec::new(); @@ -394,8 +398,13 @@ fn inner_main() -> Result<()> { lookup_clones(&mut parsed_config)?; perform_clones(&parsed_config)?; - let (python_packages, python_build_packages) = collect_python_packages(&mut parsed_config)?; - trace!("python packages: {:?} {:?}", python_packages, python_build_packages); + let (python_packages, mut python_build_packages) = collect_python_packages(&mut parsed_config)?; + trace!( + "python packages: {:?} {:?}", + python_packages, + python_build_packages + ); + apply_trust_on_first_use(&parsed_config, &mut python_build_packages)?; let use_generated_file_instead = parsed_config.anysnake2.do_not_modify_flake.unwrap_or(false); let flake_changed = flake_writer::write_flake( @@ -1375,3 +1384,81 @@ fn write_develop_python_path( )?; Ok(()) } + +fn apply_trust_on_first_use( + config: &config::ConfigToml, + python_build_packages: &mut Vec<(String, HashMap)>, +) -> Result<()> { + if !python_build_packages.is_empty() { + use toml_edit::{value, Document}; + let toml = std::fs::read_to_string(&config.source).expect("Could not reread config file"); + let mut doc = toml.parse::().expect("invalid doc"); + let mut write = false; + + for (k, spec) in python_build_packages.iter_mut() { + let method = spec + .get("method") + .expect("missing method - should have been caught earlier"); + if method == "fetchFromGitHub" { + write = true; + println!("Using Trust-On-First-Use for python package {}, updating your anysnake2.toml", k); + + let hash = prefetch_github_hash( + spec.get("owner").expect("missing owner"), + spec.get("repo").expect("missing repo"), + spec.get("rev").expect("missing rev"), + )?; + println!("hash is {}", hash); + let key = k.to_owned(); + doc["python"]["packages"][key]["hash"] = value(&hash); + spec.insert("hash".to_string(), hash.to_owned()); + } + } + if write { + let out_toml = doc.to_string(); + std::fs::write("anysnake2.toml", out_toml).expect("failed to rewrite config file"); + } + } + Ok(()) +} + +fn prefetch_github_hash(owner: &str, repo: &str, git_hash: &str) -> Result { + let url = format!( + "https://github.com/{owner}/{repo}/archive/{git_hash}.tar.gz", + owner = owner, + repo = repo, + git_hash = git_hash + ); + + let old_format = Command::new("nix-prefetch-url") + .args(&[&url, "--type", "sha256", "--unpack"]) + .output() + .context(format!("Failed to nix-prefetch {url}", url = url))? + .stdout; + let old_format = std::str::from_utf8(&old_format) + .context("nix-prefetch result was no utf8")? + .trim(); + let new_format = convert_hash_to_subresource_format(old_format)?; + println!("before convert: {}, after: {}", &old_format, &new_format); + Ok(new_format) +} + +fn convert_hash_to_subresource_format(hash: &str) -> Result { + let res = Command::new("nix") + .args(&["hash", "to-sri", "--type", "sha256", hash]) + .output() + .context(format!( + "Failed to nix hash to-sri --type sha256 '{hash}'", + hash = hash + ))? + .stdout; + let res = std::str::from_utf8(&res) + .context("nix hash output was not utf8")? + .trim() + .to_owned(); + if res.is_empty() { + Err(anyhow!("nix hash to-sri returned empty result")) + } else { + Ok(res) + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2e142c0..b5fab01 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,6 +1,7 @@ use named_lock::NamedLock; use std::path::PathBuf; use std::process::Command; +use tempdir::TempDir; fn run_test(cwd: &str, args: &[&str]) -> (i32, String, String) { //can't have more than one running from a given folder at a time @@ -17,8 +18,6 @@ fn run_test(cwd: &str, args: &[&str]) -> (i32, String, String) { std::fs::remove_file(result_dir).unwrap(); } - - let p = std::env::current_exe() .expect("No current exe?") .parent() @@ -61,8 +60,14 @@ fn test_minimal_bash_version() { #[test] fn test_just_python() { + // needs to be copied to test the tofu functionality. + let td = TempDir::new("anysnake_test").expect("could not create tempdir"); + std::fs::copy( + "examples/just_python/anysnake2.toml", + td.path().join("anysnake2.toml"), + ).expect("Could not create anysnake2.toml in tempdir"); let (_code, stdout, _stderr) = run_test( - "examples/just_python", + &td.path().to_string_lossy(), &["run", "--", "python", "--version"], ); assert!(stdout.contains("3.8.9")); @@ -85,14 +90,10 @@ fn test_just_python_pandas_version() { #[test] fn test_just_python_venv_bin() { - let (_code, stdout, _stderr) = run_test( - "examples/just_python", - &["run", "--", "hello"], - ); + let (_code, stdout, _stderr) = run_test("examples/just_python", &["run", "--", "hello"]); assert!(stdout.contains("Argument strings:")); } - #[test] fn test_no_anysnake_toml() { let (code, _stdout, stderr) = run_test( @@ -171,12 +172,12 @@ fn test_full() { .current_dir("examples/full/code/dppd_plotnine") .output() .expect("git log failed"); - assert!(std::str::from_utf8(&out.stdout).unwrap().split('\n') - .next().unwrap().contains("8ed7651af759f3f0b715a2fbda7bf5119f7145d7")) - - - - + assert!(std::str::from_utf8(&out.stdout) + .unwrap() + .split('\n') + .next() + .unwrap() + .contains("8ed7651af759f3f0b715a2fbda7bf5119f7145d7")) } #[test] @@ -185,11 +186,13 @@ fn test_full_r_packages() { let _guad = lock.lock().unwrap(); rm_clones("examples/full"); - let (_code, stdout, _stderr) = run_test("examples/full", &["run", "--", "R", "-e", "'library(ACA);sessionInfo();'"]); + let (_code, stdout, _stderr) = run_test( + "examples/full", + &["run", "--", "R", "-e", "'library(ACA);sessionInfo();'"], + ); assert!(stdout.contains("ACA_1.1")); } - #[test] fn test_full_hello() { let lock = NamedLock::create("anysnaketest_full").unwrap(); @@ -223,48 +226,26 @@ fn test_full_rpy2_sitepaths() { let _guad = lock.lock().unwrap(); rm_clones("examples/full"); - let (_code, stdout, _stderr) = run_test( - "examples/full", - &[ - "test_rpy2" - ], - ); + let (_code, stdout, _stderr) = run_test("examples/full", &["test_rpy2"]); assert!(stdout.contains("Rcpp_1.0.7")); assert!(!stdout.contains("Rcpp_1.0.5")); assert!(stdout.contains("ACA_1.1")); } - #[test] fn test_just_r() { - let (_code, stdout, _stderr) = run_test( "examples/just_r", - &[ - "run", - "--", - "R", - "-e", - "'library(Rcpp); sessionInfo()'" - ], + &["run", "--", "R", "-e", "'library(Rcpp); sessionInfo()'"], ); assert!(stdout.contains("Rcpp_1.0.7")); } - #[test] fn test_flake_with_dir() { - let (_code, stdout, _stderr) = run_test( "examples/flake_in_non_root_github", - &[ - "run", - "--", - "fastq-dump", - "--version", - ], + &["run", "--", "fastq-dump", "--version"], ); assert!(stdout.contains("\"fastq-dump\" version 2.11.2")); } - -