From c7d9e31641bb97a66bca8f4947b55a6ac92506c9 Mon Sep 17 00:00:00 2001 From: Florian Finkernagel Date: Fri, 15 Jul 2022 12:01:05 +0200 Subject: [PATCH] allow 'find-rev-on-first-use' for fetchhg/fetchgit/fetchFromGitHub python packages --- .../anysnake2.toml} | 19 ++- .../t.txt | 11 ++ src/config.rs | 9 +- src/main.rs | 135 +++++++++++++++--- 4 files changed, 137 insertions(+), 37 deletions(-) rename examples/{just_python_trust_on_first_use/anysnake2.toml.backup => just_python_discover_newest_revision/anysnake2.toml} (52%) create mode 100644 examples/just_python_discover_newest_revision/t.txt diff --git a/examples/just_python_trust_on_first_use/anysnake2.toml.backup b/examples/just_python_discover_newest_revision/anysnake2.toml similarity index 52% rename from examples/just_python_trust_on_first_use/anysnake2.toml.backup rename to examples/just_python_discover_newest_revision/anysnake2.toml index 3fa7203..3987b1f 100644 --- a/examples/just_python_trust_on_first_use/anysnake2.toml.backup +++ b/examples/just_python_discover_newest_revision/anysnake2.toml @@ -20,17 +20,16 @@ ecosystem_date="2022-02-16" # you get whatever packages the solver would have pr [python.packages] -# you can use standard python requirements.txt version specification syntax -# i.e. version specifiers from https://www.python.org/dev/peps/pep-0440/#id53 -# you can refer to the repos you cloned -plotnine = { method = "fetchgit", url = "https://github.com/has2k1/plotnine", rev = "6c82cdc20d6f81c96772da73fc07a672a0a0a6ef"} -# and for mercurial -# mercurial example -wormhole = {method = "fetchhg", url="https://hg.sr.ht/~cwt/wormhole", rev="9cabfaa1c96f0d8d362613c99f562796b8a5c1c1"} -# -# and you can fetch from github, git and mercurial (any nix fetcher actually, see +# see other examples for the regular syntax, this +# is just 'how to wrap nix-source-fetchers' +# for github +plotnine = { method = "fetchgit", url = "https://github.com/has2k1/plotnine", rev = "69a5289f625202411e2e543aad9c7f2cb9954fc7", hash_69a5289f625202411e2e543aad9c7f2cb9954fc7 = "sha256-EWf6zs54UOHUG1G51tzogSN4VYAL9UrZXdsmesYaCIc=", branchName = "dev" } +# for mercurial +lvr = {method = "fetchhg", url="https://hg.sr.ht/~bwe/lvr", rev = "db6f0a3254fbd3939d6b6b8c6d1711e7129faba1", hash_db6f0a3254fbd3939d6b6b8c6d1711e7129faba1 = "sha256-r2yDQ4JuOAZ7oWfjat2R/5OcMi0q7BY1QCK/Z9hyeyY=" } +## +# you can fetch from github, git and mercurial (any nix fetcher actually, see # https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers) -# if using fetchFromGitHub, the necessary hash will be added to this file +# if using fetchFromGitHub or fetchhg, the necessary hash will be added to this file # on a trust-on-first-use-basis diff --git a/examples/just_python_discover_newest_revision/t.txt b/examples/just_python_discover_newest_revision/t.txt new file mode 100644 index 0000000..df3fb8e --- /dev/null +++ b/examples/just_python_discover_newest_revision/t.txt @@ -0,0 +1,11 @@ +using https://hg.sr.ht/~bwe/lvr +sending capabilities command +sending lookup command +preparing listkeys for "namespaces" +sending listkeys command +received listkey for "namespaces": 30 bytes +preparing listkeys for "bookmarks" +sending listkeys command +received listkey for "bookmarks": 0 bytes +db6f0a3254fbd3939d6b6b8c6d1711e7129faba1 +(sent 4 HTTP requests and 1029 bytes; received 562 bytes in responses) diff --git a/src/config.rs b/src/config.rs index 4b4e0c2..ab0d8aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -206,18 +206,11 @@ where if !def.contains_key("repo") { errors.push("Was missing 'repo' key.") } - if !def.contains_key("rev") { - errors.push("Was missing 'rev' key.") - } } - "fetchGit" => { + "fetchGit" | "fetchhg" => { if !def.contains_key("url") { errors.push("Was missing 'url' key.") } - - if !def.contains_key("rev") { - errors.push("Was missing 'rev' key.") - } } _ => {} }, diff --git a/src/main.rs b/src/main.rs index e13348c..3301bd0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ use log::{debug, error, info, trace, warn}; use regex::Regex; use serde::Deserialize; use serde_json::json; +use std::borrow::Cow; use std::io::BufRead; use std::io::Write; use std::path::{Path, PathBuf}; @@ -1503,6 +1504,8 @@ enum PrefetchHashResult { HaveToUseFetchGit, } +/// if no hash_{rev} is set, discover it and update anysnake2.toml +/// if no rev is set, discover it as well fn apply_trust_on_first_use( config: &config::ConfigToml, python_build_packages: &mut Vec<(String, HashMap)>, @@ -1522,7 +1525,22 @@ fn apply_trust_on_first_use( let mut hash_key = "".to_string(); match &method[..] { "fetchFromGitHub" => { - let rev = spec.get("rev").expect("missing rev").to_string(); + let rev = { + match spec.get("rev") { + Some(x) => x.to_string(), + None => { + println!("Using discover-newest on first use for python package {}, updating your anysnake2.toml", k); + let owner = spec.get("owner").expect("missing owner").to_string(); + let repo = spec.get("repo").expect("missing repo").to_string(); + let url = format!("https://github.com/{}/{}", owner, repo); + let rev = discover_newest_rev_git(&url, spec.get("branchName"))?; + store_rev(spec, &mut doc, k.to_owned(), &rev); + write = true; + rev + } + } + }; + hash_key = format!("hash_{}", rev); if !spec.contains_key(&hash_key) { write = true; @@ -1548,11 +1566,14 @@ fn apply_trust_on_first_use( value(format!("https://github.com/{}/{}", owner, repo)); out["rev"] = value(&rev); out[&hash_key] = value(&hash); + if spec.contains_key("branchName") { + out["branchName"] = value(spec.get("branchName").unwrap()); + } doc["python"]["packages"][k.to_owned()] = toml_edit::Item::Value( toml_edit::Value::InlineTable(out.into_inline_table()), ); - spec.retain(|_, _| false); + spec.retain(|k, _| {k == "branchName"}); spec.insert("method".to_string(), "fetchgit".into()); spec.insert(hash_key.clone(), hash); spec.insert("url".to_string(), fetchgit_url); @@ -1568,10 +1589,20 @@ fn apply_trust_on_first_use( .get("url") .expect("missing url on fetchgit") .to_string(); - let rev = spec - .get("rev") - .expect("missing rev on fetchgit") - .to_string(); + let rev = { + match spec.get("rev") { + Some(x) => x.to_string(), + None => { + println!("Using discover-newest on first use for python package {}, updating your anysnake2.toml", k); + let rev = discover_newest_rev_git(&url, spec.get("branchName"))?; + println!("\tDiscovered revision {}", &rev); + store_rev(spec, &mut doc, k.to_owned(), &rev); + write = true; + rev + } + } + }; + hash_key = format!("hash_{}", rev); if !spec.contains_key(&hash_key) { println!("Using Trust-On-First-Use for python package {}, updating your anysnake2.toml", k); @@ -1583,14 +1614,19 @@ fn apply_trust_on_first_use( } } "fetchhg" => { - let url = spec - .get("url") - .expect("missing url on fetchhg") - .to_string(); - let rev = spec - .get("rev") - .expect("missing rev on fetchhg") - .to_string(); + let url = spec.get("url").expect("missing url on fetchhg").to_string(); + let rev = { + match spec.get("rev") { + Some(x) => x.to_string(), + None => { + println!("Using discover-newest on first use for python package {}, updating your anysnake2.toml", k); + let rev = discover_newest_rev_hg(&url)?; + println!("\tDiscovered revision {}", &rev); + store_rev(spec, &mut doc, k.to_owned(), &rev); + rev + } + } + }; hash_key = format!("hash_{}", rev); if !spec.contains_key(&hash_key) { println!("Using Trust-On-First-Use for python package {}, updating your anysnake2.toml", k); @@ -1636,6 +1672,18 @@ fn store_hash( spec.insert(hash_key.to_string(), hash.to_owned()); } +/// helper for discover_rev_on_first_use +fn store_rev( + spec: &mut HashMap, + doc: &mut toml_edit::Document, + key: String, // teh package + rev: &String, +) -> () { + use toml_edit::value; + doc["python"]["packages"][key]["rev"] = value(rev); + spec.insert("rev".to_string(), rev.to_owned()); +} + fn prefetch_git_hash(url: &str, rev: &str, outside_nixpkgs_url: &str) -> Result { let nix_prefetch_git_url = format!("{}#nix-prefetch-git", outside_nixpkgs_url); let nix_prefetch_git_url_args = &[ @@ -1683,14 +1731,14 @@ fn prefetch_hg_hash(url: &str, rev: &str, outside_nixpkgs_url: &str) -> Result Result { let url = format!( "https://github.com/{owner}/{repo}/archive/{git_hash}.tar.gz", @@ -1708,10 +1756,10 @@ fn prefetch_github_hash(owner: &str, repo: &str, git_hash: &str) -> Result Result<( }; Ok(()) } + +fn discover_newest_rev_git(url: &String, branch: Option<&String>) -> Result { + let refs = match branch { + Some(x) => Cow::from(format!("refs/heads/{}", x)), + None => Cow::from("HEAD"), + }; + let output = run_without_ctrl_c(|| { + //todo: run this is in the provided nixpkgs! + Ok(std::process::Command::new("git") + .args(&["ls-remote", url, &refs]) + .output()?) + }) + .expect("git ls-remote failed"); + let stdout = + std::str::from_utf8(&output.stdout).expect("utf-8 decoding failed no hg id --debug"); + let hash_re = Regex::new(&format!("^([0-9a-z]{{40}})\\s+{}", &refs)).unwrap(); //hash is on a line together with the ref... + for group in hash_re.captures_iter(stdout) { + return Ok(group[1].to_string()); + } + Err(anyhow!( + "Could not find revision hash in 'git ls-remote {} {}' output.{}", + url, + refs, + if branch.is_some() { + " Is your branchName correct?" + } else { + "" + } + )) +} +fn discover_newest_rev_hg(url: &String) -> Result { + let output = run_without_ctrl_c(|| { + //todo: run this is in the provided nixpkgs! + Ok(std::process::Command::new("hg") + .args(&["id", "--debug", url, "--id"]) + .output()?) + }) + .with_context(||format!("hg id --debug {} failed", url))?; + let stdout = + std::str::from_utf8(&output.stdout).expect("utf-8 decoding failed no hg id --debug"); + let hash_re = Regex::new("(?m)^([0-9a-z]{40})$").unwrap(); //hash is on it's own line. + for group in hash_re.captures_iter(stdout) { + return Ok(group[0].to_string()); + } + Err(anyhow!( + "Could not find revision hash in 'hg id --debug {}' output", + url + )) +}