trust-on-first-use for github python packages

Cargo.toml

@@ -11,21 +11,25 @@ anyhow = {version="1.0.44", features=["backtrace"]}
 chrono = "0.4"
 clap = "2.33"
 ctrlc = {version="3.2.1", features = ["termination"] }
+ex = "0.1.3"
+lazy_static = "1.4.0"
 log = "0.4.14"
 regex = "1"
 serde = {version="1.0.130", features = ["derive"]}
 serde_json = "1.0"
-# serde_with = "1.10.0" # broke clippy on darling_marco
+sha256 = "1.0.2"
 stderrlog = "0.5.1"
 tempdir = "0.3.7"
+terminal_size = "0.1.17"
 toml = "0.5"
+toml_edit = "0.13.0"
 ureq = "2.0"
-lazy_static = "1.4.0"
-sha256 = "1.0.2"
-whoami = "1.1.5"
-terminal_size = "0.1.17"
-ex = "0.1.3"
 url = "2.2.2"
+whoami = "1.1.5"
+base32 = "0.4.0"
+base64 = "0.13.0"
 
 [dev-dependencies]
 named-lock = "0.1.1"
+hex = "0.4"
+nix-base32="0.1"

examples/just_python/anysnake2.toml

@@ -30,12 +30,13 @@ example-cli-python="editable/code"
 # https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers)
 
 #ugly syntax:
-# plotnine = {method = "fetchFromGitHub", owner = "TyberiusPrime", repo = "dppd", rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156", hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="}
+plotnine = {method = "fetchFromGitHub", owner = "has2k1", repo = "plotnine", rev = "6c82cdc20d6f81c96772da73fc07a672a0a0a6ef", hash = "sha256-E5nR5xK+sqV3tlxnPDNE0TdTtYtPK47zgwzTG/KmXF0="}
 # pretty syntax
-[python.packages.plotnine] 
+[python.packages.dppd] 
 	method = "fetchFromGitHub"
 	owner = "TyberiusPrime"
 	repo = "dppd"
 	rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156"
-	hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="
+hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="
+	#hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="
 # pandas="<1.0"

examples/just_python/test.toml

@@ -0,0 +1,46 @@
+# basic anysnake2.toml example
+# package settings
+[anysnake2]
+rev = "dev"
+
+[outside_nixpkgs]
+rev = "21.05" # the nixpgks version or github hash
+
+[nixpkgs]
+# the nixpkgs used inside the container
+rev = "21.05" # the nixpgks version or github hash
+
+
+[python] # python section is optional
+version="3.8" # does not go down to 3.8.x. That's implicit in the nixpkgs (for now)
+ecosystem_date="2021-08-16" # you get whatever packages the solver would have produced on that day
+
+[clones.code]
+example-cli-python="git+https://github.com/ojixzzz/example-cli-python"
+
+[python.packages]
+# you can use standard python requirements.txt version specification syntax
+# i.e. version specifiers from https://www.python.org/dev/peps/pep-0440/#id53
+pandas="1.2"
+# you can refer to the repos you cloned
+
+example-cli-python="editable/code"
+
+# and you can fetch from github, git and mercurial (any nix fetcher actually, see
+# https://nixos.org/manual/nixpkgs/stable/#chap-pkgs-fetchers)
+
+#ugly syntax:
+plotnine = {method = "fetchFromGitHub", owner = "has2k1", repo = "plotnine", rev = "6c82cdc20d6f81c96772da73fc07a672a0a0a6ef", hash = """
+sha256-E5nR5xK+sqV3tlxnPDNE0TdTtYtPK47zgwzTG/KmXF0=
+"""}
+# pretty syntax
+[python.packages.dppd] 
+	method = "fetchFromGitHub"
+	owner = "TyberiusPrime"
+	repo = "dppd"
+	rev = "b55ac32ef322a8edfc7fa1b6e4553f66da26a156"
+hash = """
+sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI=
+"""
+	#hash = "sha256-fyDDeJRbm9hMkefqiyxHazZut38rxgZVcyp+YpUglGI="
+# pandas="<1.0"

src/config.rs

@@ -34,6 +34,8 @@ pub struct ConfigToml {
     pub dev_shell: DevShell,
     #[serde(rename = "R")]
     pub r: Option<R>,
+    #[serde(skip)]
+    pub source: PathBuf,
 }
 
 impl ConfigToml {

src/main.rs

@@ -233,6 +233,7 @@ fn read_config(matches: &ArgMatches<'static>) -> Result<config::ConfigToml> {
             ErrorWithExitCode::new(65, format!("Failure parsing {:?}", &abs_config_path))
         })?;
     parsed_config.anysnake2_toml_path = Some(abs_config_path);
+    parsed_config.source = config_file.into();
     Ok(parsed_config)
 }
 
@@ -286,7 +287,10 @@ fn switch_to_configured_version(
 
 fn collect_python_packages(
     parsed_config: &mut config::ConfigToml,
-) -> Result<(Vec<(String, String)>, Vec<(String, HashMap<String, String>)>)> {
+) -> Result<(
+    Vec<(String, String)>,
+    Vec<(String, HashMap<String, String>)>,
+)> {
     Ok(match &mut parsed_config.python {
         Some(python) => {
             let mut requirement_packages: Vec<(String, String)> = Vec::new();
@@ -394,8 +398,13 @@ fn inner_main() -> Result<()> {
     lookup_clones(&mut parsed_config)?;
     perform_clones(&parsed_config)?;
 
-    let (python_packages, python_build_packages) = collect_python_packages(&mut parsed_config)?;
-    trace!("python packages: {:?} {:?}", python_packages, python_build_packages);
+    let (python_packages, mut python_build_packages) = collect_python_packages(&mut parsed_config)?;
+    trace!(
+        "python packages: {:?} {:?}",
+        python_packages,
+        python_build_packages
+    );
+    apply_trust_on_first_use(&parsed_config, &mut python_build_packages)?;
     let use_generated_file_instead = parsed_config.anysnake2.do_not_modify_flake.unwrap_or(false);
 
     let flake_changed = flake_writer::write_flake(
@@ -1375,3 +1384,81 @@ fn write_develop_python_path(
     )?;
     Ok(())
 }
+
+fn apply_trust_on_first_use(
+    config: &config::ConfigToml,
+    python_build_packages: &mut Vec<(String, HashMap<String, String>)>,
+) -> Result<()> {
+    if !python_build_packages.is_empty() {
+        use toml_edit::{value, Document};
+        let toml = std::fs::read_to_string(&config.source).expect("Could not reread config file");
+        let mut doc = toml.parse::<Document>().expect("invalid doc");
+        let mut write = false;
+
+        for (k, spec) in python_build_packages.iter_mut() {
+            let method = spec
+                .get("method")
+                .expect("missing method - should have been caught earlier");
+            if method == "fetchFromGitHub" {
+                write = true;
+                println!("Using Trust-On-First-Use for python package {}, updating your anysnake2.toml", k);
+
+                let hash = prefetch_github_hash(
+                    spec.get("owner").expect("missing owner"),
+                    spec.get("repo").expect("missing repo"),
+                    spec.get("rev").expect("missing rev"),
+                )?;
+                println!("hash is {}", hash);
+                let key = k.to_owned();
+                doc["python"]["packages"][key]["hash"] = value(&hash);
+                spec.insert("hash".to_string(), hash.to_owned());
+            }
+        }
+        if write {
+            let out_toml = doc.to_string();
+            std::fs::write("anysnake2.toml", out_toml).expect("failed to rewrite config file");
+        }
+    }
+    Ok(())
+}
+
+fn prefetch_github_hash(owner: &str, repo: &str, git_hash: &str) -> Result<String> {
+    let url = format!(
+        "https://github.com/{owner}/{repo}/archive/{git_hash}.tar.gz",
+        owner = owner,
+        repo = repo,
+        git_hash = git_hash
+    );
+
+    let old_format = Command::new("nix-prefetch-url")
+        .args(&[&url, "--type", "sha256", "--unpack"])
+        .output()
+        .context(format!("Failed to nix-prefetch {url}", url = url))?
+        .stdout;
+    let old_format = std::str::from_utf8(&old_format)
+        .context("nix-prefetch result was no utf8")?
+        .trim();
+    let new_format = convert_hash_to_subresource_format(old_format)?;
+    println!("before convert: {}, after: {}", &old_format, &new_format);
+    Ok(new_format)
+}
+
+fn convert_hash_to_subresource_format(hash: &str) -> Result<String> {
+    let res = Command::new("nix")
+        .args(&["hash", "to-sri", "--type", "sha256", hash])
+        .output()
+        .context(format!(
+            "Failed to nix hash to-sri --type sha256 '{hash}'",
+            hash = hash
+        ))?
+        .stdout;
+    let res = std::str::from_utf8(&res)
+        .context("nix hash output was not utf8")?
+        .trim()
+        .to_owned();
+    if res.is_empty() {
+        Err(anyhow!("nix hash to-sri returned empty result"))
+    } else {
+        Ok(res)
+    }
+}