vendorGitDeps: perform basic URL decoding of attributes

ipetkov · Dec 16, 2024 · 763a1be · 763a1be
1 parent 82477bb
commit 763a1be
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * `crateNameFromCargoToml` will ignore store contexts when parsing a Cargo.toml
   file (avoiding errors like `the string ... is not allowed to refer to a store
   path`).
+* `vendorGitDeps` will perform a basic URL-decoding of git dependency entries in
+  the `Cargo.lock` file since lockfiles now encode special characters starting
+  at version 4
 
 ### Meta
 * Dropped support for publishing releases to https://flakestry.dev/

diff --git a/checks/default.nix b/checks/default.nix
@@ -535,9 +535,10 @@ in
   simpleGitWithHashes = myLib.buildPackage {
     src = myLib.cleanCargoSource ./simple-git;
     outputHashes = {
-      "git+https://github.com/BurntSushi/byteorder.git#18f32ca3a41c9823138e782752bc439e99ef7ec8" = "sha256-B98zLO/PICVR6QL9dq9Z8LMKnko3kdiYQgj5L0F+gOk=";
+      "git+https://github.com/BurntSushi/byteorder.git#5a82625fae462e8ba64cec8146b24a372b4d75c6" = "sha256-wDzXLhmBNcqrIrH/k7gcu+k/52XqidbpfrlAEFnY47c=";
       "git+https://github.com/dtolnay/rustversion.git?rev=2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb#2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb" = "sha256-deS6eoNuWPZ1V3XO9UzR07vLHZjT9arAYL0xEJCoU6E=";
       "git+https://github.com/dtolnay/unicode-ident.git?rev=a8736e7e62be959d87970d2d137a098ba533d78b#a8736e7e62be959d87970d2d137a098ba533d78b" = "sha256-++OSSdXBaHKeJnC8LOq/ouL+UAJMasDVsBzFClLnjaU=";
+      "git+https://github.com/ipetkov/crane-test-repo?branch=something/or/other#be0d3c039d260457a55c26e229ad9aa30242c2cf" = "sha256-X6Unf7eirBm6Lat99nROpPd9EUQUL0ru++zDkubj57I=";
       "git+https://github.com/rust-lang/libc.git?branch=main#a0f5b4b21391252fe38b2df9310dc65e37b07d9f" = "sha256-UwNxrPk6jrmtXeYef+RYYNfpNSlHQllANs/U4bmxlok=";
       "git+https://github.com/seanmonstar/num_cpus.git?tag=v1.13.1#5f1b03332000b4c4274b5bd35fac516049ff1c6b" = "sha256-mNMxS/WXjNokO9mFXQSwyuIpIp/n94EQ9Ni0Bl40es8=";
     };

diff --git a/checks/simple-git/Cargo.lock b/checks/simple-git/Cargo.lock
diff --git a/checks/simple-git/Cargo.toml b/checks/simple-git/Cargo.toml
@@ -7,6 +7,11 @@ edition = "2021"
 version = "*"
 git = "https://github.com/BurntSushi/byteorder.git"
 
+[dependencies.crane-test-repo]
+package = "print"
+git = "https://github.com/ipetkov/crane-test-repo"
+branch = "something/or/other"
+
 [dependencies.libc]
 version = "*"
 git = "https://github.com/rust-lang/libc.git"

diff --git a/checks/simple-git/src/main.rs b/checks/simple-git/src/main.rs
@@ -7,4 +7,5 @@ fn main() {
     println!("{:?}", std::any::TypeId::of::<byteorder::LittleEndian>());
     println!("{:?}", std::any::TypeId::of::<libc::c_int>());
     println!("{}: {}", CHANNEL, num_cpus::get());
+    crane_test_repo::print();
 }
diff --git a/lib/default.nix b/lib/default.nix
@@ -45,6 +45,7 @@ let
       internalCrateNameForCleanSource = callPackage ./internalCrateNameForCleanSource.nix {
         inherit internalCrateNameFromCargoToml;
       };
+      internalPercentDecode = callPackage ./internalPercentDecode.nix { };
     in
     {
       appendCrateRegistries = input: self.overrideScope (_final: prev: {
@@ -134,7 +135,9 @@ let
       vendorCargoDeps = callPackage ./vendorCargoDeps.nix { };
       vendorMultipleCargoDeps = callPackage ./vendorMultipleCargoDeps.nix { };
       vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { };
-      vendorGitDeps = callPackage ./vendorGitDeps.nix { };
+      vendorGitDeps = callPackage ./vendorGitDeps.nix {
+        inherit internalPercentDecode;
+      };
       writeTOML = callPackage ./writeTOML.nix { };
     };
 

diff --git a/lib/internalPercentDecode.nix b/lib/internalPercentDecode.nix
@@ -0,0 +1,42 @@
+{
+}:
+
+let
+  codes = {
+    "%09" = "\t";
+    "%0A" = "\n";
+    "%0D" = "\r";
+    "%20" = " ";
+    "%21" = "!";
+    "%22" = "\"";
+    "%23" = "#";
+    "%24" = "$";
+    "%25" = "%";
+    "%26" = "&";
+    "%27" = "'";
+    "%28" = "(";
+    "%29" = ")";
+    "%2A" = "*";
+    "%2B" = "+";
+    "%2C" = ",";
+    "%2F" = "/";
+    "%3A" = ":";
+    "%3B" = ";";
+    "%3C" = "<";
+    "%3D" = "=";
+    "%3E" = ">";
+    "%3F" = "?";
+    "%40" = "@";
+    "%5B" = "[";
+    "%5C" = "\\";
+    "%5D" = "]";
+    "%5E" = "^";
+    "%60" = "`";
+    "%7B" = "{";
+    "%7C" = "|";
+    "%7D" = "}";
+  };
+in
+builtins.replaceStrings
+  (builtins.attrNames codes)
+  (builtins.attrValues codes)
diff --git a/lib/vendorGitDeps.nix b/lib/vendorGitDeps.nix
@@ -1,4 +1,5 @@
 { downloadCargoPackageFromGit
+, internalPercentDecode
 , lib
 , pkgsBuildBuild
 }:
@@ -77,9 +78,13 @@ let
     };
 
   # Local crates will show up in the lock file with no checksum/source
-  lockedPackagesFromGit = filter
+  # NB: cargo started url encoding source urls starting with version 4
+  # but we need to undo that as package fetching and cargo configs expect
+  # the unencoded URLs.
+  lockedPackagesFromGit = map (lib.mapAttrs (_: internalPercentDecode)) (filter
     (p: hasPrefix "git" (p.source or ""))
-    lockPackages;
+    lockPackages
+  );
   lockedGitGroups = groupBy (p: p.id) (map
     (p: (parseGitUrl p) // { package = p; })
     lockedPackagesFromGit