diff --git a/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05 b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05 new file mode 120000 index 0000000..05692dc --- /dev/null +++ b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05 @@ -0,0 +1 @@ +/nix/store/wzx1ba5hqqfa23vfrvqmfmkpj25p37mr-source \ No newline at end of file diff --git a/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#dtach b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#dtach new file mode 120000 index 0000000..04c699d --- /dev/null +++ b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#dtach @@ -0,0 +1 @@ +/nix/store/v7qy11nyx968dgw93c6c03l0m7bvi4cl-dtach-0.9 \ No newline at end of file diff --git a/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt new file mode 120000 index 0000000..a2a0f06 --- /dev/null +++ b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt @@ -0,0 +1 @@ +/nix/store/yw7xxl0kgha3f9gk4yn9sf0k9v45l26q-nixfmt-0.6.0-bin \ No newline at end of file diff --git a/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#singularity b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#singularity new file mode 120000 index 0000000..64f48c4 --- /dev/null +++ b/examples/flake_subdependency/after/.gcroots/github:NixOS_nixpkgs_24.05#singularity @@ -0,0 +1 @@ +/nix/store/cdvdij4yvz9hj9mhycaz56jfjizxqbg7-singularity-ce-4.1.3 \ No newline at end of file diff --git a/examples/flake_subdependency/after/.gitignore b/examples/flake_subdependency/after/.gitignore new file mode 100644 index 0000000..1c7574c --- /dev/null +++ b/examples/flake_subdependency/after/.gitignore @@ -0,0 +1,5 @@ +result + run_scripts/ + .*.json + .gc_roots + \ No newline at end of file diff --git a/examples/flake_subdependency/after/flake.lock b/examples/flake_subdependency/after/flake.lock new file mode 100644 index 0000000..ac8423b --- /dev/null +++ b/examples/flake_subdependency/after/flake.lock @@ -0,0 +1,213 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1729742964, + "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "e04df33f62cdcf93d73e9a04142464753a16db67", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1730157240, + "narHash": "sha256-P8wF4ag6Srmpb/gwskYpnIsnspbjZlRvu47iN527ABQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "75e28c029ef2605f9841e0baa335d70065fe7ae2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "test": "test" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "test": { + "inputs": { + "flake-utils": "flake-utils_2", + "nix-github-actions": "nix-github-actions", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1731205797, + "narHash": "sha256-F7N1mxH1VrkVNHR3JGNMRvp9+98KYO4b832KS8Gl2xI=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "f554d27c1544d9c56e5f1f8e2b8aff399803674e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "f554d27c1544d9c56e5f1f8e2b8aff399803674e", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1730120726, + "narHash": "sha256-LqHYIxMrl/1p3/kvm2ir925tZ8DkI0KA10djk8wecSk=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "9ef337e492a5555d8e17a51c911ff1f02635be15", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/flake_subdependency/after/flake.nix b/examples/flake_subdependency/after/flake.nix new file mode 100644 index 0000000..0345fcd --- /dev/null +++ b/examples/flake_subdependency/after/flake.nix @@ -0,0 +1,73 @@ +{ + description = "Anysnake2 generated flake"; + inputs = rec { + + flake-utils = { + url = + "github:numtide/flake-utils/b1d9ab70662946ef0850d488da1c9019f3a9752a"; + + }; + nixpkgs = { + url = "github:NixOS/nixpkgs/24.05"; + + }; + rust-overlay = { + url = + "github:oxalica/rust-overlay/0be641045af6d8666c11c2c40e45ffc9667839b5"; + inputs.nixpkgs.follows = "nixpkgs"; + + }; + test = { + url = + "github:nix-community/poetry2nix/f554d27c1544d9c56e5f1f8e2b8aff399803674e"; + + }; + }; + + outputs = { self, flake-utils, nixpkgs, rust-overlay, test }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system overlays; + config = { allowUnfree = false; }; + }; + R_tracked = null; + overlays = [ (import rust-overlay) ]; + rust = pkgs.rust-bin.stable."1.55.0".minimal.override { + extensions = [ "rustfmt" "clippy" ]; + }; + _args = with pkgs; { + name = "anysnake2_container"; + #later entries beat earlier entries in terms of /bin symlinks + script = '' + ${coreutils} + ${bashInteractive_5} + ${bash} + ${cacert} + ${fish} + ${rust} + ${stdenv.cc} + ''; + }; + helpers = import ./functions.nix { inherit pkgs; }; + in rec { + defaultPackage = (helpers.buildSymlinkImage _args).derivation; + oci_image = helpers.buildOCIimage _args; + devShell = pkgs.stdenv.mkDerivation { + name = "anysnake2-devshell"; + shellHook = '' + export PATH=${defaultPackage}/rootfs/bin:$PATH; + if test -f "develop_python_path.bash"; then + source "develop_python_path.bash" + fi + '' + (if R_tracked != null then '' + export R_LIBS_SITE=${R_tracked}/lib/R/library/ + '' else + ""); + nativeBuildInputs = with pkgs; + [ + #%DEVSHELL_INPUTS% + ]; + }; + }); +} diff --git a/examples/flake_subdependency/after/functions.nix b/examples/flake_subdependency/after/functions.nix new file mode 100644 index 0000000..ba667e4 --- /dev/null +++ b/examples/flake_subdependency/after/functions.nix @@ -0,0 +1,160 @@ +{pkgs}: rec { + buildSymlinkImage = { + name, + script, + }: let + in rec { + script_file = pkgs.writeScript "reqs.sh" script; + derivation = pkgs.runCommand "${name}-2" {} '' + set -o pipefail + shopt -s nullglob + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/share + cp ${script_file} $out/reqs.sh + + # so singularity fills in the outside users + mkdir -p $out/rootfs/etc + touch $out/rootfs/etc/passwd + touch $out/rootfs/etc/group + + mkdir -p $out/rootfs/{bin,etc,share,tmp,var/tmp} + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/lib/share + mkdir -p $out/rootfs/R_libs + + set -x + # the later entries shadow the earlier ones. + # symlink the direct dependencies ... + for path in $(tac ${script_file}); + do + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/bin $out/rootfs/bin/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/etc $out/rootfs/etc || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib $out/rootfs/usr/lib/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/share $out/rootfs/usr/share/ || true + if [ -d $path/lib/R/library/ ]; then + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib/R/library $out/rootfs/R_libs/ || true + fi + done + + ln -s $out/rootfs/bin $out/rootfs/usr/bin + + mkdir -p $out/rootfs/etc/profile.d + echo "export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + echo "export SSL_CERT_DIR=/etc/ssl/certs" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + #echo "export PATH=/python_env/bin:/bin:/usr/bin/" >>$out/rootfs/etc/bashrc + #echo "export PYTHONPATH=$PYTHONPATH:/python_env/lib/python%PYTHON_MAJOR_DOT_MINOR%/site-packages" >>$out/rootfs/etc/bashrc + + + ''; + }; + + # can't use pkgs.ociTools.buildContainerImage + # because it a) does not work from a rootfs + # and b) doesn't actually build an image, just a runtime bundle + buildOCIimage = { + name, + script, + }: let + symlink_image = buildSymlinkImage { + inherit name script; + }; + umoci = pkgs.umoci; + in + pkgs.runCommand "${name}.oci" {} '' + set -o pipefail + shopt -s nullglob + ${umoci}/bin/umoci init --layout "${name}" + ${umoci}/bin/umoci new --image "${name}:latest" + mkdir tmp-oci-unpack + + mkdir fakeroot/etc -p + touch fakeroot/etc/resolv.conf + + # umoci tries to read /etc/resolv.conf, so let's give it one.. + ${pkgs.bubblewrap}/bin/bwrap \ + --proc /proc \ + --dev /dev \ + --bind /build /build \ + --ro-bind /build/fakeroot/etc /etc \ + --ro-bind /nix /nix \ + ${umoci}/bin/umoci unpack --image "${name}:latest" tmp-oci-unpack --rootless + + echo "rsyncing symlink forest" + ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ tmp-oci-unpack/rootfs/ + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + echo "rsyncing necessary nix paths" + ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + pkgs.writeClosure [symlink_image.script_file] + } / tmp-oci-unpack/rootfs/ + + + # check that we have a rootfs/bin/sh + if [ ! -e tmp-oci-unpack/rootfs/bin/sh ]; then + echo "No rootfs/bin/sh found" + exit 1 + fi + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + ${umoci}/bin/umoci repack --image "${name}:latest" tmp-oci-unpack + ${umoci}/bin/umoci config --image "${name}:latest" + # strip the first component + cd "${name}" && tar cf $out . + + + ''; + + # this never quite worked and since like 22.05, singularity image building on nixos seems broken + # buildSingularityImage = { + # name, + # script, + # }: let + # symlink_image = buildSymlinkImage { + # inherit name script; + # }; + # in + # pkgs.runCommand name {} '' + # set -o pipefail + # shopt -s nullglob + # mkdir -p $out/rootfs/ + + # ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ $out/rootfs/ + # chmod +w $out/rootfs -R # because we don't have write on the directories + # #make sure we got everything from the nix store, right? + # ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + # pkgs.writeClosure [symlink_image.script_file] + # } / $out/rootfs/ + + # rm $out/rootfs/${symlink_image.script_file} + # chmod 755 $out/rootfs + + # # # singularity tries to read resolv.conf, hosts and user definitions + # # # when converting the container + # # # so let's fake them + # mkdir $out/etc + # mkdir $out/build + # touch $out/etc/resolv.conf + # touch $out/etc/hosts + # echo "nixbld:x:1000:2000:imtseq:/home/installer:/bin/bash\n" >$out/etc/passwd + # echo "xxx:x: 2000:\n" >$out/etc/group + # echo ${pkgs.singularity}/bin/singularity + # ${pkgs.coreutils}/bin/whoami + + # # # also consider NIX_REDIRECT and libredirect for this + # # the bash binding is needed for singularity to find 'sh' + # ${pkgs.bubblewrap}/bin/bwrap \ + # --proc /proc \ + # --dev /dev \ + # --bind $out/ $out/ \ + # --bind $out/build /build \ + # --ro-bind $out/etc /etc \ + # --ro-bind /nix /nix \ + # --ro-bind "${pkgs.bash}/bin" /usr/bin \ + # ${pkgs.singularity}/bin/singularity build /build/${name}.sif $out/rootfs -v + # mv $out/build/*.sif $out/ + # rm -rf $out/build + # rm -rf $out/etc + # # chmod +w $out/rootfs -R # because we don't have write on the directories + # # rm -rf $out/rootfs + # ''; +} diff --git a/examples/flake_subdependency/after/run_scripts/run/outer_run.sh b/examples/flake_subdependency/after/run_scripts/run/outer_run.sh new file mode 100644 index 0000000..292d61d --- /dev/null +++ b/examples/flake_subdependency/after/run_scripts/run/outer_run.sh @@ -0,0 +1,2 @@ +#/bin/bash +bash -i /anysnake2/run.sh diff --git a/examples/flake_subdependency/after/run_scripts/run/post_run.sh b/examples/flake_subdependency/after/run_scripts/run/post_run.sh new file mode 100644 index 0000000..e69de29 diff --git a/examples/flake_subdependency/after/run_scripts/run/run.sh b/examples/flake_subdependency/after/run_scripts/run/run.sh new file mode 100644 index 0000000..20b92ab --- /dev/null +++ b/examples/flake_subdependency/after/run_scripts/run/run.sh @@ -0,0 +1 @@ +shaeu \ No newline at end of file diff --git a/examples/flake_subdependency/after/run_scripts/run/singularity.bash b/examples/flake_subdependency/after/run_scripts/run/singularity.bash new file mode 100644 index 0000000..b2dfd93 --- /dev/null +++ b/examples/flake_subdependency/after/run_scripts/run/singularity.bash @@ -0,0 +1,24 @@ +nix shell \ + github:NixOS/nixpkgs/24.05#dtach \ + -c \ + dtach \ + -c \ + .anysnake2_flake/dtach/run_2024-11-18_15:04:24 \ + nix \ + shell \ + github:NixOS/nixpkgs/24.05#singularity \ + -c \ + singularity exec \ + --userns \ + --cleanenv \ + --home /home/finkernagel \ + --bind /nix/store:/nix/store:ro \ + --bind .anysnake2_flake/run_scripts/run/run.sh:/anysnake2/run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/post_run.sh:/anysnake2/post_run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/outer_run.sh:/anysnake2/outer_run.sh:ro \ + --bind /home/finkernagel/upstream/anysnake2/examples/flake_subdependency:/project:rw \ + --env ANYSNAKE2=1 \ + --env PATH=/bin \ + .anysnake2_flake/result/rootfs \ + /bin/bash \ + /anysnake2/outer_run.sh \ diff --git a/examples/flake_subdependency/anysnake2.toml b/examples/flake_subdependency/anysnake2.toml new file mode 100644 index 0000000..f5be799 --- /dev/null +++ b/examples/flake_subdependency/anysnake2.toml @@ -0,0 +1,60 @@ +# test case for test_flake_change_updates_dependant_flakes +# package settings +[anysnake2] + rev = "dev" # pre 2.0 - 2.0+ uses url2 + url = "github:TyberiusPrime/anysnake2_release_flakes" # pre 2.0 - 2.0+ uses url2 + url2 = "dev" + + +[nixpkgs] +# the nixpkgs used inside the container + packages = [ + "bash", + "fish"] + url = "github:NixOS/nixpkgs/master/24.05" + + +[rust] + url = "github:oxalica/rust-overlay/master/0be641045af6d8666c11c2c40e45ffc9667839b5" + version="1.55.0" + + +[flakes.test] + packages = [] + url = "github:nix-community/poetry2nix/master/8810f7d31d4d8372f764d567ea140270745fe173" + + +[container.env] + ANYSNAKE2="1" + + +# container settings +[container.volumes_rw] + "." = "/project" # map the current folder to /project + + +[dev_shell] + inputs = [] + shell = "bash" + + +[outside_nixpkgs] +# the nixpkgs used to run singularity and nixfmt + url = "github:NixOS/nixpkgs/master/24.05" + + +[ancient_poetry] + url = "git+https://codeberg.org/TyberiusPrime/ancient-poetry.git?ref=main&rev=54a06abec3273f42f9d86a36f184dbb3089cd9c9" + + +[poetry2nix] + url = "github:nix-community/poetry2nix/master/8810f7d31d4d8372f764d567ea140270745fe174" + + +[flake-util] + url = "github:numtide/flake-utils/main/b1d9ab70662946ef0850d488da1c9019f3a9752a" + + +[devshell] + inputs = [] + shell = "bash" diff --git a/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05 b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05 new file mode 120000 index 0000000..05692dc --- /dev/null +++ b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05 @@ -0,0 +1 @@ +/nix/store/wzx1ba5hqqfa23vfrvqmfmkpj25p37mr-source \ No newline at end of file diff --git a/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#dtach b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#dtach new file mode 120000 index 0000000..04c699d --- /dev/null +++ b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#dtach @@ -0,0 +1 @@ +/nix/store/v7qy11nyx968dgw93c6c03l0m7bvi4cl-dtach-0.9 \ No newline at end of file diff --git a/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt new file mode 120000 index 0000000..a2a0f06 --- /dev/null +++ b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt @@ -0,0 +1 @@ +/nix/store/yw7xxl0kgha3f9gk4yn9sf0k9v45l26q-nixfmt-0.6.0-bin \ No newline at end of file diff --git a/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#singularity b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#singularity new file mode 120000 index 0000000..64f48c4 --- /dev/null +++ b/examples/flake_subdependency/before/.gcroots/github:NixOS_nixpkgs_24.05#singularity @@ -0,0 +1 @@ +/nix/store/cdvdij4yvz9hj9mhycaz56jfjizxqbg7-singularity-ce-4.1.3 \ No newline at end of file diff --git a/examples/flake_subdependency/before/.gitignore b/examples/flake_subdependency/before/.gitignore new file mode 100644 index 0000000..1c7574c --- /dev/null +++ b/examples/flake_subdependency/before/.gitignore @@ -0,0 +1,5 @@ +result + run_scripts/ + .*.json + .gc_roots + \ No newline at end of file diff --git a/examples/flake_subdependency/before/flake.lock b/examples/flake_subdependency/before/flake.lock new file mode 100644 index 0000000..65e1a93 --- /dev/null +++ b/examples/flake_subdependency/before/flake.lock @@ -0,0 +1,213 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698974481, + "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1699229614, + "narHash": "sha256-0n8d9Nx+J8v2o9Oog5MPPtM2YAq+gDeirQpj9l02YuQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ae52f963b8f098befb9914cc9220c29c93eb845f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "master", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "test": "test" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "test": { + "inputs": { + "flake-utils": "flake-utils_2", + "nix-github-actions": "nix-github-actions", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1699231189, + "narHash": "sha256-sW+/iiWdew5mftukqVm7AXPimyAWd6dMOfDkqnyBIec=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "8810f7d31d4d8372f764d567ea140270745fe173", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "8810f7d31d4d8372f764d567ea140270745fe173", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698438538, + "narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/flake_subdependency/before/flake.nix b/examples/flake_subdependency/before/flake.nix new file mode 100644 index 0000000..2281750 --- /dev/null +++ b/examples/flake_subdependency/before/flake.nix @@ -0,0 +1,73 @@ +{ + description = "Anysnake2 generated flake"; + inputs = rec { + + flake-utils = { + url = + "github:numtide/flake-utils/b1d9ab70662946ef0850d488da1c9019f3a9752a"; + + }; + nixpkgs = { + url = "github:NixOS/nixpkgs/24.05"; + + }; + rust-overlay = { + url = + "github:oxalica/rust-overlay/0be641045af6d8666c11c2c40e45ffc9667839b5"; + inputs.nixpkgs.follows = "nixpkgs"; + + }; + test = { + url = + "github:nix-community/poetry2nix/8810f7d31d4d8372f764d567ea140270745fe173"; + + }; + }; + + outputs = { self, flake-utils, nixpkgs, rust-overlay, test }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system overlays; + config = { allowUnfree = false; }; + }; + R_tracked = null; + overlays = [ (import rust-overlay) ]; + rust = pkgs.rust-bin.stable."1.55.0".minimal.override { + extensions = [ "rustfmt" "clippy" ]; + }; + _args = with pkgs; { + name = "anysnake2_container"; + #later entries beat earlier entries in terms of /bin symlinks + script = '' + ${coreutils} + ${bashInteractive_5} + ${bash} + ${cacert} + ${fish} + ${rust} + ${stdenv.cc} + ''; + }; + helpers = import ./functions.nix { inherit pkgs; }; + in rec { + defaultPackage = (helpers.buildSymlinkImage _args).derivation; + oci_image = helpers.buildOCIimage _args; + devShell = pkgs.stdenv.mkDerivation { + name = "anysnake2-devshell"; + shellHook = '' + export PATH=${defaultPackage}/rootfs/bin:$PATH; + if test -f "develop_python_path.bash"; then + source "develop_python_path.bash" + fi + '' + (if R_tracked != null then '' + export R_LIBS_SITE=${R_tracked}/lib/R/library/ + '' else + ""); + nativeBuildInputs = with pkgs; + [ + #%DEVSHELL_INPUTS% + ]; + }; + }); +} diff --git a/examples/flake_subdependency/before/functions.nix b/examples/flake_subdependency/before/functions.nix new file mode 100644 index 0000000..ba667e4 --- /dev/null +++ b/examples/flake_subdependency/before/functions.nix @@ -0,0 +1,160 @@ +{pkgs}: rec { + buildSymlinkImage = { + name, + script, + }: let + in rec { + script_file = pkgs.writeScript "reqs.sh" script; + derivation = pkgs.runCommand "${name}-2" {} '' + set -o pipefail + shopt -s nullglob + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/share + cp ${script_file} $out/reqs.sh + + # so singularity fills in the outside users + mkdir -p $out/rootfs/etc + touch $out/rootfs/etc/passwd + touch $out/rootfs/etc/group + + mkdir -p $out/rootfs/{bin,etc,share,tmp,var/tmp} + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/lib/share + mkdir -p $out/rootfs/R_libs + + set -x + # the later entries shadow the earlier ones. + # symlink the direct dependencies ... + for path in $(tac ${script_file}); + do + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/bin $out/rootfs/bin/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/etc $out/rootfs/etc || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib $out/rootfs/usr/lib/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/share $out/rootfs/usr/share/ || true + if [ -d $path/lib/R/library/ ]; then + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib/R/library $out/rootfs/R_libs/ || true + fi + done + + ln -s $out/rootfs/bin $out/rootfs/usr/bin + + mkdir -p $out/rootfs/etc/profile.d + echo "export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + echo "export SSL_CERT_DIR=/etc/ssl/certs" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + #echo "export PATH=/python_env/bin:/bin:/usr/bin/" >>$out/rootfs/etc/bashrc + #echo "export PYTHONPATH=$PYTHONPATH:/python_env/lib/python%PYTHON_MAJOR_DOT_MINOR%/site-packages" >>$out/rootfs/etc/bashrc + + + ''; + }; + + # can't use pkgs.ociTools.buildContainerImage + # because it a) does not work from a rootfs + # and b) doesn't actually build an image, just a runtime bundle + buildOCIimage = { + name, + script, + }: let + symlink_image = buildSymlinkImage { + inherit name script; + }; + umoci = pkgs.umoci; + in + pkgs.runCommand "${name}.oci" {} '' + set -o pipefail + shopt -s nullglob + ${umoci}/bin/umoci init --layout "${name}" + ${umoci}/bin/umoci new --image "${name}:latest" + mkdir tmp-oci-unpack + + mkdir fakeroot/etc -p + touch fakeroot/etc/resolv.conf + + # umoci tries to read /etc/resolv.conf, so let's give it one.. + ${pkgs.bubblewrap}/bin/bwrap \ + --proc /proc \ + --dev /dev \ + --bind /build /build \ + --ro-bind /build/fakeroot/etc /etc \ + --ro-bind /nix /nix \ + ${umoci}/bin/umoci unpack --image "${name}:latest" tmp-oci-unpack --rootless + + echo "rsyncing symlink forest" + ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ tmp-oci-unpack/rootfs/ + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + echo "rsyncing necessary nix paths" + ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + pkgs.writeClosure [symlink_image.script_file] + } / tmp-oci-unpack/rootfs/ + + + # check that we have a rootfs/bin/sh + if [ ! -e tmp-oci-unpack/rootfs/bin/sh ]; then + echo "No rootfs/bin/sh found" + exit 1 + fi + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + ${umoci}/bin/umoci repack --image "${name}:latest" tmp-oci-unpack + ${umoci}/bin/umoci config --image "${name}:latest" + # strip the first component + cd "${name}" && tar cf $out . + + + ''; + + # this never quite worked and since like 22.05, singularity image building on nixos seems broken + # buildSingularityImage = { + # name, + # script, + # }: let + # symlink_image = buildSymlinkImage { + # inherit name script; + # }; + # in + # pkgs.runCommand name {} '' + # set -o pipefail + # shopt -s nullglob + # mkdir -p $out/rootfs/ + + # ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ $out/rootfs/ + # chmod +w $out/rootfs -R # because we don't have write on the directories + # #make sure we got everything from the nix store, right? + # ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + # pkgs.writeClosure [symlink_image.script_file] + # } / $out/rootfs/ + + # rm $out/rootfs/${symlink_image.script_file} + # chmod 755 $out/rootfs + + # # # singularity tries to read resolv.conf, hosts and user definitions + # # # when converting the container + # # # so let's fake them + # mkdir $out/etc + # mkdir $out/build + # touch $out/etc/resolv.conf + # touch $out/etc/hosts + # echo "nixbld:x:1000:2000:imtseq:/home/installer:/bin/bash\n" >$out/etc/passwd + # echo "xxx:x: 2000:\n" >$out/etc/group + # echo ${pkgs.singularity}/bin/singularity + # ${pkgs.coreutils}/bin/whoami + + # # # also consider NIX_REDIRECT and libredirect for this + # # the bash binding is needed for singularity to find 'sh' + # ${pkgs.bubblewrap}/bin/bwrap \ + # --proc /proc \ + # --dev /dev \ + # --bind $out/ $out/ \ + # --bind $out/build /build \ + # --ro-bind $out/etc /etc \ + # --ro-bind /nix /nix \ + # --ro-bind "${pkgs.bash}/bin" /usr/bin \ + # ${pkgs.singularity}/bin/singularity build /build/${name}.sif $out/rootfs -v + # mv $out/build/*.sif $out/ + # rm -rf $out/build + # rm -rf $out/etc + # # chmod +w $out/rootfs -R # because we don't have write on the directories + # # rm -rf $out/rootfs + # ''; +} diff --git a/examples/flake_subdependency/before/run_scripts/run/outer_run.sh b/examples/flake_subdependency/before/run_scripts/run/outer_run.sh new file mode 100644 index 0000000..292d61d --- /dev/null +++ b/examples/flake_subdependency/before/run_scripts/run/outer_run.sh @@ -0,0 +1,2 @@ +#/bin/bash +bash -i /anysnake2/run.sh diff --git a/examples/flake_subdependency/before/run_scripts/run/post_run.sh b/examples/flake_subdependency/before/run_scripts/run/post_run.sh new file mode 100644 index 0000000..e69de29 diff --git a/examples/flake_subdependency/before/run_scripts/run/run.sh b/examples/flake_subdependency/before/run_scripts/run/run.sh new file mode 100644 index 0000000..20b92ab --- /dev/null +++ b/examples/flake_subdependency/before/run_scripts/run/run.sh @@ -0,0 +1 @@ +shaeu \ No newline at end of file diff --git a/examples/flake_subdependency/before/run_scripts/run/singularity.bash b/examples/flake_subdependency/before/run_scripts/run/singularity.bash new file mode 100644 index 0000000..02b8e1e --- /dev/null +++ b/examples/flake_subdependency/before/run_scripts/run/singularity.bash @@ -0,0 +1,24 @@ +nix shell \ + github:NixOS/nixpkgs/24.05#dtach \ + -c \ + dtach \ + -c \ + .anysnake2_flake/dtach/run_2024-11-18_15:02:48 \ + nix \ + shell \ + github:NixOS/nixpkgs/24.05#singularity \ + -c \ + singularity exec \ + --userns \ + --cleanenv \ + --home /home/finkernagel \ + --bind /nix/store:/nix/store:ro \ + --bind .anysnake2_flake/run_scripts/run/run.sh:/anysnake2/run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/post_run.sh:/anysnake2/post_run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/outer_run.sh:/anysnake2/outer_run.sh:ro \ + --bind /home/finkernagel/upstream/anysnake2/examples/flake_subdependency:/project:rw \ + --env ANYSNAKE2=1 \ + --env PATH=/bin \ + .anysnake2_flake/result/rootfs \ + /bin/bash \ + /anysnake2/outer_run.sh \ diff --git a/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05 b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05 new file mode 120000 index 0000000..05692dc --- /dev/null +++ b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05 @@ -0,0 +1 @@ +/nix/store/wzx1ba5hqqfa23vfrvqmfmkpj25p37mr-source \ No newline at end of file diff --git a/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#dtach b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#dtach new file mode 120000 index 0000000..04c699d --- /dev/null +++ b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#dtach @@ -0,0 +1 @@ +/nix/store/v7qy11nyx968dgw93c6c03l0m7bvi4cl-dtach-0.9 \ No newline at end of file diff --git a/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt new file mode 120000 index 0000000..a2a0f06 --- /dev/null +++ b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#nixfmt @@ -0,0 +1 @@ +/nix/store/yw7xxl0kgha3f9gk4yn9sf0k9v45l26q-nixfmt-0.6.0-bin \ No newline at end of file diff --git a/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#singularity b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#singularity new file mode 120000 index 0000000..64f48c4 --- /dev/null +++ b/examples/flake_subdependency/updated/.gcroots/github:NixOS_nixpkgs_24.05#singularity @@ -0,0 +1 @@ +/nix/store/cdvdij4yvz9hj9mhycaz56jfjizxqbg7-singularity-ce-4.1.3 \ No newline at end of file diff --git a/examples/flake_subdependency/updated/.gitignore b/examples/flake_subdependency/updated/.gitignore new file mode 100644 index 0000000..1c7574c --- /dev/null +++ b/examples/flake_subdependency/updated/.gitignore @@ -0,0 +1,5 @@ +result + run_scripts/ + .*.json + .gc_roots + \ No newline at end of file diff --git a/examples/flake_subdependency/updated/flake.lock b/examples/flake_subdependency/updated/flake.lock new file mode 100644 index 0000000..e34cd68 --- /dev/null +++ b/examples/flake_subdependency/updated/flake.lock @@ -0,0 +1,213 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698974481, + "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1717179513, + "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1731919951, + "narHash": "sha256-vOM6ETpl1yu9KLi/icTmLJIPbbdJCdAVYUXZceO/Ce4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "04386ac325a813047fc314d4b4d838a5b1e3c7fe", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "test": "test" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "test": { + "inputs": { + "flake-utils": "flake-utils_2", + "nix-github-actions": "nix-github-actions", + "nixpkgs": "nixpkgs_2", + "systems": "systems_3", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1731205797, + "narHash": "sha256-F7N1mxH1VrkVNHR3JGNMRvp9+98KYO4b832KS8Gl2xI=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "f554d27c1544d9c56e5f1f8e2b8aff399803674e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "f554d27c1544d9c56e5f1f8e2b8aff399803674e", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "test", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698438538, + "narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/flake_subdependency/updated/flake.nix b/examples/flake_subdependency/updated/flake.nix new file mode 100644 index 0000000..0345fcd --- /dev/null +++ b/examples/flake_subdependency/updated/flake.nix @@ -0,0 +1,73 @@ +{ + description = "Anysnake2 generated flake"; + inputs = rec { + + flake-utils = { + url = + "github:numtide/flake-utils/b1d9ab70662946ef0850d488da1c9019f3a9752a"; + + }; + nixpkgs = { + url = "github:NixOS/nixpkgs/24.05"; + + }; + rust-overlay = { + url = + "github:oxalica/rust-overlay/0be641045af6d8666c11c2c40e45ffc9667839b5"; + inputs.nixpkgs.follows = "nixpkgs"; + + }; + test = { + url = + "github:nix-community/poetry2nix/f554d27c1544d9c56e5f1f8e2b8aff399803674e"; + + }; + }; + + outputs = { self, flake-utils, nixpkgs, rust-overlay, test }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system overlays; + config = { allowUnfree = false; }; + }; + R_tracked = null; + overlays = [ (import rust-overlay) ]; + rust = pkgs.rust-bin.stable."1.55.0".minimal.override { + extensions = [ "rustfmt" "clippy" ]; + }; + _args = with pkgs; { + name = "anysnake2_container"; + #later entries beat earlier entries in terms of /bin symlinks + script = '' + ${coreutils} + ${bashInteractive_5} + ${bash} + ${cacert} + ${fish} + ${rust} + ${stdenv.cc} + ''; + }; + helpers = import ./functions.nix { inherit pkgs; }; + in rec { + defaultPackage = (helpers.buildSymlinkImage _args).derivation; + oci_image = helpers.buildOCIimage _args; + devShell = pkgs.stdenv.mkDerivation { + name = "anysnake2-devshell"; + shellHook = '' + export PATH=${defaultPackage}/rootfs/bin:$PATH; + if test -f "develop_python_path.bash"; then + source "develop_python_path.bash" + fi + '' + (if R_tracked != null then '' + export R_LIBS_SITE=${R_tracked}/lib/R/library/ + '' else + ""); + nativeBuildInputs = with pkgs; + [ + #%DEVSHELL_INPUTS% + ]; + }; + }); +} diff --git a/examples/flake_subdependency/updated/functions.nix b/examples/flake_subdependency/updated/functions.nix new file mode 100644 index 0000000..ba667e4 --- /dev/null +++ b/examples/flake_subdependency/updated/functions.nix @@ -0,0 +1,160 @@ +{pkgs}: rec { + buildSymlinkImage = { + name, + script, + }: let + in rec { + script_file = pkgs.writeScript "reqs.sh" script; + derivation = pkgs.runCommand "${name}-2" {} '' + set -o pipefail + shopt -s nullglob + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/share + cp ${script_file} $out/reqs.sh + + # so singularity fills in the outside users + mkdir -p $out/rootfs/etc + touch $out/rootfs/etc/passwd + touch $out/rootfs/etc/group + + mkdir -p $out/rootfs/{bin,etc,share,tmp,var/tmp} + mkdir -p $out/rootfs/usr/lib + mkdir -p $out/rootfs/usr/lib/share + mkdir -p $out/rootfs/R_libs + + set -x + # the later entries shadow the earlier ones. + # symlink the direct dependencies ... + for path in $(tac ${script_file}); + do + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/bin $out/rootfs/bin/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/etc $out/rootfs/etc || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib $out/rootfs/usr/lib/ || true + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/share $out/rootfs/usr/share/ || true + if [ -d $path/lib/R/library/ ]; then + ${pkgs.xorg.lndir}/bin/lndir -ignorelinks $path/lib/R/library $out/rootfs/R_libs/ || true + fi + done + + ln -s $out/rootfs/bin $out/rootfs/usr/bin + + mkdir -p $out/rootfs/etc/profile.d + echo "export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + echo "export SSL_CERT_DIR=/etc/ssl/certs" >>$out/rootfs/etc/bashrc # singularity pulls that from the env otherwise apperantly + #echo "export PATH=/python_env/bin:/bin:/usr/bin/" >>$out/rootfs/etc/bashrc + #echo "export PYTHONPATH=$PYTHONPATH:/python_env/lib/python%PYTHON_MAJOR_DOT_MINOR%/site-packages" >>$out/rootfs/etc/bashrc + + + ''; + }; + + # can't use pkgs.ociTools.buildContainerImage + # because it a) does not work from a rootfs + # and b) doesn't actually build an image, just a runtime bundle + buildOCIimage = { + name, + script, + }: let + symlink_image = buildSymlinkImage { + inherit name script; + }; + umoci = pkgs.umoci; + in + pkgs.runCommand "${name}.oci" {} '' + set -o pipefail + shopt -s nullglob + ${umoci}/bin/umoci init --layout "${name}" + ${umoci}/bin/umoci new --image "${name}:latest" + mkdir tmp-oci-unpack + + mkdir fakeroot/etc -p + touch fakeroot/etc/resolv.conf + + # umoci tries to read /etc/resolv.conf, so let's give it one.. + ${pkgs.bubblewrap}/bin/bwrap \ + --proc /proc \ + --dev /dev \ + --bind /build /build \ + --ro-bind /build/fakeroot/etc /etc \ + --ro-bind /nix /nix \ + ${umoci}/bin/umoci unpack --image "${name}:latest" tmp-oci-unpack --rootless + + echo "rsyncing symlink forest" + ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ tmp-oci-unpack/rootfs/ + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + echo "rsyncing necessary nix paths" + ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + pkgs.writeClosure [symlink_image.script_file] + } / tmp-oci-unpack/rootfs/ + + + # check that we have a rootfs/bin/sh + if [ ! -e tmp-oci-unpack/rootfs/bin/sh ]; then + echo "No rootfs/bin/sh found" + exit 1 + fi + chmod +w tmp-oci-unpack/rootfs -R # because we don't have write on the directories + + ${umoci}/bin/umoci repack --image "${name}:latest" tmp-oci-unpack + ${umoci}/bin/umoci config --image "${name}:latest" + # strip the first component + cd "${name}" && tar cf $out . + + + ''; + + # this never quite worked and since like 22.05, singularity image building on nixos seems broken + # buildSingularityImage = { + # name, + # script, + # }: let + # symlink_image = buildSymlinkImage { + # inherit name script; + # }; + # in + # pkgs.runCommand name {} '' + # set -o pipefail + # shopt -s nullglob + # mkdir -p $out/rootfs/ + + # ${pkgs.rsync}/bin/rsync -arW ${symlink_image.derivation}/rootfs/ $out/rootfs/ + # chmod +w $out/rootfs -R # because we don't have write on the directories + # #make sure we got everything from the nix store, right? + # ${pkgs.rsync}/bin/rsync -arW --exclude=* --files-from=${ + # pkgs.writeClosure [symlink_image.script_file] + # } / $out/rootfs/ + + # rm $out/rootfs/${symlink_image.script_file} + # chmod 755 $out/rootfs + + # # # singularity tries to read resolv.conf, hosts and user definitions + # # # when converting the container + # # # so let's fake them + # mkdir $out/etc + # mkdir $out/build + # touch $out/etc/resolv.conf + # touch $out/etc/hosts + # echo "nixbld:x:1000:2000:imtseq:/home/installer:/bin/bash\n" >$out/etc/passwd + # echo "xxx:x: 2000:\n" >$out/etc/group + # echo ${pkgs.singularity}/bin/singularity + # ${pkgs.coreutils}/bin/whoami + + # # # also consider NIX_REDIRECT and libredirect for this + # # the bash binding is needed for singularity to find 'sh' + # ${pkgs.bubblewrap}/bin/bwrap \ + # --proc /proc \ + # --dev /dev \ + # --bind $out/ $out/ \ + # --bind $out/build /build \ + # --ro-bind $out/etc /etc \ + # --ro-bind /nix /nix \ + # --ro-bind "${pkgs.bash}/bin" /usr/bin \ + # ${pkgs.singularity}/bin/singularity build /build/${name}.sif $out/rootfs -v + # mv $out/build/*.sif $out/ + # rm -rf $out/build + # rm -rf $out/etc + # # chmod +w $out/rootfs -R # because we don't have write on the directories + # # rm -rf $out/rootfs + # ''; +} diff --git a/examples/flake_subdependency/updated/run_scripts/run/outer_run.sh b/examples/flake_subdependency/updated/run_scripts/run/outer_run.sh new file mode 100644 index 0000000..292d61d --- /dev/null +++ b/examples/flake_subdependency/updated/run_scripts/run/outer_run.sh @@ -0,0 +1,2 @@ +#/bin/bash +bash -i /anysnake2/run.sh diff --git a/examples/flake_subdependency/updated/run_scripts/run/post_run.sh b/examples/flake_subdependency/updated/run_scripts/run/post_run.sh new file mode 100644 index 0000000..e69de29 diff --git a/examples/flake_subdependency/updated/run_scripts/run/run.sh b/examples/flake_subdependency/updated/run_scripts/run/run.sh new file mode 100644 index 0000000..20b92ab --- /dev/null +++ b/examples/flake_subdependency/updated/run_scripts/run/run.sh @@ -0,0 +1 @@ +shaeu \ No newline at end of file diff --git a/examples/flake_subdependency/updated/run_scripts/run/singularity.bash b/examples/flake_subdependency/updated/run_scripts/run/singularity.bash new file mode 100644 index 0000000..3e9978e --- /dev/null +++ b/examples/flake_subdependency/updated/run_scripts/run/singularity.bash @@ -0,0 +1,24 @@ +nix shell \ + github:NixOS/nixpkgs/24.05#dtach \ + -c \ + dtach \ + -c \ + .anysnake2_flake/dtach/run_2024-11-18_15:03:22 \ + nix \ + shell \ + github:NixOS/nixpkgs/24.05#singularity \ + -c \ + singularity exec \ + --userns \ + --cleanenv \ + --home /home/finkernagel \ + --bind /nix/store:/nix/store:ro \ + --bind .anysnake2_flake/run_scripts/run/run.sh:/anysnake2/run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/post_run.sh:/anysnake2/post_run.sh:ro \ + --bind .anysnake2_flake/run_scripts/run/outer_run.sh:/anysnake2/outer_run.sh:ro \ + --bind /home/finkernagel/upstream/anysnake2/examples/flake_subdependency:/project:rw \ + --env ANYSNAKE2=1 \ + --env PATH=/bin \ + .anysnake2_flake/result/rootfs \ + /bin/bash \ + /anysnake2/outer_run.sh \ diff --git a/examples/full/anysnake2.toml b/examples/full/anysnake2.toml index 000a081..0f1d4dd 100644 --- a/examples/full/anysnake2.toml +++ b/examples/full/anysnake2.toml @@ -31,14 +31,12 @@ [nixpkgs] allow_unfree = true # set to true to allow unfree packages from nixpkgs - packages = [ # use https://search.nixos.org/packages to search + packages = [ "fish", - "netcat", - "varscan", # unfree example "gnumake", "gnused", - #"docker-client" - ] # optional + "netcat", + "varscan"] url = "github:NixOS/nixpkgs/master/24.05" diff --git a/src/config.rs b/src/config.rs index b18ca4d..6a84885 100644 --- a/src/config.rs +++ b/src/config.rs @@ -664,7 +664,7 @@ pub struct TofuFlake { pub url: TofuVCS, pub dir: Option, pub follows: Option>, - pub packages: Vec, + pub packages: Option>, } #[derive(Deserialize, Debug, Default)] diff --git a/src/flake_writer.rs b/src/flake_writer.rs index a6188be..6ad3f46 100644 --- a/src/flake_writer.rs +++ b/src/flake_writer.rs @@ -60,13 +60,18 @@ fn get_filenames(flake_dir: impl AsRef, use_generated_file_instead: bool) } } +pub struct WriteFlakeResult { + pub flake_nix_changed: bool, + pub python_lock_changed: bool +} + pub fn write_flake( flake_dir: impl AsRef, parsed_config: &mut config::TofuConfigToml, use_generated_file_instead: bool, // which is set if do_not_modify_flake is in effect. in_non_spec_but_cached_values: &HashMap, out_non_spec_but_cached_values: &mut HashMap, -) -> Result { +) -> Result { let template = std::include_str!("nix/flake_template.nix"); let flake_dir: &Path = flake_dir.as_ref(); @@ -200,20 +205,24 @@ pub fn write_flake( gitargs.push("poetry/poetry.lock"); } */ - let mut res = write_flake_contents( + let flake_nix_changed = write_flake_contents( &old_flake_contents, &flake_contents, use_generated_file_instead, &flake_filename, flake_dir, )?; - res |= python_locks_changed; run_git_add(&git_tracked_files, flake_dir)?; run_git_commit(flake_dir)?; //after nix 2.23 we will need to commit the flake, possibly. At //least if we wanted to reference it from another flake - Ok(res) + Ok({ + WriteFlakeResult { + flake_nix_changed, + python_lock_changed: python_locks_changed, + } + }) } /// format the list of input flakes for the inputs = {} section of a flake.nix @@ -927,13 +936,13 @@ fn add_flakes( flake.dir.clone(), &rev_follows[..], )); - if flake.packages.is_empty() { + if flake.packages.is_none() { nixpkgs_pkgs.insert(format!( "({}.defaultPackage.x86_64-linux or {}.packages.x86_64-linux.defaults)", name, name )); - } else { - for pkg in &flake.packages { + } else if let Some(pkgs) = &flake.packages{ + for pkg in pkgs { nixpkgs_pkgs.insert(format!("{name}.{pkg}")); } } diff --git a/src/main.rs b/src/main.rs index 51346ae..79ad12b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -366,7 +366,12 @@ fn inner_main() -> Result<()> { match sc.subcommand() { Some(("flake", _)) => { info!("Writing just flake/flake.nix"); - rebuild_flake(use_generated_file_instead, "flake", &flake_dir)?; + rebuild_flake( + use_generated_file_instead, + "flake", + &flake_dir, + flake_changed.flake_nix_changed, + )?; } Some(("oci", _)) => { info!("Building oci-image in flake/result"); @@ -374,11 +379,17 @@ fn inner_main() -> Result<()> { use_generated_file_instead, "oci_image.x86_64-linux", &flake_dir, + flake_changed.flake_nix_changed, )?; } Some(("rootfs", _)) => { info!("Building rootfs in flake/result"); - rebuild_flake(use_generated_file_instead, "", &flake_dir)?; + rebuild_flake( + use_generated_file_instead, + "", + &flake_dir, + flake_changed.flake_nix_changed, + )?; } _ => { info!("Please pass a subcommand as to what to build (use --help to list)"); @@ -404,9 +415,18 @@ fn inner_main() -> Result<()> { let build_unfinished_file = flake_dir.join(".build_unfinished"); // ie. the flake build failed // //early error exit if you try to run an non-existant command - if flake_changed || !build_output.exists() || build_unfinished_file.exists() { + if flake_changed.flake_nix_changed + || flake_changed.python_lock_changed + || !build_output.exists() + || build_unfinished_file.exists() + { info!("Rebuilding flake"); - rebuild_flake(use_generated_file_instead, "", &flake_dir)?; + rebuild_flake( + use_generated_file_instead, + "", + &flake_dir, + flake_changed.flake_nix_changed, + )?; } if let Some(python) = &tofued_config.python { @@ -923,10 +943,22 @@ fn rebuild_flake( use_generated_file_instead: bool, target: &str, flake_dir: impl AsRef, + flake_content_changed: bool, ) -> Result<()> { debug!("writing flake"); if !use_generated_file_instead { + if flake_content_changed { + let flake_lock_path = flake_dir.as_ref().join("flake.lock"); + if flake_lock_path.exists() { + fs::remove_file(&flake_lock_path)?; + } + Command::new("nix") + .args(["flake", "lock"]) + .current_dir(&flake_dir) + .status() + .context("(Re)-locking flake failed")?; + } run_without_ctrl_c(|| { Command::new("git") .args(["commit", "-m", "autocommit"]) diff --git a/src/tofu.rs b/src/tofu.rs index 47aaef5..1d0a1ce 100644 --- a/src/tofu.rs +++ b/src/tofu.rs @@ -514,7 +514,7 @@ impl Tofu> for Option, query: &str, replacement: &str) { + let raw = ex::fs::read_to_string(&path).unwrap(); + assert!(raw.contains(query)); + let out = raw.replace(query, replacement); + ex::fs::write(path, out).unwrap(); +} + +#[test] +fn test_flake_change_updates_dependant_flakes() { + let ((_code, _stdout, _stderr), td) = + run_test_tempdir("examples/flake_subdependency", &["run", "--", "bash" ,"--version"]); + let before = ex::fs::read_to_string(td.path().join(".anysnake2_flake/flake.lock")).unwrap(); + assert!(before.contains("8810f7d31d4d8372f764d567ea140270745fe173")); + replace_in_file( + &td.path().join("anysnake2.toml"), + "8810f7d31d4d8372f764d567ea140270745fe173", + "f554d27c1544d9c56e5f1f8e2b8aff399803674e", + ); + let updated_anysnake2_toml = ex::fs::read_to_string(td.path().join("anysnake2.toml")).unwrap(); + assert!(updated_anysnake2_toml.contains("f554d27c1544d9c56e5f1f8e2b8aff399803674e")); + run_test( + &td.path().to_string_lossy(), + &["run", "--", "bash" ,"--version"], + false, + ); + let updated = ex::fs::read_to_string(td.path().join(".anysnake2_flake/flake.lock")).unwrap(); + assert!(updated != before); + run_test( + &td.path().to_string_lossy(), + &["run", "--", "bash" ,"--version"], + true, + ); + let after = ex::fs::read_to_string(td.path().join(".anysnake2_flake/flake.lock")).unwrap(); + assert_eq!(after, updated); +}