Merge pull request #1033 from phaer/editables

editables: fix path ordering, remove relics...

docs/src/guides/pip.md

@@ -6,7 +6,7 @@ title: Build a python project with pip
 
     We recommend reading our [Getting Started](./getting-started.md) guide first if you have not done so yet!
 
-this guide we are going to take a look at two annotated examples using the [pip module](../reference/pip/index.md):
+In this guide we are going to take a look at two annotated examples using the [pip module](../reference/pip/index.md):
 
 - The first one builds [Pillow](https://python-pillow.org/) from upstream sources fetched from PyPi.
 - The second one builds a fictional python project living in the same repository as the nix sources
@@ -317,8 +317,10 @@ see where they are imported from:
 
 ```shell-session
 $ nix develop
-Some python dependencies of my-tool are installed in editable mode
-To disable editable mode for a package, remove the corresponding entry from the 'editables' field in the dream2nix configuration file.
+evaluating derivation 'git+file://[path_to_your_repo]#devShells.x86_64-linux.default'
+Some python dependencies of /Users/phaer/src/dream2nix/examples/packages/languages/python-local-development are installed in editable mode
+  my-tool
+    installed at: .
 $ ipython
 [...]
 In [1]: import my_tool

examples/packages/languages/python-local-development/default.nix

@@ -38,9 +38,9 @@ in {
 
   paths.lockFile = "lock.${config.deps.stdenv.system}.json";
   pip = {
-    # Setting editables.$pkg to an absolute path will link this path as an editable
-    # install to .dream2nix/editables in devShells. The root package is always installed
-    # as editable.
+    # Setting editables.$pkg to a relative or absolute path, as a string, will
+    # link this path as an editable install to .dream2nix/editables in
+    # devShells. The root package is always installed as editable.
     # editables.charset-normalizer = "/home/my-user/src/charset-normalizer";
 
     requirementsList =

modules/dream2nix/python-editables/editable.py

@@ -41,50 +41,42 @@ def make_editable(
     python_environment,
     bin_dir,
     site_dir,
-    editables_dir,
-    site_packages,
     name,
     path,
-    root_dir,
 ):
     normalized_name = name.replace("-", "_")
-    editable_dir = editables_dir / name
     full_path = path if path.is_absolute() else root_dir / path
-    if editable_dir.exists():
-        relative_editable_dir = editable_dir.relative_to(root_dir)
-        return
     if not full_path.exists():
         print(
             f"Error: The python dependency {name} of {root_name} is configured to be installed in editable mode, but the provided source location {full_path} does not exist.\n"
             f"Please provide a path to a local copy of the source code of {name}.",
             file=sys.stderr,
         )
         exit(1)
-    # link_editable_source(editable_dir, site_dir, normalized_name, full_path)
-    make_pth(site_dir, full_path, normalized_name)
-    editable_dist_info = make_dist_info(
-        site_dir, full_path, site_packages, normalized_name
-    )
-    make_entrypoints(python_environment, bin_dir, editable_dist_info)
-
-
-def make_pth(site_dir, editable_dir, normalized_name):
     # Check if source uses a "src"-layout or a flat-layout and
     # write the .pth file
-    if (editable_dir / "src").exists():
-        pth = editable_dir / "src"
+    if (full_path / "src").exists():
+        editable_dir = full_path / "src"
     else:
         # TODO this approach is risky as it puts everything inside
         # upstreams repo on $PYTHONPATH. Maybe we should try to
         # get packages from toplevel.txt first and if found,
         # create a dir with only them linked?
-        pth = editable_dir
+        editable_dir = full_path
+
+    make_pth(site_dir, editable_dir, normalized_name)
+    editable_dist_info = make_dist_info(site_dir, full_path)
+    make_entrypoints(python_environment, bin_dir, editable_dist_info)
+    return editable_dir
+
+
+def make_pth(site_dir, editable_dir, normalized_name):
     with open((site_dir / normalized_name).with_suffix(".pth"), "w") as f:
-        f.write(f"{pth}\n")
+        f.write(f"{editable_dir}\n")
 
 
 # create a packages .dist-info by calling its build backend
-def make_dist_info(site_dir, editable_dir, site_packages, normalized_name):
+def make_dist_info(site_dir, editable_dir):
     os.chdir(editable_dir)
     pyproject_file = editable_dir / "pyproject.toml"
     if pyproject_file.exists():
@@ -182,7 +174,7 @@ def export_environment_vars(python_environment, bin_dir, site_dir, site_packages
     )
 
 
-def pretty_print_editables(editables, root_dir, root_name):
+def pretty_print_editables(editables, root_name):
     if os.environ.get("D2N_QUIET"):
         return
     C = Colors
@@ -213,12 +205,11 @@ def pretty_print_editables(editables, root_dir, root_name):
     python_environment = Path(args["pyEnv"])
     root_name = args["rootName"]
     site_packages = args["sitePackages"]
-    editables = args["editables"]
+    editables = {k: Path(v) for k, v in args["editables"].items()}
 
     # directories to use
     root_dir = Path(run([find_root]))
     dream2nix_python_dir = root_dir / ".dream2nix" / "python"
-    editables_dir = dream2nix_python_dir / "editables"
     bin_dir = dream2nix_python_dir / "bin"
     site_dir = dream2nix_python_dir / "site"
 
@@ -228,25 +219,22 @@ def pretty_print_editables(editables, root_dir, root_name):
             shutil.rmtree(dream2nix_python_dir)
     else:
         export_environment_vars(python_environment, bin_dir, site_dir, site_packages)
-        pretty_print_editables(editables, root_dir, root_name)
+        pretty_print_editables(editables, root_dir)
         exit(0)
 
     bin_dir.mkdir(parents=True, exist_ok=True)
-    editables_dir.mkdir(parents=True, exist_ok=True)
     site_dir.mkdir(parents=True, exist_ok=True)
 
+    editable_dirs = []
     for name, path in editables.items():
-        make_editable(
+        editable_dir = make_editable(
             python_environment,
             bin_dir,
             site_dir,
-            editables_dir,
-            site_packages,
             name,
-            Path(path),
-            root_dir,
+            path,
         )
-
+        editable_dirs.append(str(editable_dir.absolute()))
     with open(site_dir / "sitecustomize.py", "w") as f:
         f.write(
             f"""import sys
@@ -264,7 +252,7 @@ def pretty_print_editables(editables, root_dir, root_name):
 # in our pyEnv, those would shadow the editables. So we move
 # the editables to the front of sys.path.
 for index, path in enumerate(sys.path):
-  if path.startswith("{editables_dir}"):
+  if path in {editable_dirs}:
     sys.path.insert(0, sys.path.pop(index))
         """
         )
@@ -273,4 +261,4 @@ def pretty_print_editables(editables, root_dir, root_name):
         json.dump(args, f, indent=2)
 
     export_environment_vars(python_environment, bin_dir, site_dir, site_packages)
-    pretty_print_editables(editables, root_dir, root_name)
+    pretty_print_editables(editables, root_dir)

modules/dream2nix/python-editables/interface.nix

@@ -8,15 +8,21 @@ in {
   options = {
     editables = lib.mkOption {
       description = ''
-        An attribute set mapping package names to absolute paths of source directories
-        which should be installed in editable mode in [editablesShellHook](#pipeditablesshellhook).
+        An attribute set mapping package names to a path, absolute or relative,
+        of source directories which should be installed in editable mode in
+        [editablesShellHook](#pipeditablesshellhook).
         i.e.
 
         ```
           pip.editables.charset-normalizer = "/home/user/src/charset-normalizer".
         ```
 
-        The top-level package is added automatically.
+        The top-level package is included automatically.
+
+        This must be a string, as otherwise content would be copied to the nix store
+        and loaded from there, voiding the benefits of editable installs.
+        For the same reason, it is advised to use source filtering if you
+        use a path inside the current repo to avoid unecessary rebuilds.
       '';
       type = t.attrsOf t.str;
     };