Skip to content

Commit

Permalink
environment_yml_parser: Fix faulty package name/version parsing
Browse files Browse the repository at this point in the history
The code for extracting the package name from a Conda dependency
specifier was overly simplistic: it assumed that any version/build
info in the string always started with a "=" character. This is
obviously incorrect (e.g. "numpy>=1.23" is a simple counter-example).

Fix the parser to look for the same characters that the upstream looks
for, when separating package name from version + build info (see
https://github.com/conda/conda/blob/aa0fb6f3ae669a5ade575d340555aa6a9f71ef5e/conda/models/match_spec.py#L716
for details).

Add a test case copied from the cartopy project to illustrate the
correct use of such specifiers, and also add a couple of tests to verify
that we properly deal with invalid package names in dependency
specifiers (both Conda and Pip) inside an environment.yml file.
  • Loading branch information
jherland committed Dec 16, 2024
1 parent 06df200 commit 1b10a18
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 4 deletions.
24 changes: 20 additions & 4 deletions fawltydeps/extract_deps/environment_yml_parser.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Code for parsing dependencies from environment.yml files."""

import logging
import re
from pathlib import Path
from typing import Any, Callable, Dict, Iterator, List, Union

Expand All @@ -13,6 +14,13 @@
YamlDependencyData = Union[List[str], Dict[str, "YamlDependencyData"], Any, None] # type: ignore[misc]


class InvalidCondaRequirement(ValueError): # noqa: N818
"""Invalid Conda dependency specifier.
Modeled on packaging.requirements.InvalidRequirement.
"""


def parse_one_conda_dep(req_str: str, source: Location) -> DeclaredDependency:
"""Parse a single Conda dependency string.
Expand All @@ -24,9 +32,13 @@ def parse_one_conda_dep(req_str: str, source: Location) -> DeclaredDependency:
_, req_str = req_str.split("::", 1)
if "[" in req_str: # remove bracket stuff from back
req_str, _ = req_str.split("[]", 1)
req_str, *_ = req_str.split() # remove anything after whitespace
name, *_ = req_str.split("=") # remove version/build info
return DeclaredDependency(name, source)
# package name comes before version/build info, which starts with one of "><!=~ "
name_match = re.match(r"[^ =<>!~]+", req_str)
if not name_match:
raise InvalidCondaRequirement(
f"Expected package name at the start of dependency specifier: {req_str!r}"
)
return DeclaredDependency(name_match.group(0), source)


def parse_environment_yml_deps(
Expand All @@ -47,7 +59,11 @@ def parse_environment_yml_deps(
return
for dep_item in parsed_deps:
if isinstance(dep_item, str):
yield parse_item(dep_item, source)
try:
yield parse_item(dep_item, source)
except ValueError as e:
# InvalidCondaRequirement or packaging.requirements.InvalidRequirement
logger.error(f"{error_msg} {e}")
elif isinstance(dep_item, dict) and len(dep_item) == 1 and "pip" in dep_item:
pip_deps = dep_item.get("pip")
yield from parse_environment_yml_deps(
Expand Down
85 changes: 85 additions & 0 deletions tests/test_extract_deps_environment_yml.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,70 @@
["scikit-learn", "pip"],
id="mixed_conda_and_zero_pip_deps",
),
pytest.param(
"""\
# To set up a development environment using conda, run:
#
# conda env create -f environment.yml
# conda activate cartopy-dev
# pip install -e .
#
name: cartopy-dev
channels:
- conda-forge
dependencies:
- cython>=0.29.28
- numpy>=1.23
- shapely>=2.0
- pyshp>=2.3
- pyproj>=3.3.1
- packaging>=21
# The testing label has the proper version of freetype included
- conda-forge/label/testing::matplotlib-base>=3.6
# OWS
- owslib>=0.27
- pillow>=9.1
# Plotting
- scipy>=1.9
# Testing
- pytest
- pytest-mpl
- pytest-xdist
# Documentation
- pydata-sphinx-theme
- sphinx
- sphinx-gallery
# Extras
- pre-commit
- pykdtree
- ruff
- setuptools_scm
""",
[
"cython",
"numpy",
"shapely",
"pyshp",
"pyproj",
"packaging",
"matplotlib-base",
"owslib",
"pillow",
"scipy",
"pytest",
"pytest-mpl",
"pytest-xdist",
"pydata-sphinx-theme",
"sphinx",
"sphinx-gallery",
"pre-commit",
"pykdtree",
"ruff",
"setuptools_scm",
],
id="cartopy_example",
),
],
)
def test_parse_environment_yml__wellformed_dependencies__yields_dependencies(
Expand Down Expand Up @@ -239,6 +303,27 @@ class CondaTestVector:
error_msg_fragment="Failed to parse Pip dependencies in {path}: Not a string: ",
expect=["foo", "bar"],
),
CondaTestVector(
id="invalid_dependencies_malformed_names",
data="""\
dependencies:
- ">foo<"
- "bar"
""",
error_msg_fragment="Failed to parse Conda dependencies in {path}: Expected package name at the start of dependency specifier",
expect=["bar"],
),
CondaTestVector(
id="invalid_dependencies_malformed_names",
data="""\
dependencies:
- "pip":
- "~foo"
- "bar"
""",
error_msg_fragment="Failed to parse Pip dependencies in {path}: Expected package name at the start of dependency specifier",
expect=["bar"],
),
]


Expand Down

0 comments on commit 1b10a18

Please sign in to comment.