Merge pull request #3168 from regro/more-pypi-fixes

fix: allow for more pypi recipes to be updated

conda_forge_tick/update_recipe/version.py

@@ -141,24 +141,55 @@ def _render_jinja2(tmpl, context):
     )
 
 
-def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str):
-    if "name" not in context:
-        return None, None
-
+def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: Any):
     if "version" not in context:
         return None, None
 
     if not any(
         pypi_slug in url_tmpl
-        for pypi_slug in ["/pypi.org", "/pypi.io", "/files.pythonhosted.org"]
+        for pypi_slug in ["/pypi.org/", "/pypi.io/", "/files.pythonhosted.org/"]
     ):
         return None, None
 
-    r = requests.get(
-        f"https://pypi.org/simple/{context['name']}/",
-        headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+    orig_pypi_name = None
+    orig_pypi_name_candidates = [
+        url_tmpl.split("/")[-2],
+        context.get("name", None),
+        (cmeta.meta.get("package", {}) or {}).get("name", None),
+    ]
+    if "outputs" in cmeta.meta:
+        for output in cmeta.meta["outputs"]:
+            output = output or {}
+            orig_pypi_name_candidates.append(output.get("name", None))
+    orig_pypi_name_candidates = sorted(
+        {nc for nc in orig_pypi_name_candidates if nc is not None and len(nc) > 0},
+        key=lambda x: len(x),
     )
-    r.raise_for_status()
+    logger.info("PyPI name candidates: %s", orig_pypi_name_candidates)
+    for _orig_pypi_name in orig_pypi_name_candidates:
+        if _orig_pypi_name is None:
+            continue
+
+        if "{{ name }}" in _orig_pypi_name:
+            _orig_pypi_name = _render_jinja2(_orig_pypi_name, context)
+
+        try:
+            r = requests.get(
+                f"https://pypi.org/simple/{_orig_pypi_name}/",
+                headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+            )
+            r.raise_for_status()
+        except Exception as e:
+            logger.debug("PyPI API request failed: %s", repr(e), exc_info=e)
+        else:
+            orig_pypi_name = _orig_pypi_name
+            break
+
+    if orig_pypi_name is None or not r.ok:
+        logger.error("PyPI name not found!")
+        r.raise_for_status()
+
+    logger.info("PyPI name: %s", orig_pypi_name)
 
     data = r.json()
     logger.debug("PyPI API data:\n%s", pprint.pformat(data))
@@ -174,26 +205,29 @@ def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str):
                 break
 
     if finfo is None or ext is None:
-        logger.debug(
-            "src dist for version %s not found in PyPI API", context["version"]
+        logger.error(
+            "src dist for version %s not found in PyPI API for name %s",
+            context["version"],
+            orig_pypi_name,
         )
         return None, None
 
     bn, _ = os.path.split(url_tmpl)
     pypi_name = finfo["filename"].split(context["version"] + ext)[0]
     logger.debug("PyPI API file name: %s", pypi_name)
     name_tmpl = None
-    for tmpl in [
-        "{{ name }}",
-        "{{ name.lower() }}",
-        "{{ name.replace('-', '_') }}",
-        "{{ name.replace('_', '-') }}",
-        "{{ name.replace('-', '_').lower() }}",
-        "{{ name.replace('_', '-').lower() }}",
-    ]:
-        if pypi_name == _render_jinja2(tmpl, context) + "-":
-            name_tmpl = tmpl
-            break
+    if "name" in context:
+        for tmpl in [
+            "{{ name }}",
+            "{{ name.lower() }}",
+            "{{ name.replace('-', '_') }}",
+            "{{ name.replace('_', '-') }}",
+            "{{ name.replace('-', '_').lower() }}",
+            "{{ name.replace('_', '-').lower() }}",
+        ]:
+            if pypi_name == _render_jinja2(tmpl, context) + "-":
+                name_tmpl = tmpl
+                break
 
     if name_tmpl is not None:
         new_url_tmpl = os.path.join(bn, name_tmpl + "-" + "{{ version }}" + ext)
@@ -208,41 +242,42 @@ def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str):
     if new_hash is not None:
         return new_url_tmpl, new_hash
 
-    new_url_tmpl = finfo["url"]
+    new_url_tmpl = finfo["url"].replace(context["version"], "{{ version }}")
+    logger.debug("new url template from PyPI API: %s", new_url_tmpl)
+    url = _render_jinja2(new_url_tmpl, context)
     new_hash = _try_url_and_hash_it(url, hash_type)
     if new_hash is not None:
         return new_url_tmpl, new_hash
 
     return None, None
 
 
-def _get_new_url_tmpl_and_hash(url_tmpl: str, context: MutableMapping, hash_type: str):
+def _get_new_url_tmpl_and_hash(
+    url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: Any
+):
     logger.info(
-        "hashing URL template: %s",
+        "processing URL template: %s",
         url_tmpl,
     )
     try:
-        logger.info(
-            "rendered URL: %s",
-            _render_jinja2(url_tmpl, context),
-        )
+        url = _render_jinja2(url_tmpl, context)
+        logger.info("initial rendered URL: %s", url)
     except jinja2.UndefinedError:
         logger.info("initial URL template does not render")
         pass
 
-    try:
-        url = _render_jinja2(url_tmpl, context)
+    if url != url_tmpl:
         new_hash = _try_url_and_hash_it(url, hash_type)
         if new_hash is not None:
             return url_tmpl, new_hash
-    except jinja2.UndefinedError:
-        pass
+    else:
+        logger.info("initial URL template does not update with version. skipping it.")
 
     new_url_tmpl = None
     new_hash = None
 
     try:
-        new_url_tmpl, new_hash = _try_pypi_api(url_tmpl, context, hash_type)
+        new_url_tmpl, new_hash = _try_pypi_api(url_tmpl, context, hash_type, cmeta)
         if new_hash is not None and new_url_tmpl is not None:
             return new_url_tmpl, new_hash
     except Exception as e:
@@ -428,6 +463,7 @@ def _try_to_update_version(cmeta: Any, src: str, hash_type: str):
                     url_tmpl,
                     context,
                     hash_type,
+                    cmeta,
                 )
                 if new_hash is not None:
                     break
@@ -438,6 +474,7 @@ def _try_to_update_version(cmeta: Any, src: str, hash_type: str):
                 src[url_key],
                 context,
                 hash_type,
+                cmeta,
             )
             if new_hash is None:
                 errors.add("could not hash URL template '%s'" % src[url_key])

tests/test_version_migrator.py

@@ -62,6 +62,8 @@
         ("reproc", "14.2.5"),
         ("riskfolio_lib", "6.3.1"),
         ("algotree", "0.7.3"),
+        ("py_entitymatching", "0.4.2"),
+        ("py_entitymatching_name", "0.4.2"),
         # these contain sources that depend on conda build config variants
         pytest.param(
             "polars_mixed_selectors",

tests/test_yaml/version_py_entitymatching.yaml

@@ -0,0 +1,72 @@
+{% set version = "0.3.2" %}
+{% set sha256 = "f8e7a677901c4e35564d374b305e2beac13d615d49260787565877c0b07da1bb" %}
+
+package:
+  name: py_entitymatching
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/p/py_entitymatching/py_entitymatching-{{ version }}.tar.gz
+  sha256: {{ sha256 }}
+
+build:
+  number: 0
+  script: python -m pip install --no-deps --ignore-installed .
+  skip: True  # [not x86_64]
+
+requirements:
+  build:
+    - {{ compiler('c') }}
+    - {{ compiler('cxx') }}
+    - msinttypes  # [vc9]
+  host:
+    - python
+    - pip
+    - setuptools
+    - cython
+    - six
+    - py_stringsimjoin >=0.3.0
+    - cloudpickle
+    - pyparsing >=2.1.4
+    - scikit-learn >=0.18,<0.22.0a0
+    - xgboost  # [not win]
+    - pyqt
+    - requests
+    - ipython  # ==5.6
+    - numpy  # ==1.16.2
+  run:
+    - python
+    - {{ pin_compatible('numpy') }}
+    - six
+    - pandas
+    - joblib
+    - pyprind
+    - py_stringsimjoin >=0.3.0
+    - cloudpickle >=0.2.1
+    - pyparsing >=2.1.4
+    - scikit-learn >=0.18,<0.22.0a0
+    - xgboost  # [not win]
+    - pyqt
+    - requests
+    - ipython  # ==5.6
+    - matplotlib-base >=2.2.4
+
+test:
+  imports:
+    - py_entitymatching
+
+about:
+  home: https://sites.google.com/site/anhaidgroup/projects/magellan/py_entitymatching
+  license: BSD-3-Clause
+  license_family: BSD
+  license_file: LICENSE
+  summary: Python library for entity matching.
+  description: |
+    This project seeks to build a Python software package to match entities
+    between two tables using supervised learning.
+  doc_url: http://anhaidgroup.github.io/py_entitymatching/
+  dev_url: https://github.com/anhaidgroup/py_entitymatching
+
+extra:
+  recipe-maintainers:
+    - pjmartinkus

tests/test_yaml/version_py_entitymatching_correct.yaml

@@ -0,0 +1,72 @@
+{% set version = "0.4.2" %}
+{% set sha256 = "3656a8ab0bdce297ed3ae13c890c0091deeae79367aad44e3f84effd44b60f7f" %}
+
+package:
+  name: py_entitymatching
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/p/py_entitymatching/py-entitymatching-{{ version }}.tar.gz
+  sha256: {{ sha256 }}
+
+build:
+  number: 0
+  script: python -m pip install --no-deps --ignore-installed .
+  skip: true  # [not x86_64]
+
+requirements:
+  build:
+    - {{ compiler('c') }}
+    - {{ compiler('cxx') }}
+    - msinttypes  # [vc9]
+  host:
+    - python
+    - pip
+    - setuptools
+    - cython
+    - six
+    - py_stringsimjoin >=0.3.0
+    - cloudpickle
+    - pyparsing >=2.1.4
+    - scikit-learn >=0.18,<0.22.0a0
+    - xgboost  # [not win]
+    - pyqt
+    - requests
+    - ipython  # ==5.6
+    - numpy  # ==1.16.2
+  run:
+    - python
+    - {{ pin_compatible('numpy') }}
+    - six
+    - pandas
+    - joblib
+    - pyprind
+    - py_stringsimjoin >=0.3.0
+    - cloudpickle >=0.2.1
+    - pyparsing >=2.1.4
+    - scikit-learn >=0.18,<0.22.0a0
+    - xgboost  # [not win]
+    - pyqt
+    - requests
+    - ipython  # ==5.6
+    - matplotlib-base >=2.2.4
+
+test:
+  imports:
+    - py_entitymatching
+
+about:
+  home: https://sites.google.com/site/anhaidgroup/projects/magellan/py_entitymatching
+  license: BSD-3-Clause
+  license_family: BSD
+  license_file: LICENSE
+  summary: Python library for entity matching.
+  description: |
+    This project seeks to build a Python software package to match entities
+    between two tables using supervised learning.
+  doc_url: http://anhaidgroup.github.io/py_entitymatching/
+  dev_url: https://github.com/anhaidgroup/py_entitymatching
+
+extra:
+  recipe-maintainers:
+    - pjmartinkus