From c183323a28c429719cabc9447dae0bc284719536 Mon Sep 17 00:00:00 2001 From: Yannick DAYER Date: Thu, 25 Jul 2024 11:58:53 +0200 Subject: [PATCH] fix: add encoding spec to requests for some links. `requests.head` fails on some links due to an encoding issue. This forces the accepted encoding to be simple to accomodate most servers. --- src/auto_intersphinx/catalog.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/auto_intersphinx/catalog.py b/src/auto_intersphinx/catalog.py index 3d6fea9..c1f8f67 100644 --- a/src/auto_intersphinx/catalog.py +++ b/src/auto_intersphinx/catalog.py @@ -103,8 +103,12 @@ def docurls_from_environment(package: str) -> dict[str, str]: if k.startswith(("documentation, ", "Documentation, ")): raw_addr = k.split(",", 1)[1].strip() # Find the final address after eventual redirects. - redirected = requests.head(raw_addr, allow_redirects=True).url - addr = _ensure_webdir(redirected) + redirected = requests.head( + raw_addr, + allow_redirects=True, + headers={"Accept-Encoding": ""}, # Fixes issue on some links + ) + addr = _ensure_webdir(redirected.url) if requests.head(addr + "/objects.inv").ok: try: return {md["version"]: addr} @@ -198,8 +202,12 @@ def docurls_from_pypi(package: str, max_entries: int) -> dict[str, str]: addr = urls.get("Documentation") or urls.get("documentation") if addr is not None: # Find the final address after eventual redirects. - redirected = requests.head(addr, allow_redirects=True).url - addr = _ensure_webdir(redirected) + redirected = requests.head( + addr, + allow_redirects=True, + headers={"Accept-Encoding": ""}, # Fixes issue on some links + ) + addr = _ensure_webdir(redirected.url) if requests.head(addr + "/objects.inv").ok: versions[data["info"]["version"]] = addr