From 37cd6c5a83f0fe10146defaa53228bc853d6531f Mon Sep 17 00:00:00 2001 From: Sam Armstrong <88863522+Sam-Armstrong@users.noreply.github.com> Date: Sat, 4 Jan 2025 01:47:36 +0000 Subject: [PATCH] remove loading binaries from release pipeline --- scripts/load_relevant_binaries.py | 76 ------------------------------- scripts/shell/deploy_pypi.sh | 9 +--- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 scripts/load_relevant_binaries.py diff --git a/scripts/load_relevant_binaries.py b/scripts/load_relevant_binaries.py deleted file mode 100644 index 9d6930feac07..000000000000 --- a/scripts/load_relevant_binaries.py +++ /dev/null @@ -1,76 +0,0 @@ -import argparse -import json -import os -from packaging import tags -from urllib import request - - -def _get_paths_from_binaries(binaries, root_dir="", ext="so"): - """Get all the paths from the binaries.json into a list.""" - paths = [] - if isinstance(binaries, str): - return [os.path.join(root_dir, binaries + "." + ext)] - elif isinstance(binaries, dict): - for k, v in binaries.items(): - paths += _get_paths_from_binaries(v, os.path.join(root_dir, k), ext=ext) - else: - for i in binaries: - paths += _get_paths_from_binaries(i, root_dir, ext=ext) - return paths - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--tag", - type=str, - help="Tag for the binaries to pull", - ) - args = parser.parse_args() - tag = args.tag - ext = "pyd" if "win_" in tag else "so" - - folder_path = os.sep.join(__file__.split(os.sep)[:-2]) - binaries_path = os.path.join(folder_path, "binaries.json") - available_configs_path = os.path.join(folder_path, "available_configs.json") - if os.path.exists(binaries_path): - binaries_dict = json.load(open(binaries_path)) - available_configs = json.load(open(available_configs_path)) - binaries_paths = _get_paths_from_binaries(binaries_dict, folder_path, ext=ext) - binaries_exts = {path.split(".")[-1] for path in binaries_paths} - - # remove existing binaries - print("Removing existing binaries...", end="\r") - for root, _, files in os.walk(folder_path, topdown=True): - for file in files: - if file.split(".")[-1] in binaries_exts: - os.remove(os.path.join(root, file)) - print("Removing existing binaries --> done") - - print("Downloading new binaries...") - - version = os.environ["VERSION"] if "VERSION" in os.environ else "main" - for path in binaries_paths: - module = path[len(folder_path) :][1:].split(os.sep)[1] - if ( - os.path.exists(path) - or str(tag) not in available_configs[module] - ): - continue - folders = path.split(os.sep) - _, file_path = os.sep.join(folders[:-1]), folders[-1] - file_name = f"{file_path[:-(len(ext)+1)]}_{tag}.{ext}" - search_path = f"{module}/{file_name}" - try: - response = request.urlopen( - "https://github.com/ivy-llc/binaries/raw/" - f"{version}/{search_path}", - timeout=40, - ) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "wb") as f: - f.write(response.read()) - print("Downloaded", search_path) - except request.HTTPError: - print("Couldn't download", search_path) - break diff --git a/scripts/shell/deploy_pypi.sh b/scripts/shell/deploy_pypi.sh index e4fde97464f8..391124442146 100644 --- a/scripts/shell/deploy_pypi.sh +++ b/scripts/shell/deploy_pypi.sh @@ -1,8 +1,3 @@ -jq -c '.compiler[]' available_configs.json | while read config; do - export TAG=${config:1:${#config}-2} - export CLEAN=true - python3 scripts/load_relevant_binaries.py --tag $TAG - python -m build - python3 scripts/rename_wheels.py -done +python -m build +python3 scripts/rename_wheels.py python3 -m twine upload dist/* -u "__token__" -p "$PYPI_PASSWORD" --verbose