From 82d709124bacaf12b9c6332da5b17066abcba79a Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:39:54 +0000 Subject: [PATCH 01/12] add audit wheel to CI/CD for visibility of many_linux conformance --- .github/workflows/python-package.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a25f53f46..de1113f23 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -192,6 +192,19 @@ jobs: - name: Build wheel shell: bash run: python -m build . + - name: Show auditwheel information + if: matrix.os == 'ubuntu-latest' + run: | + pip install -q auditwheel + for whl in dist/*.whl; do + printf '%.0s=' {1..60} # separator line, same as below + echo -e "\nChecking wheel: $whl" + printf '%.0s-' {1..60} + echo + auditwheel show "$whl" + printf '%.0s-' {1..60} + echo + done - name: Upload build artifact uses: actions/upload-artifact@v4 with: From 0de046cdf4900d6a1fbead915a930d37e7b6cd95 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:28:35 +0000 Subject: [PATCH 02/12] temporarily aarch64 as not ready yet and not immediately needed --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index de1113f23..d485d3072 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -156,6 +156,8 @@ jobs: exclude: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 + - os: ubuntu-latest # Temporary. Takes too long, not ready yet. + arch: aarch64 runs-on: ${{ matrix.os }} steps: # Check out code From 012e61ad12c375a1f4109788436ad71d4672d6d7 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:12:54 +0000 Subject: [PATCH 03/12] switch to python script with GH markdown formatting --- .github/workflows/python-package.yml | 12 +----------- scripts/auditwheel_show.py | 29 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) create mode 100755 scripts/auditwheel_show.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d485d3072..e8fd5298c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -196,17 +196,7 @@ jobs: run: python -m build . - name: Show auditwheel information if: matrix.os == 'ubuntu-latest' - run: | - pip install -q auditwheel - for whl in dist/*.whl; do - printf '%.0s=' {1..60} # separator line, same as below - echo -e "\nChecking wheel: $whl" - printf '%.0s-' {1..60} - echo - auditwheel show "$whl" - printf '%.0s-' {1..60} - echo - done + run: pip install -q auditwheel && python ./scripts/auditwheel_show.py - name: Upload build artifact uses: actions/upload-artifact@v4 with: diff --git a/scripts/auditwheel_show.py b/scripts/auditwheel_show.py new file mode 100755 index 000000000..a74cba30a --- /dev/null +++ b/scripts/auditwheel_show.py @@ -0,0 +1,29 @@ +import glob +import os +import subprocess + + +def append_to_summary(content): + with open(os.getenv("GITHUB_STEP_SUMMARY"), "a") as summary_file: + summary_file.write(content + "\n") + + +subprocess.run(["pip", "install", "-q", "auditwheel"]) + +wheel_files = glob.glob("dist/*.whl") + +for whl in wheel_files: + append_to_summary("---") + append_to_summary("### šŸŽ” Auditing wheel: `" + whl + "`\n") + + audit_wheel_output = subprocess.run( + ["auditwheel", "show", whl], capture_output=True, text=True + ) + + if audit_wheel_output.stdout: + append_to_summary(audit_wheel_output.stdout + "\n") + + if audit_wheel_output.stderr: + append_to_summary("**Error:**\n```\n" + audit_wheel_output.stderr + "```\n") + + append_to_summary("\nšŸ **Slithering on to the next one...** šŸ\n") From e04b2c31b51555197a42f808233e8ec911fbc179 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 16:20:23 +0000 Subject: [PATCH 04/12] properly disable remaining arm linux builds --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e8fd5298c..1a35bf236 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -35,6 +35,8 @@ jobs: exclude: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 + - os: ubuntu-latest # Temporary. Takes too long, not ready yet. + arch: aarch64 runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents steps: # Check out code @@ -86,6 +88,8 @@ jobs: exclude: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 + - os: ubuntu-latest # Temporary. Takes too long, not ready yet. + arch: aarch64 runs-on: ${{ matrix.os }} # One day, we could run them on native agents. Azure supports this now but it's planned only for Q3 2023 for hosted agents steps: # Check out code From 8279dbcefc75e7d8c0b7071920452786aa69fc59 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:30:18 +0000 Subject: [PATCH 05/12] add debug print statement --- scripts/auditwheel_show.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auditwheel_show.py b/scripts/auditwheel_show.py index a74cba30a..5b2ce42ec 100755 --- a/scripts/auditwheel_show.py +++ b/scripts/auditwheel_show.py @@ -4,6 +4,7 @@ def append_to_summary(content): + print(content + '\n') # only for debugging now with open(os.getenv("GITHUB_STEP_SUMMARY"), "a") as summary_file: summary_file.write(content + "\n") From ad3a3e149caeca1d5a0d7de6ef63c13714f7ba07 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:43:06 +0000 Subject: [PATCH 06/12] simplify build matrices for debugging purposes --- .github/workflows/python-package.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1a35bf236..964461b66 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,8 +30,8 @@ jobs: build-shared-libs: strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - arch: [x86_64, aarch64] + os: [ubuntu-latest] # Keep only one OS + arch: [x86_64] # Keep only one architecture exclude: - os: windows-latest # This probably requires arm64 Windows agents arch: aarch64 @@ -82,8 +82,8 @@ jobs: build-shared-libs-cuda: strategy: matrix: - os: [ubuntu-latest, windows-latest] - arch: [x86_64, aarch64] + os: [ubuntu-latest] + arch: [x86_64] cuda_version: ['12.1.0'] exclude: - os: windows-latest # This probably requires arm64 Windows agents @@ -154,8 +154,8 @@ jobs: - build-shared-libs-cuda strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest] + python-version: ["3.9"] arch: [x86_64, aarch64] exclude: - os: windows-latest # This probably requires arm64 Windows agents From 21b417ac718b30b046cc7d447c4de718df25a88b Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:45:10 +0000 Subject: [PATCH 07/12] factor out audit wheels into a separate job --- .github/workflows/python-package.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 964461b66..0a59e19cf 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -198,15 +198,35 @@ jobs: - name: Build wheel shell: bash run: python -m build . - - name: Show auditwheel information - if: matrix.os == 'ubuntu-latest' - run: pip install -q auditwheel && python ./scripts/auditwheel_show.py - name: Upload build artifact uses: actions/upload-artifact@v4 with: name: bdist_wheel_${{ matrix.os }}_${{ matrix.arch }}_${{ matrix.python-version }} path: dist/bitsandbytes-*.whl retention-days: 7 + + audit-wheels: + needs: build-wheels + runs-on: ubuntu-latest + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + steps: + - name: Download all wheel artifacts + uses: actions/download-artifact@v2 + with: + path: dist/ + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + - name: Show auditwheel information + run: | + python --version + cat $GITHUB_STEP_SUMMARY + pip install -q auditwheel && python ./scripts/auditwheel_show.py + cat $GITHUB_STEP_SUMMARY + publish: needs: build-wheels runs-on: ubuntu-latest From bf3cb034cccfcae982ad681bcf617167911b553f Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:01:15 +0000 Subject: [PATCH 08/12] add missing checkout --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0a59e19cf..783c80001 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -211,6 +211,7 @@ jobs: env: PIP_DISABLE_PIP_VERSION_CHECK: 1 steps: + - uses: actions/checkout@v4 - name: Download all wheel artifacts uses: actions/download-artifact@v2 with: From 731e2c2ad30d99416fac5a0aeda7edaa0a69a650 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:01:48 +0000 Subject: [PATCH 09/12] fix indentation --- .github/workflows/python-package.yml | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 783c80001..39c4eef36 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -211,22 +211,22 @@ jobs: env: PIP_DISABLE_PIP_VERSION_CHECK: 1 steps: - - uses: actions/checkout@v4 - - name: Download all wheel artifacts - uses: actions/download-artifact@v2 - with: - path: dist/ - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: pip - - name: Show auditwheel information - run: | - python --version - cat $GITHUB_STEP_SUMMARY - pip install -q auditwheel && python ./scripts/auditwheel_show.py - cat $GITHUB_STEP_SUMMARY + - uses: actions/checkout@v4 + - name: Download all wheel artifacts + uses: actions/download-artifact@v2 + with: + path: dist/ + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + - name: Show auditwheel information + run: | + python --version + cat $GITHUB_STEP_SUMMARY + pip install -q auditwheel && python ./scripts/auditwheel_show.py + cat $GITHUB_STEP_SUMMARY publish: needs: build-wheels From d6b1471877ceec086bec9c2513d826a82de85039 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:32:09 +0000 Subject: [PATCH 10/12] improve script a bit --- scripts/auditwheel_show.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/auditwheel_show.py b/scripts/auditwheel_show.py index 5b2ce42ec..7b4e175f4 100755 --- a/scripts/auditwheel_show.py +++ b/scripts/auditwheel_show.py @@ -4,27 +4,29 @@ def append_to_summary(content): - print(content + '\n') # only for debugging now + print(content + "\n") # only for debugging now with open(os.getenv("GITHUB_STEP_SUMMARY"), "a") as summary_file: summary_file.write(content + "\n") subprocess.run(["pip", "install", "-q", "auditwheel"]) -wheel_files = glob.glob("dist/*.whl") +wheel_files = glob.glob("**/*.whl", recursive=True) + +if not wheel_files: + append_to_summary("No wheel files found in `dist/` directory.") + exit(0) for whl in wheel_files: append_to_summary("---") - append_to_summary("### šŸŽ” Auditing wheel: `" + whl + "`\n") + append_to_summary("### šŸŽ” Auditing wheel: `" + whl + "`") audit_wheel_output = subprocess.run( ["auditwheel", "show", whl], capture_output=True, text=True ) if audit_wheel_output.stdout: - append_to_summary(audit_wheel_output.stdout + "\n") + append_to_summary(audit_wheel_output.stdout) if audit_wheel_output.stderr: - append_to_summary("**Error:**\n```\n" + audit_wheel_output.stderr + "```\n") - - append_to_summary("\nšŸ **Slithering on to the next one...** šŸ\n") + append_to_summary("**Error:**\n```\n" + audit_wheel_output.stderr + "```") From 9c810b32c70741ecd43023db6124df0b84b9964c Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:33:00 +0000 Subject: [PATCH 11/12] add debugging print --- scripts/auditwheel_show.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auditwheel_show.py b/scripts/auditwheel_show.py index 7b4e175f4..57a115c12 100755 --- a/scripts/auditwheel_show.py +++ b/scripts/auditwheel_show.py @@ -12,6 +12,7 @@ def append_to_summary(content): subprocess.run(["pip", "install", "-q", "auditwheel"]) wheel_files = glob.glob("**/*.whl", recursive=True) +print(wheel_files) # only for debugging now if not wheel_files: append_to_summary("No wheel files found in `dist/` directory.") From 0d5c3b8a5aee52f655b00a289f5f0afb5a891907 Mon Sep 17 00:00:00 2001 From: Titus von Koeller <9048635+Titus-von-Koeller@users.noreply.github.com> Date: Thu, 7 Mar 2024 19:10:49 +0000 Subject: [PATCH 12/12] add tree for debugging --- .github/workflows/python-package.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 39c4eef36..0886e8a8e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -223,8 +223,9 @@ jobs: cache: pip - name: Show auditwheel information run: | - python --version - cat $GITHUB_STEP_SUMMARY + sudo apt-get update && sudo apt-get install -y tree + tree + echo $GITHUB_STEP_SUMMARY pip install -q auditwheel && python ./scripts/auditwheel_show.py cat $GITHUB_STEP_SUMMARY