From d308549118211bd80956a961ced36c719f04ea94 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 13 Dec 2023 03:01:17 -0600 Subject: [PATCH] build no-rtti wheels too (TODO: mlir-aie-no-rtti) --- .github/workflows/mlirAIEDistro.yml | 28 ++-- .github/workflows/mlirDistro.yml | 125 +++++++++++++++--- utils/mlir_aie_wheels/pyproject.toml | 1 + .../mlir_aie_wheels/python_bindings/setup.py | 23 +++- .../mlir_aie_wheels/scripts/download_mlir.sh | 15 ++- utils/mlir_aie_wheels/setup.py | 40 +++++- utils/mlir_wheels/pyproject.toml | 1 + utils/mlir_wheels/scripts/build_local.sh | 4 +- utils/mlir_wheels/scripts/pip_install_mlir.sh | 29 ---- utils/mlir_wheels/setup.py | 32 +++-- 10 files changed, 210 insertions(+), 88 deletions(-) delete mode 100755 utils/mlir_wheels/scripts/pip_install_mlir.sh diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index da283abf5b..f6d177fff7 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -31,7 +31,14 @@ on: # this is only for debugging this same yamls # comment it out when you're not working on these yamls -# pull_request: + pull_request: + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). + group: ci-build-mlir-aie-distro-${{ github.event.number || github.sha }} + cancel-in-progress: true jobs: @@ -209,7 +216,7 @@ jobs: shell: bash run: | pip install pkginfo - WHL=$(ls wheelhouse/mlir_aie-*whl) + WHL=$(ls wheelhouse/mlir_aie_*whl) echo "MLIR_AIE_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT - name: Download cache from container ubuntu @@ -248,19 +255,19 @@ jobs: shell: bash run: | - rename 's/cp310-cp310/py3-none/' wheelhouse/mlir_aie-*whl - rename 's/cp311-cp311/py3-none/' wheelhouse/mlir_aie-*whl + rename 's/cp310-cp310/py3-none/' wheelhouse/mlir_aie_*whl + rename 's/cp311-cp311/py3-none/' wheelhouse/mlir_aie_*whl if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then - rename 's/x86_64/aarch64/' wheelhouse/mlir_aie-*whl + rename 's/x86_64/aarch64/' wheelhouse/mlir_aie_*whl fi - name: rename windows if: ${{ matrix.OS == 'windows-2019' }} working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} run: | - ls wheelhouse/mlir_aie-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } - ls wheelhouse/mlir_aie-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } + ls wheelhouse/mlir_aie_*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } + ls wheelhouse/mlir_aie_*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } - name: build python bindings shell: bash @@ -271,11 +278,11 @@ jobs: export PIP_NO_BUILD_ISOLATION=false cp requirements.txt python_bindings - cp wheelhouse/mlir_aie-*.whl python_bindings/ + cp wheelhouse/mlir-aie-*.whl python_bindings/ cp -r scripts python_bindings/scripts pushd python_bindings - unzip -q mlir_aie-*.whl + unzip -q mlir-aie-*.whl CIBW_ARCHS=${{ matrix.ARCH }} \ CMAKE_GENERATOR="Ninja" \ @@ -381,7 +388,7 @@ jobs: cp requirements.txt python_bindings cp -R scripts python_bindings/scripts pushd python_bindings - unzip -q ../wheelhouse/mlir_aie-*-linux_aarch64.whl + unzip -q ../wheelhouse/mlir-aie-*-linux_aarch64.whl CIBW_ARCHS=${{ matrix.ARCH }} \ CIBW_BUILD=${{ matrix.PY_VERSION }}-manylinux_aarch64 \ @@ -471,3 +478,4 @@ jobs: allowUpdates: true replacesArtifacts: true makeLatest: true + body: "`pip download mlir -f https://github.com/Xilinx/mlir-aie/releases/expanded_assets/latest-wheels && unzip mlir_aie*.whl`" diff --git a/.github/workflows/mlirDistro.yml b/.github/workflows/mlirDistro.yml index 51e21100fa..7d2023e668 100644 --- a/.github/workflows/mlirDistro.yml +++ b/.github/workflows/mlirDistro.yml @@ -36,11 +36,19 @@ on: # this is only for debugging this same yamls # comment it out when you're not working on these yamls -# pull_request: + # TODO(max): remove + pull_request: schedule: - # At minute 0 past every 6th hour. (see https://crontab.guru) - - cron: '0 */6 * * *' + # At minute 0 past every 4th hour. (see https://crontab.guru) + - cron: '0 */4 * * *' + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). + group: ci-build-mlir-distro-${{ github.event.number || github.sha }} + cancel-in-progress: true jobs: @@ -73,33 +81,55 @@ jobs: needs: get_llvm_project_commit - name: ${{ matrix.OS }} ${{ matrix.ARCH }} + name: ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.ENABLE_RTTI }} continue-on-error: true runs-on: ${{ matrix.OS }} - outputs: - MLIR_WHEEL_VERSION: ${{ steps.get_wheel_version.outputs.MLIR_WHEEL_VERSION }} - strategy: fail-fast: false matrix: include: - OS: ubuntu-20.04 ARCH: x86_64 + ENABLE_RTTI: ON + + - OS: ubuntu-20.04 + ARCH: aarch64 + ENABLE_RTTI: ON + + - OS: windows-2019 + ARCH: AMD64 + ENABLE_RTTI: ON + + - OS: macos-11 + ARCH: x86_64 + ENABLE_RTTI: ON + + - OS: macos-11 + ARCH: arm64 + ENABLE_RTTI: ON + + - OS: ubuntu-20.04 + ARCH: x86_64 + ENABLE_RTTI: OFF - OS: ubuntu-20.04 ARCH: aarch64 + ENABLE_RTTI: OFF - OS: windows-2019 ARCH: AMD64 + ENABLE_RTTI: OFF - OS: macos-11 ARCH: x86_64 + ENABLE_RTTI: OFF - OS: macos-11 ARCH: arm64 + ENABLE_RTTI: OFF steps: @@ -108,8 +138,7 @@ jobs: with: # checkout just the actions in order to pick and choose # where the actual repo is checked out manually (see actions/setup_base) - sparse-checkout: | - .github/actions + sparse-checkout: .github/actions - uses: ./.github/actions/setup_base id: setup_base @@ -128,7 +157,7 @@ jobs: with: MATRIX_OS: ${{ matrix.OS }} MATRIX_ARCH: ${{ matrix.ARCH }} - EXTRA_KEY: mlir-distro + EXTRA_KEY: mlir-distro-enable-rtti-${{ matrix.ENABLE_RTTI }} # default workspace is repo root but we're not build mlir-aie here, we're building llvm+mlir # and thus llvm-project needs to be adjacent to utils/mlir_wheels/setup.py @@ -169,6 +198,7 @@ jobs: working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} run: | + ENABLE_RTTI=${{ matrix.ENABLE_RTTI }} \ APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} \ CIBW_ARCHS=${{ matrix.ARCH }} \ CMAKE_GENERATOR=Ninja \ @@ -188,6 +218,7 @@ jobs: export APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} ./scripts/apply_patches.sh + ENABLE_RTTI=${{ matrix.ENABLE_RTTI }} \ CIBW_ARCHS=${{ matrix.ARCH }} \ CMAKE_GENERATOR=Ninja \ DATETIME=${{ needs.get_llvm_project_commit.outputs.DATETIME }} \ @@ -217,7 +248,7 @@ jobs: shell: bash run: | pip install pkginfo - WHL=$(ls wheelhouse/mlir-*whl) + WHL=$(ls wheelhouse/mlir*whl) echo "MLIR_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT - name: Download cache from container ubuntu @@ -256,19 +287,19 @@ jobs: shell: bash run: | - rename 's/cp310-cp310/py3-none/' wheelhouse/mlir-*whl - rename 's/cp311-cp311/py3-none/' wheelhouse/mlir-*whl + rename 's/cp310-cp310/py3-none/' wheelhouse/mlir*whl + rename 's/cp311-cp311/py3-none/' wheelhouse/mlir*whl if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then - rename 's/x86_64/aarch64/' wheelhouse/mlir-*whl + rename 's/x86_64/aarch64/' wheelhouse/mlir*whl fi - name: rename windows if: ${{ matrix.OS == 'windows-2019' }} working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} run: | - ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } - ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } + ls wheelhouse/mlir*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } + ls wheelhouse/mlir*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } # The "native tools" MLIR utilities that are necessary for cross-compiling MLIR - basically just tblgen. # Now if you build a whole distro you naturally do get those utilities but it's easier to just bundle them @@ -276,6 +307,7 @@ jobs: # instead of asking/expecting someone to download the entire distro just for tblgen. - name: Build native_tools wheel working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} + if: matrix.ENABLE_RTTI == 'ON' shell: bash id: build_native_tools_wheel run: | @@ -284,7 +316,7 @@ jobs: if [ x"${{ matrix.OS }}" == x"windows-2019" ]; then TOOL="$TOOL.exe" fi - unzip -j wheelhouse/mlir-*whl "mlir/bin/$TOOL" -d native_tools/ + unzip -j wheelhouse/mlir*whl "mlir/bin/$TOOL" -d native_tools/ done if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ]; then @@ -306,17 +338,71 @@ jobs: # done - name: Upload wheels - if: github.event_name == 'workflow_dispatch' && (success() || failure()) +# TODO(max): remove +# if: github.event_name == 'workflow_dispatch' && (success() || failure()) uses: actions/upload-artifact@v3 with: path: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl name: build_artifact + smoke_test_wheels: + + name: Smoketest ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.NO_RTTI }} + + needs: [build] + + runs-on: ${{ matrix.OS }} + strategy: + fail-fast: false + matrix: + include: + - OS: ubuntu-20.04 + ARCH: x86_64 + NO_RTTI: '' + + - OS: windows-2019 + ARCH: AMD64 + NO_RTTI: '' + + - OS: macos-11 + ARCH: x86_64 + NO_RTTI: '' + + - OS: ubuntu-20.04 + ARCH: x86_64 + NO_RTTI: '_no_rtti' + + - OS: windows-2019 + ARCH: AMD64 + NO_RTTI: '_no_rtti' + + - OS: macos-11 + ARCH: x86_64 + NO_RTTI: '_no_rtti' + + steps: + - uses: actions/download-artifact@v3 + with: + name: build_artifact + path: dist + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: test + shell: bash + run: | + pip install numpy PyYAML + unzip mlir${{ matrix.NO_RTTI }}*.whl + + PYTHONPATH=$(find . -name mlir_core) python -c 'import mlir${{ matrix.NO_RTTI }}.ir' + upload_distro_wheels: if: github.event_name == 'workflow_dispatch' - needs: build + needs: smoke_test_wheels runs-on: ubuntu-latest @@ -343,3 +429,4 @@ jobs: allowUpdates: true replacesArtifacts: true makeLatest: true + body: "`pip download mlir -f https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro && unzip mlir*.whl`" diff --git a/utils/mlir_aie_wheels/pyproject.toml b/utils/mlir_aie_wheels/pyproject.toml index 07fc3d047a..7247f16b47 100644 --- a/utils/mlir_aie_wheels/pyproject.toml +++ b/utils/mlir_aie_wheels/pyproject.toml @@ -23,6 +23,7 @@ environment-pass = [ "PARALLEL_LEVEL", "PIP_FIND_LINKS", "PIP_NO_BUILD_ISOLATION", + "ENABLE_RTTI", ] repair-wheel-command = [ "auditwheel repair -w {dest_dir} {wheel} --exclude libmlir_float16_utils.so" diff --git a/utils/mlir_aie_wheels/python_bindings/setup.py b/utils/mlir_aie_wheels/python_bindings/setup.py index c56908d8d5..dc3b483245 100644 --- a/utils/mlir_aie_wheels/python_bindings/setup.py +++ b/utils/mlir_aie_wheels/python_bindings/setup.py @@ -4,7 +4,6 @@ import shutil import subprocess import sys -from datetime import datetime from pathlib import Path from pprint import pprint @@ -12,8 +11,8 @@ from setuptools.command.build_ext import build_ext -def check_env(build): - return os.environ.get(build, 0) in {"1", "true", "True", "ON", "YES"} +def check_env(build, default=0): + return os.environ.get(build, default) in {"1", "true", "True", "ON", "YES"} class CMakeExtension(Extension): @@ -62,8 +61,14 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_generator = os.environ.get("CMAKE_GENERATOR", "Ninja") - MLIR_AIE_INSTALL_ABS_PATH = (Path(__file__).parent / "mlir_aie").absolute() - MLIR_INSTALL_ABS_PATH = (Path(__file__).parent / "mlir").absolute() + MLIR_AIE_INSTALL_ABS_PATH = ( + Path(__file__).parent + / ("mlir_aie" if check_env("ENABLE_RTTI", 1) else "mlir_aie_no_rtti") + ).absolute() + MLIR_INSTALL_ABS_PATH = ( + Path(__file__).parent + / ("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti") + ).absolute() if platform.system() == "Windows": # fatal error LNK1170: line in command file contains 131071 or more characters if not Path("/tmp/a").exists(): @@ -75,7 +80,7 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_args = [ f"-G {cmake_generator}", - f"-DMLIR_DIR={MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir'}", + f"-DMLIR_DIR={MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir' if check_env('ENABLE_RTTI') else 'mlir_no_rtti'}", f"-DAIE_DIR={MLIR_AIE_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'aie'}", f"-DCMAKE_INSTALL_PREFIX={install_dir}", f"-DPython3_EXECUTABLE={sys.executable}", @@ -151,12 +156,16 @@ def build_extension(self, ext: CMakeExtension) -> None: print("CMAKE_ARGS", cmake_args, file=sys.stderr) subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True + ["cmake", ext.sourcedir, *cmake_args], + cwd=build_temp, + check=True, + capture_output=True, ) subprocess.run( ["cmake", "--build", ".", "--target", "install", *build_args], cwd=build_temp, check=True, + capture_output=True, ) shutil.copy( MLIR_AIE_INSTALL_ABS_PATH diff --git a/utils/mlir_aie_wheels/scripts/download_mlir.sh b/utils/mlir_aie_wheels/scripts/download_mlir.sh index 7c8a4d9ea3..cb5cf21ffc 100755 --- a/utils/mlir_aie_wheels/scripts/download_mlir.sh +++ b/utils/mlir_aie_wheels/scripts/download_mlir.sh @@ -1,22 +1,23 @@ #!/usr/bin/env bash set -xe -rm -rf mlir || true -rm -rf mlir-18* || true - -SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") +rm -rf mlir* || true pip install mlir-native-tools --force -U +if [ x"$ENABLE_RTTI" == x"OFF" ]; then + NO_RTTI="_no_rtti" +fi + if [ x"$CIBW_ARCHS" == x"arm64" ] || [ x"$CIBW_ARCHS" == x"aarch64" ]; then if [ x"$MATRIX_OS" == x"macos-11" ] && [ x"$CIBW_ARCHS" == x"arm64" ]; then PLAT=macosx_11_0_arm64 elif [ x"$MATRIX_OS" == x"ubuntu-20.04" ] && [ x"$CIBW_ARCHS" == x"aarch64" ]; then PLAT=linux_aarch64 fi - pip -q download mlir --platform $PLAT --only-binary=:all: + pip -q download mlir$NO_RTTI --platform $PLAT --only-binary=:all: else - pip -q download mlir + pip -q download mlir$NO_RTTI fi -unzip -q mlir-*whl +unzip -q mlir*whl diff --git a/utils/mlir_aie_wheels/setup.py b/utils/mlir_aie_wheels/setup.py index e36d6eb077..9da60f1dec 100644 --- a/utils/mlir_aie_wheels/setup.py +++ b/utils/mlir_aie_wheels/setup.py @@ -14,6 +14,10 @@ from setuptools.command.build_ext import build_ext +def check_env(build, default=0): + return os.environ.get(build, default) in {"1", "true", "True", "ON", "YES"} + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: str = "") -> None: super().__init__(name, sources=[]) @@ -32,7 +36,9 @@ def native_tools(): if f.name.startswith("mlir-tblgen") ) mlir_tblgen_target = next( - f.locate() for f in files("mlir") if f.name.startswith("mlir-tblgen") + f.locate() + for f in files("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti") + if f.name.startswith("mlir-tblgen") ) os.remove(mlir_tblgen_target) shutil.copy(mlir_tblgen_host, mlir_tblgen_target) @@ -42,7 +48,9 @@ def native_tools(): if f.name.startswith("mlir-pdll") ) mlir_pdll_target = next( - f.locate() for f in files("mlir") if f.name.startswith("mlir-pdll") + f.locate() + for f in files("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti") + if f.name.startswith("mlir-pdll") ) os.remove(mlir_pdll_target) shutil.copy(mlir_pdll_host, mlir_pdll_target) @@ -93,7 +101,9 @@ class CMakeBuild(build_ext): def build_extension(self, ext: CMakeExtension) -> None: ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) extdir = ext_fullpath.parent.resolve() - install_dir = extdir / "mlir_aie" + install_dir = extdir / ( + "mlir_aie" if check_env("ENABLE_RTTI", 1) else "mlir_aie_no_rtti" + ) cfg = "Release" cmake_generator = os.environ.get("CMAKE_GENERATOR", "Ninja") @@ -101,7 +111,10 @@ def build_extension(self, ext: CMakeExtension) -> None: if platform.system() == "Windows": cmake_module_path = cmake_module_path.replace("\\", "\\\\") - MLIR_INSTALL_ABS_PATH = (Path(__file__).parent / "mlir").absolute() + MLIR_INSTALL_ABS_PATH = ( + Path(__file__).parent + / ("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti") + ).absolute() if platform.system() == "Windows": # fatal error LNK1170: line in command file contains 131071 or more characters shutil.move(MLIR_INSTALL_ABS_PATH, "/tmp/m") @@ -120,6 +133,7 @@ def build_extension(self, ext: CMakeExtension) -> None: # causes pure duplication of various shlibs for Python wheels. "-DCMAKE_PLATFORM_NO_VERSIONED_SONAME=ON", "-DLLVM_CCACHE_BUILD=ON", + f"-DLLVM_ENABLE_RTTI={os.getenv('ENABLE_RTTI', 'ON')}", "-DAIE_ENABLE_BINDINGS_PYTHON=ON", f"-DCMAKE_INSTALL_PREFIX={install_dir}", f"-DCMAKE_PREFIX_PATH={MLIR_INSTALL_ABS_PATH}", @@ -202,14 +216,28 @@ def build_extension(self, ext: CMakeExtension) -> None: print("CMAKE_ARGS", cmake_args, file=sys.stderr) subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True + ["cmake", ext.sourcedir, *cmake_args], + cwd=build_temp, + check=True, + stdout=sys.stderr, + stderr=sys.stderr ) subprocess.run( ["cmake", "--build", ".", "--target", "install", *build_args], cwd=build_temp, check=True, + stdout=sys.stderr, + stderr=sys.stderr ) + # cibuildwheel containers are in the future? and this messes with ninja which checks timestamps + # when configuring cmake + for root, dirs, files in os.walk(install_dir): + for name in files: + os.utime( + os.path.join(root, name), (1602179630, 1602179630) + ) # just some random timestamp in the past + commit_hash = os.environ.get("AIE_PROJECT_COMMIT", "deadbeef") release_version = "0.0.1" @@ -222,7 +250,7 @@ def build_extension(self, ext: CMakeExtension) -> None: setup( version=version, author="", - name="mlir-aie", + name="mlir-aie" if check_env("ENABLE_RTTI", 1) else "mlir-aie-no-rtti", include_package_data=True, description=f"An MLIR-based toolchain for Xilinx Versal AIEngine-based devices.", long_description=dedent( diff --git a/utils/mlir_wheels/pyproject.toml b/utils/mlir_wheels/pyproject.toml index 6fa0e09de8..a09a8f0e8e 100644 --- a/utils/mlir_wheels/pyproject.toml +++ b/utils/mlir_wheels/pyproject.toml @@ -36,6 +36,7 @@ environment-pass = [ "PARALLEL_LEVEL", "PIP_FIND_LINKS", "PIP_NO_BUILD_ISOLATION", + "ENABLE_RTTI", ] [tool.cibuildwheel.macos] diff --git a/utils/mlir_wheels/scripts/build_local.sh b/utils/mlir_wheels/scripts/build_local.sh index c0f3c3603d..3e3496e9a2 100755 --- a/utils/mlir_wheels/scripts/build_local.sh +++ b/utils/mlir_wheels/scripts/build_local.sh @@ -43,7 +43,7 @@ ccache --show-config export HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" cibuildwheel "$HERE"/.. --platform "$machine" -rename 's/cp311-cp311/py3-none/' "$HERE/../wheelhouse/"mlir-*whl +rename 's/cp311-cp311/py3-none/' "$HERE/../wheelhouse/"mlir*whl if [ -d "$HERE/../wheelhouse/.ccache" ]; then cp -R "$HERE/../wheelhouse/.ccache/"* "$HOST_CCACHE_DIR/" @@ -53,7 +53,7 @@ for TOOL in "llvm-tblgen" "mlir-tblgen" "mlir-linalg-ods-yaml-gen" "mlir-pdll" " if [ x"$MATRIX_OS" == x"windows-2019" ]; then TOOL="$TOOL.exe" fi - unzip -j "$HERE/../wheelhouse/"mlir-*whl "mlir/bin/$TOOL" -d "$HERE/../native_tools/" + unzip -j "$HERE/../wheelhouse/"mlir*whl "mlir/bin/$TOOL" -d "$HERE/../native_tools/" done if [ x"$MATRIX_OS" == x"ubuntu-20.04" ]; then diff --git a/utils/mlir_wheels/scripts/pip_install_mlir.sh b/utils/mlir_wheels/scripts/pip_install_mlir.sh deleted file mode 100755 index fc7c6bf6e2..0000000000 --- a/utils/mlir_wheels/scripts/pip_install_mlir.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -xe - -export PIP_FIND_LINKS="wheelhouse https://makslevental.github.io/wheels" - -SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") - -if [ ! -z "$MLIR_WHEEL_VERSION" ]; then - pip install mlir-native-tools==$MLIR_WHEEL_VERSION --force -U - - DATE=$(echo $MLIR_WHEEL_VERSION | cut -d "+" -f 1) - HASH=$(echo $MLIR_WHEEL_VERSION | cut -d "+" -f 2) - LOCAL_VERSION="$LOCAL_VERSION $HASH" - LOCAL_VERSION=$(echo $LOCAL_VERSION | tr ' ' '.') - MLIR_WHEEL_VERSION="==$DATE+$LOCAL_VERSION" -else - pip install mlir-native-tools --force -U -fi - -if [ x"$CIBW_ARCHS" == x"arm64" ] || [ x"$CIBW_ARCHS" == x"aarch64" ]; then - if [ x"$MATRIX_OS" == x"macos-11" ] && [ x"$CIBW_ARCHS" == x"arm64" ]; then - PLAT=macosx_11_0_arm64 - elif [ x"$MATRIX_OS" == x"ubuntu-20.04" ] && [ x"$CIBW_ARCHS" == x"aarch64" ]; then - PLAT=linux_aarch64 - fi - pip install mlir$MLIR_WHEEL_VERSION --platform $PLAT --only-binary=:all: --target $SITE_PACKAGES --no-deps --force -U -else - pip install mlir$MLIR_WHEEL_VERSION --force -U -fi diff --git a/utils/mlir_wheels/setup.py b/utils/mlir_wheels/setup.py index 46146d160d..813910823b 100644 --- a/utils/mlir_wheels/setup.py +++ b/utils/mlir_wheels/setup.py @@ -11,6 +11,10 @@ from setuptools.command.build_ext import build_ext +def check_env(build, default=0): + return os.environ.get(build, default) in {"1", "true", "True", "ON", "YES"} + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: str = "") -> None: super().__init__(name, sources=[]) @@ -77,7 +81,9 @@ class CMakeBuild(build_ext): def build_extension(self, ext: CMakeExtension) -> None: ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) extdir = ext_fullpath.parent.resolve() - install_dir = extdir / "mlir" + install_dir = extdir / ( + "mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti" + ) cfg = "Release" cmake_generator = os.environ.get("CMAKE_GENERATOR", "Ninja") @@ -89,11 +95,10 @@ def build_extension(self, ext: CMakeExtension) -> None: "-DLLVM_BUILD_EXAMPLES=OFF", "-DLLVM_BUILD_RUNTIMES=OFF", "-DLLVM_BUILD_TESTS=OFF", - "-DLLVM_BUILD_TOOLS=ON", "-DLLVM_BUILD_UTILS=ON", "-DLLVM_CCACHE_BUILD=ON", "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DLLVM_ENABLE_RTTI=ON", + f"-DLLVM_ENABLE_RTTI={os.getenv('ENABLE_RTTI', 'ON')}", "-DLLVM_ENABLE_ZSTD=OFF", "-DLLVM_INCLUDE_BENCHMARKS=OFF", "-DLLVM_INCLUDE_EXAMPLES=OFF", @@ -194,17 +199,28 @@ def build_extension(self, ext: CMakeExtension) -> None: print("CMAKE_ARGS", cmake_args, file=sys.stderr) subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True + ["cmake", ext.sourcedir, *cmake_args], + cwd=build_temp, + check=True, + # cibuildwheel swallows stdout + stdout=sys.stderr, + stderr=sys.stderr ) subprocess.run( ["cmake", "--build", ".", "--target", "install", *build_args], cwd=build_temp, check=True, + stdout=sys.stderr, + stderr=sys.stderr ) - -def check_env(build): - return os.environ.get(build, 0) in {"1", "true", "True", "ON", "YES"} + # cibuildwheel containers are in the future? and this messes with ninja which checks timestamps + # when configuring cmake + for root, dirs, files in os.walk(install_dir): + for name in files: + os.utime( + os.path.join(root, name), (1602179630, 1602179630) + ) # just some random timestamp in the past cmake_txt = open("llvm-project/llvm/CMakeLists.txt").read() @@ -226,7 +242,7 @@ def check_env(build): llvm_url = f"https://github.com/llvm/llvm-project/commit/{commit_hash}" setup( - name="mlir", + name="mlir" if check_env("ENABLE_RTTI", 1) else "mlir-no-rtti", version=version, author="Maksim Levental", author_email="maksim.levental@gmail.com",