build no-rtti wheels too (TODO: mlir-aie-no-rtti)

.github/workflows/mlirAIEDistro.yml

@@ -31,7 +31,14 @@ on:
 
   # this is only for debugging this same yamls
   # comment it out when you're not working on these yamls
-#  pull_request:
+  pull_request:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit).
+  group: ci-build-mlir-aie-distro-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
 
 jobs:
 
@@ -209,7 +216,7 @@ jobs:
       shell: bash
       run: |
         pip install pkginfo
-        WHL=$(ls wheelhouse/mlir_aie-*whl)
+        WHL=$(ls wheelhouse/mlir_aie_*whl)
         echo "MLIR_AIE_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT
 
     - name: Download cache from container ubuntu
@@ -248,19 +255,19 @@ jobs:
       shell: bash
       run: |
         
-        rename 's/cp310-cp310/py3-none/' wheelhouse/mlir_aie-*whl
-        rename 's/cp311-cp311/py3-none/' wheelhouse/mlir_aie-*whl
+        rename 's/cp310-cp310/py3-none/' wheelhouse/mlir_aie_*whl
+        rename 's/cp311-cp311/py3-none/' wheelhouse/mlir_aie_*whl
         
         if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then
-          rename 's/x86_64/aarch64/' wheelhouse/mlir_aie-*whl
+          rename 's/x86_64/aarch64/' wheelhouse/mlir_aie_*whl
         fi
 
     - name: rename windows
       if: ${{ matrix.OS == 'windows-2019' }}
       working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
       run: |
-        ls wheelhouse/mlir_aie-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' }
-        ls wheelhouse/mlir_aie-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' }
+        ls wheelhouse/mlir_aie_*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' }
+        ls wheelhouse/mlir_aie_*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' }
 
     - name: build python bindings
       shell: bash
@@ -271,11 +278,11 @@ jobs:
         export PIP_NO_BUILD_ISOLATION=false
         
         cp requirements.txt python_bindings
-        cp wheelhouse/mlir_aie-*.whl python_bindings/
+        cp wheelhouse/mlir-aie-*.whl python_bindings/
         cp -r scripts python_bindings/scripts
         
         pushd python_bindings
-        unzip -q mlir_aie-*.whl
+        unzip -q mlir-aie-*.whl
         
         CIBW_ARCHS=${{ matrix.ARCH }} \
         CMAKE_GENERATOR="Ninja" \
@@ -381,7 +388,7 @@ jobs:
           cp requirements.txt python_bindings
           cp -R scripts python_bindings/scripts
           pushd python_bindings
-          unzip -q ../wheelhouse/mlir_aie-*-linux_aarch64.whl
+          unzip -q ../wheelhouse/mlir-aie-*-linux_aarch64.whl
           
           CIBW_ARCHS=${{ matrix.ARCH }} \
           CIBW_BUILD=${{ matrix.PY_VERSION }}-manylinux_aarch64 \
@@ -471,3 +478,4 @@ jobs:
           allowUpdates: true
           replacesArtifacts: true
           makeLatest: true
+          body: "`pip download mlir -f https://github.com/Xilinx/mlir-aie/releases/expanded_assets/latest-wheels && unzip mlir_aie*.whl`"

.github/workflows/mlirDistro.yml

@@ -36,11 +36,19 @@ on:
 
   # this is only for debugging this same yamls
   # comment it out when you're not working on these yamls
-#  pull_request:
+  # TODO(max): remove
+  pull_request:
 
   schedule:
-    # At minute 0 past every 6th hour. (see https://crontab.guru)
-    - cron: '0 */6 * * *'
+    # At minute 0 past every 4th hour. (see https://crontab.guru)
+    - cron: '0 */4 * * *'
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit).
+  group: ci-build-mlir-distro-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
 
 jobs:
 
@@ -73,33 +81,55 @@ jobs:
 
     needs: get_llvm_project_commit
 
-    name: ${{ matrix.OS }} ${{ matrix.ARCH }}
+    name: ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.ENABLE_RTTI }}
 
     continue-on-error: true
 
     runs-on: ${{ matrix.OS }}
 
-    outputs:
-      MLIR_WHEEL_VERSION: ${{ steps.get_wheel_version.outputs.MLIR_WHEEL_VERSION }}
-
     strategy:
       fail-fast: false
       matrix:
         include:
           - OS: ubuntu-20.04
             ARCH: x86_64
+            ENABLE_RTTI: ON
+
+          - OS: ubuntu-20.04
+            ARCH: aarch64
+            ENABLE_RTTI: ON
+
+          - OS: windows-2019
+            ARCH: AMD64
+            ENABLE_RTTI: ON
+
+          - OS: macos-11
+            ARCH: x86_64
+            ENABLE_RTTI: ON
+
+          - OS: macos-11
+            ARCH: arm64
+            ENABLE_RTTI: ON
+
+          - OS: ubuntu-20.04
+            ARCH: x86_64
+            ENABLE_RTTI: OFF
 
           - OS: ubuntu-20.04
             ARCH: aarch64
+            ENABLE_RTTI: OFF
 
           - OS: windows-2019
             ARCH: AMD64
+            ENABLE_RTTI: OFF
 
           - OS: macos-11
             ARCH: x86_64
+            ENABLE_RTTI: OFF
 
           - OS: macos-11
             ARCH: arm64
+            ENABLE_RTTI: OFF
 
     steps:
 
@@ -108,8 +138,7 @@ jobs:
       with:
         # checkout just the actions in order to pick and choose
         # where the actual repo is checked out manually (see actions/setup_base)
-        sparse-checkout: |
-          .github/actions
+        sparse-checkout: .github/actions
 
     - uses: ./.github/actions/setup_base
       id: setup_base
@@ -128,7 +157,7 @@ jobs:
       with:
         MATRIX_OS: ${{ matrix.OS }}
         MATRIX_ARCH: ${{ matrix.ARCH }}
-        EXTRA_KEY: mlir-distro
+        EXTRA_KEY: mlir-distro-enable-rtti-${{ matrix.ENABLE_RTTI }}
 
     # default workspace is repo root but we're not build mlir-aie here, we're building llvm+mlir
     # and thus llvm-project needs to be adjacent to utils/mlir_wheels/setup.py
@@ -169,6 +198,7 @@ jobs:
       working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
       run: |
         
+        ENABLE_RTTI=${{ matrix.ENABLE_RTTI }} \
         APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} \
         CIBW_ARCHS=${{ matrix.ARCH }} \
         CMAKE_GENERATOR=Ninja \
@@ -188,6 +218,7 @@ jobs:
         export APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }}
         ./scripts/apply_patches.sh
         
+        ENABLE_RTTI=${{ matrix.ENABLE_RTTI }} \
         CIBW_ARCHS=${{ matrix.ARCH }} \
         CMAKE_GENERATOR=Ninja \
         DATETIME=${{ needs.get_llvm_project_commit.outputs.DATETIME }} \
@@ -217,7 +248,7 @@ jobs:
       shell: bash
       run: |
         pip install pkginfo
-        WHL=$(ls wheelhouse/mlir-*whl)
+        WHL=$(ls wheelhouse/mlir*whl)
         echo "MLIR_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT
 
     - name: Download cache from container ubuntu
@@ -256,26 +287,27 @@ jobs:
       shell: bash
       run: |
         
-        rename 's/cp310-cp310/py3-none/' wheelhouse/mlir-*whl
-        rename 's/cp311-cp311/py3-none/' wheelhouse/mlir-*whl
+        rename 's/cp310-cp310/py3-none/' wheelhouse/mlir*whl
+        rename 's/cp311-cp311/py3-none/' wheelhouse/mlir*whl
         
         if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then
-          rename 's/x86_64/aarch64/' wheelhouse/mlir-*whl
+          rename 's/x86_64/aarch64/' wheelhouse/mlir*whl
         fi
 
     - name: rename windows
       if: ${{ matrix.OS == 'windows-2019' }}
       working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
       run: |
-        ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' }
-        ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' }
+        ls wheelhouse/mlir*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' }
+        ls wheelhouse/mlir*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' }
 
     # The "native tools" MLIR utilities that are necessary for cross-compiling MLIR - basically just tblgen.
     # Now if you build a whole distro you naturally do get those utilities but it's easier to just bundle them
     # together here and now and make them also available as a separate, much lighter wheel,
     # instead of asking/expecting someone to download the entire distro just for tblgen.
     - name: Build native_tools wheel
       working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}
+      if: matrix.ENABLE_RTTI == 'ON'
       shell: bash
       id: build_native_tools_wheel
       run: |
@@ -284,7 +316,7 @@ jobs:
           if [ x"${{ matrix.OS }}" == x"windows-2019" ]; then
             TOOL="$TOOL.exe"
           fi
-          unzip -j wheelhouse/mlir-*whl "mlir/bin/$TOOL" -d native_tools/
+          unzip -j wheelhouse/mlir*whl "mlir/bin/$TOOL" -d native_tools/
         done
         
         if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ]; then
@@ -306,17 +338,71 @@ jobs:
     # done
 
     - name: Upload wheels
-      if: github.event_name == 'workflow_dispatch' && (success() || failure())
+#      TODO(max): remove
+#      if: github.event_name == 'workflow_dispatch' && (success() || failure())
       uses: actions/upload-artifact@v3
       with:
         path: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl
         name: build_artifact
 
+  smoke_test_wheels:
+
+    name: Smoketest ${{ matrix.OS }} ${{ matrix.ARCH }} rtti=${{ matrix.NO_RTTI }}
+
+    needs: [build]
+
+    runs-on: ${{ matrix.OS }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - OS: ubuntu-20.04
+            ARCH: x86_64
+            NO_RTTI: ''
+
+          - OS: windows-2019
+            ARCH: AMD64
+            NO_RTTI: ''
+
+          - OS: macos-11
+            ARCH: x86_64
+            NO_RTTI: ''
+
+          - OS: ubuntu-20.04
+            ARCH: x86_64
+            NO_RTTI: '_no_rtti'
+
+          - OS: windows-2019
+            ARCH: AMD64
+            NO_RTTI: '_no_rtti'
+
+          - OS: macos-11
+            ARCH: x86_64
+            NO_RTTI: '_no_rtti'
+
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: build_artifact
+          path: dist
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: test
+        shell: bash
+        run: |
+          pip install numpy PyYAML
+          unzip mlir${{ matrix.NO_RTTI }}*.whl 
+          
+          PYTHONPATH=$(find . -name mlir_core) python -c 'import mlir${{ matrix.NO_RTTI }}.ir'
+
   upload_distro_wheels:
 
     if: github.event_name == 'workflow_dispatch'
 
-    needs: build
+    needs: smoke_test_wheels
 
     runs-on: ubuntu-latest
 
@@ -343,3 +429,4 @@ jobs:
           allowUpdates: true
           replacesArtifacts: true
           makeLatest: true
+          body: "`pip download mlir -f https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro && unzip mlir*.whl`"

utils/mlir_aie_wheels/pyproject.toml

@@ -23,6 +23,7 @@ environment-pass = [
     "PARALLEL_LEVEL",
     "PIP_FIND_LINKS",
     "PIP_NO_BUILD_ISOLATION",
+    "ENABLE_RTTI",
 ]
 repair-wheel-command = [
     "auditwheel repair -w {dest_dir} {wheel} --exclude libmlir_float16_utils.so"

utils/mlir_aie_wheels/python_bindings/setup.py

@@ -4,16 +4,15 @@
 import shutil
 import subprocess
 import sys
-from datetime import datetime
 from pathlib import Path
 from pprint import pprint
 
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 
 
-def check_env(build):
-    return os.environ.get(build, 0) in {"1", "true", "True", "ON", "YES"}
+def check_env(build, default=0):
+    return os.environ.get(build, default) in {"1", "true", "True", "ON", "YES"}
 
 
 class CMakeExtension(Extension):
@@ -62,8 +61,14 @@ def build_extension(self, ext: CMakeExtension) -> None:
 
         cmake_generator = os.environ.get("CMAKE_GENERATOR", "Ninja")
 
-        MLIR_AIE_INSTALL_ABS_PATH = (Path(__file__).parent / "mlir_aie").absolute()
-        MLIR_INSTALL_ABS_PATH = (Path(__file__).parent / "mlir").absolute()
+        MLIR_AIE_INSTALL_ABS_PATH = (
+            Path(__file__).parent
+            / ("mlir_aie" if check_env("ENABLE_RTTI", 1) else "mlir_aie_no_rtti")
+        ).absolute()
+        MLIR_INSTALL_ABS_PATH = (
+            Path(__file__).parent
+            / ("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti")
+        ).absolute()
         if platform.system() == "Windows":
             # fatal error LNK1170: line in command file contains 131071 or more characters
             if not Path("/tmp/a").exists():
@@ -75,7 +80,7 @@ def build_extension(self, ext: CMakeExtension) -> None:
 
         cmake_args = [
             f"-G {cmake_generator}",
-            f"-DMLIR_DIR={MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir'}",
+            f"-DMLIR_DIR={MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir' if check_env('ENABLE_RTTI') else 'mlir_no_rtti'}",
             f"-DAIE_DIR={MLIR_AIE_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'aie'}",
             f"-DCMAKE_INSTALL_PREFIX={install_dir}",
             f"-DPython3_EXECUTABLE={sys.executable}",
@@ -151,12 +156,16 @@ def build_extension(self, ext: CMakeExtension) -> None:
         print("CMAKE_ARGS", cmake_args, file=sys.stderr)
 
         subprocess.run(
-            ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True
+            ["cmake", ext.sourcedir, *cmake_args],
+            cwd=build_temp,
+            check=True,
+            capture_output=True,
         )
         subprocess.run(
             ["cmake", "--build", ".", "--target", "install", *build_args],
             cwd=build_temp,
             check=True,
+            capture_output=True,
         )
         shutil.copy(
             MLIR_AIE_INSTALL_ABS_PATH