#493: Added runtime and performance silicon tests in CI (#502)

.github/workflows/build-and-test.yml

@@ -0,0 +1,192 @@
+name: Build and Test
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+
+  build-ttmlir:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: self-hosted, enable_perf: OFF, name: "run"},
+          {runs-on: self-hosted, enable_perf: ON, name: "perf"},
+        ]
+
+    runs-on: ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        create-symlink: true
+        key: ${{ matrix.build.runs-on }}-run-ON-perf-${{ matrix.build.enable_perf }}-${{ env.SDK_VERSION }}
+
+    - name: Configure CMake
+      shell: bash
+      run: |
+        source env/activate
+        cmake -G Ninja \
+        -B ${{ steps.strings.outputs.build-output-dir }} \
+        -DCMAKE_CXX_COMPILER=clang++-17 \
+        -DCMAKE_C_COMPILER=clang-17 \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DTTMLIR_ENABLE_RUNTIME=ON \
+        -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
+        -DTT_RUNTIME_ENABLE_PERF_TRACE=${{ matrix.build.enable_perf }} \
+        -DTTMLIR_ENABLE_STABLEHLO=ON \
+        -S ${{ steps.strings.outputs.work-dir }}
+
+    - name: Build
+      shell: bash
+      run: |
+        source env/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }}
+
+    - name: Build ttrt
+      shell: bash
+      run: |
+        source env/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }} -- ttrt
+
+    - name: Upload ttrt whl
+      uses: actions/upload-artifact@v4
+      with:
+        name: ttrt-whl-${{ matrix.build.name }}
+        path: build/runtime/tools/python/build/ttrt*.whl
+
+    - name: Upload build folder to archive
+      uses: actions/upload-artifact@v4
+      with:
+        name: build-artifacts-${{ matrix.build.name }}
+        path: build
+
+    - name: Get the latest tag
+      shell: bash
+      run: |
+        latest_tag=$(git describe --tags --abbrev=0)
+        latest_tag=${latest_tag#v}
+        echo "latest_tag=$latest_tag" >> $GITHUB_ENV
+        commit_count=$(git rev-list ${{ env.latest_tag }}..HEAD --count)
+        echo "commit_count=$commit_count" >> $GITHUB_ENV
+        version="${{ env.latest_tag }}.${{ env.commit_count }}"
+        echo "version=$version" >> $GITHUB_ENV
+        echo $version
+
+  run-tests:
+    needs: build-ttmlir
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: n150, enable_perf: OFF, name: "run"},
+          {runs-on: n150, enable_perf: ON, name: "perf"},
+          {runs-on: n300, enable_perf: OFF, name: "run"},
+          {runs-on: n300, enable_perf: ON, name: "perf"},
+        ]
+
+    runs-on:
+      - self-hosted
+      - ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G
+        - /etc/udev/rules.d:/etc/udev/rules.d
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+    - name: Use build artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: build-artifacts-${{ matrix.build.name }}
+        path: build
+
+    - name: Config compiler
+      shell: bash
+      run: |
+        source env/activate
+        cmake -G Ninja \
+        -B build \
+        -DCMAKE_CXX_COMPILER=clang++-17 \
+        -DCMAKE_C_COMPILER=clang-17 \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DTTMLIR_ENABLE_RUNTIME=ON \
+        -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
+        -DTT_RUNTIME_ENABLE_PERF_TRACE=${{ matrix.build.enable_perf }} \
+        -DTTMLIR_ENABLE_STABLEHLO=ON \
+
+    - name: Remove existing whls files
+      shell: bash
+      run: |
+        rm -f *.whl
+
+    - name: Download ttrt run whls
+      uses: actions/download-artifact@v4
+      with:
+        name: ttrt-whl-${{ matrix.build.name }}
+
+    - name: Install ttrt run whls
+      shell: bash
+      run: |
+        source env/activate
+        pip show ttrt && pip uninstall -y ttrt
+        pip install ttrt-${{ env.version }}*.whl --force-reinstall
+
+    - name: Generate system descriptor
+      shell: bash
+      run: |
+        source env/activate
+        ttrt query --save-artifacts
+
+    - name: Generate tests
+      shell: bash
+      run: |
+        source env/activate
+        export SYSTEM_DESC_PATH="${GITHUB_WORKSPACE}/ttrt-artifacts/system_desc.ttsys"
+        cmake --build build -- check-ttmlir
+
+    - name: Run tests
+      shell: bash
+      run: |
+        source env/activate
+        ttrt ${{ matrix.build.name }} ${GITHUB_WORKSPACE}/build/test/ttmlir/Silicon

.github/workflows/on-pr.yml

@@ -18,3 +18,6 @@ jobs:
   docker-build:
     uses: ./.github/workflows/docker-build.yml
     secrets: inherit
+  build-and-test:
+    uses: ./.github/workflows/build-and-test.yml
+    secrets: inherit

runtime/tools/python/CMakeLists.txt

@@ -4,6 +4,7 @@ add_custom_target(ttrt-copy-files
 )
 
 add_custom_target(ttrt
+  COMMAND rm -f *.whl
   COMMAND TTMLIR_ENABLE_RUNTIME=${TTMLIR_ENABLE_RUNTIME}
           TT_RUNTIME_ENABLE_TTNN=${TT_RUNTIME_ENABLE_TTNN}
           TT_RUNTIME_ENABLE_TTMETAL=${TT_RUNTIME_ENABLE_TTMETAL}

runtime/tools/python/setup.py

@@ -51,6 +51,8 @@
 runlibs = []
 perflibs = []
 metallibs = []
+install_requires = []
+install_requires += ["pybind11"]
 
 linklibs = ["TTBinary", "TTRuntimeSysDesc"]
 if enable_ttnn:
@@ -68,6 +70,7 @@
 
 if enable_runtime:
     assert enable_ttmetal or enable_ttnn, "At least one runtime must be enabled"
+    install_requires += ["torch"]
 
     for dylib in runlibs:
         shutil.copy(
@@ -188,6 +191,12 @@ def package_files(directory):
 packages = ["ttrt", "ttrt.common", "ttrt.binary", "ttrt.runtime"]
 package_dir = {}
 if enable_perf:
+    install_requires += ["loguru"]
+    install_requires += ["pandas"]
+    install_requires += ["seaborn"]
+    install_requires += ["graphviz"]
+    install_requires += ["pyyaml"]
+    install_requires += ["click"]
     packages += ["tracy"]
     packages += ["tt_metal"]
     package_dir["tracy"] = f"{ttmetalhome}/ttnn/tracy"
@@ -205,7 +214,7 @@ def package_files(directory):
     cmdclass={"build_ext": build_ext},
     packages=packages,
     package_dir=package_dir,
-    install_requires=["pybind11"],
+    install_requires=install_requires,
     entry_points={
         "console_scripts": ["ttrt = ttrt:main"],
     },

runtime/tools/python/ttrt/common/api.py

@@ -1092,7 +1092,7 @@ def check_constraints(self):
                 self.tracy_csvexport_tool_path
             ), f"perf tool={self.tracy_csvexport_tool_path} does not exist - rebuild using perf mode"
 
-            if "binary" not in self:
+            if not hasattr(self, "binary"):
                 # Load from Capsule instead. only TTNN Path is supported for now
                 bin = Binary(self.logger, self.file_manager, "", self["capsule"])
                 if not bin.check_version():

test/ttmlir/Silicon/TTMetal/simple_eltwise.mlir

@@ -1,4 +1,5 @@
 // RUN: ttmlir-opt --ttir-load-system-desc="path=%system_desc_path%" --ttir-to-ttmetal-backend-pipeline --ttmetal-serialize-to-binary="output=%t.ttm" %s | FileCheck %s
+// UNSUPPORTED: true
 #any_device = #tt.operand_constraint<dram|l1|scalar|tile|any_device|any_device_tile>
 
 func.func @multiply(%arg0: tensor<64x128xf32>, %arg1: tensor<64x128xf32>) -> tensor<64x128xf32> {