#493: Added runtime and performance silicon tests in CI

tenstorrent · Aug 27, 2024 · 181e40d · 181e40d
1 parent 9277218
commit 181e40d
Show file tree

Hide file tree

Showing 3 changed files with 322 additions and 0 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -0,0 +1,318 @@
+name: Build and Test
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  push:
+    branches:
+      - tpatel/issue-493
+
+jobs:
+
+  build-ttmlir:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: self-hosted, enable_perf: OFF, name: "run"},
+          {runs-on: self-hosted, enable_perf: ON, name: "perf"},
+        ]
+
+    runs-on: ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        create-symlink: true
+        key: ${{ matrix.build.runs-on }}-run-ON-perf-${{ matrix.build.enable_perf }}-${{ env.SDK_VERSION }}
+
+    - name: Configure CMake
+      shell: bash
+      run: |
+        source env/activate
+        cmake -G Ninja \
+        -B ${{ steps.strings.outputs.build-output-dir }} \
+        -DCMAKE_CXX_COMPILER=clang++-17 \
+        -DCMAKE_C_COMPILER=clang-17 \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DTTMLIR_ENABLE_RUNTIME=ON \
+        -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
+        -DTT_RUNTIME_ENABLE_PERF_TRACE=${{ matrix.build.enable_perf }} \
+        -DTTMLIR_ENABLE_STABLEHLO=ON \
+        -S ${{ steps.strings.outputs.work-dir }}
+
+    - name: Build
+      shell: bash
+      run: |
+        source env/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }}
+
+    - name: Build ttrt
+      shell: bash
+      run: |
+        source env/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }} -- ttrt
+
+    - name: Upload ttrt whl
+      uses: actions/upload-artifact@v4
+      with:
+        name: ttrt-whl-${{ matrix.build.name }}
+        path: build/runtime/tools/python/build/ttrt*.whl
+
+    - name: Upload build folder to archive
+      uses: actions/upload-artifact@v4
+      with:
+        name: build-artifacts-${{ matrix.build.name }}
+        path: build
+
+    - name: Get the latest tag
+      shell: bash
+      run: |
+        latest_tag=$(git describe --tags --abbrev=0)
+        latest_tag=${latest_tag#v}
+        echo "latest_tag=$latest_tag" >> $GITHUB_ENV
+        commit_count=$(git rev-list ${{ env.latest_tag }}..HEAD --count)
+        echo "commit_count=$commit_count" >> $GITHUB_ENV
+        version="${{ env.latest_tag }}.${{ env.commit_count }}"
+        echo "version=$version" >> $GITHUB_ENV
+        echo $version
+
+  get-system-descs:
+    needs: build-ttmlir
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: n150},
+          {runs-on: n300},
+        ]
+
+    runs-on:
+      - self-hosted
+      - ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G
+        - /etc/udev/rules.d:/etc/udev/rules.d
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+
+    steps:
+    - name: Remove existing whls files
+      shell: bash
+      run: |
+        rm -f *.whl
+
+    - name: Download ttrt run whls
+      uses: actions/download-artifact@v4
+      with:
+        name: ttrt-whl-run
+
+    - name: Install ttrt run whls
+      shell: bash
+      run: |
+        rm -rf whls_env
+        python3 -m venv whls_env
+        source whls_env/bin/activate
+        pip show ttrt && pip uninstall -y ttrt
+        pip install ttrt-${{ env.version }}*.whl --force-reinstall
+        pip install torch
+        pip install loguru
+        pip install pandas
+        pip install seaborn
+        pip install graphviz
+        pip install pyyaml
+        pip install click
+
+    - name: Generate system descriptor
+      shell: bash
+      run: |
+        source whls_env/bin/activate
+        ttrt query --save-artifacts
+
+    - name: Upload system descriptor
+      uses: actions/upload-artifact@v4
+      with:
+        name: system-desc-${{ matrix.build.runs-on }}
+        path: ttrt-artifacts/system_desc.ttsys
+
+  generate-tests:
+    needs: get-system-descs
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: self-hosted, enable_perf: OFF, name: "run"},
+          {runs-on: self-hosted, enable_perf: ON, name: "perf"},
+        ]
+
+    runs-on: ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+    - name: Use build artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: build-artifacts-${{ matrix.build.name }}
+        path: build
+
+    - name: Config compiler
+      shell: bash
+      run: |
+        source env/activate
+        cmake -G Ninja \
+        -B build \
+        -DCMAKE_CXX_COMPILER=clang++-17 \
+        -DCMAKE_C_COMPILER=clang-17 \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DTTMLIR_ENABLE_RUNTIME=ON \
+        -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
+        -DTT_RUNTIME_ENABLE_PERF_TRACE=${{ matrix.build.enable_perf }} \
+        -DTTMLIR_ENABLE_STABLEHLO=ON \
+
+    - name: (Host) Run tests
+      shell: bash
+      run: |
+        source env/activate
+        cmake --build build -- check-ttmlir
+
+    - name: Download n150 system descriptor
+      uses: actions/download-artifact@v4
+      with:
+        name: system-desc-n150
+
+    - name: (n150) Run tests
+      shell: bash
+      run: |
+        source env/activate
+        export SYSTEM_DESC_PATH="${GITHUB_WORKSPACE}/system_desc.ttsys"
+        cmake --build build -- check-ttmlir
+
+    - name: (n150) Upload test artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: n150-Silicon-${{ matrix.build.name }}
+        path: build/test/ttmlir/Silicon/
+
+    - name: Download n300 system descriptor
+      uses: actions/download-artifact@v4
+      with:
+        name: system-desc-n300
+
+    - name: (n300) Run tests
+      shell: bash
+      run: |
+        source env/activate
+        export SYSTEM_DESC_PATH="${GITHUB_WORKSPACE}/system_desc.ttsys"
+        cmake --build build -- check-ttmlir
+
+    - name: (n300) Upload test artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: n300-Silicon-${{ matrix.build.name }}
+        path: build/test/ttmlir/Silicon/
+
+  run-tests:
+    needs: generate-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: n150, name: "run"},
+          {runs-on: n150, name: "perf"},
+          {runs-on: n300, name: "run"},
+          {runs-on: n300, name: "perf"},
+        ]
+
+    runs-on:
+      - self-hosted
+      - ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-mlir-ubuntu-22-04:latest
+      options: --user root --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G
+        - /etc/udev/rules.d:/etc/udev/rules.d
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+
+    steps:
+    - name: Remove existing whls files
+      shell: bash
+      run: |
+        rm -f *.whl
+
+    - name: Download ttrt whls
+      uses: actions/download-artifact@v4
+      with:
+        name: ttrt-whl-${{ matrix.build.name }}
+
+    - name: Install ttrt whls
+      shell: bash
+      run: |
+        rm -rf whls_env
+        python3 -m venv whls_env
+        source whls_env/bin/activate
+        pip show ttrt && pip uninstall -y ttrt
+        pip install ttrt-${{ env.version }}*.whl --force-reinstall
+        pip install torch
+        pip install loguru
+        pip install pandas
+        pip install seaborn
+        pip install graphviz
+        pip install pyyaml
+        pip install click
+
+    - name: Download tests
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ matrix.build.runs-on }}-Silicon-${{ matrix.build.name }}
+
+    - name: Run tests
+      shell: bash
+      run: |
+        source whls_env/bin/activate
+        ttrt ${{ matrix.build.name }} Silicon/
diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml
@@ -18,3 +18,6 @@ jobs:
   docker-build:
     uses: ./.github/workflows/docker-build.yml
     secrets: inherit
+  build-and-test:
+    uses: ./.github/workflows/build-and-test.yml
+    secrets: inherit
diff --git a/runtime/tools/python/CMakeLists.txt b/runtime/tools/python/CMakeLists.txt
@@ -4,6 +4,7 @@ add_custom_target(ttrt-copy-files
 )
 
 add_custom_target(ttrt
+  COMMAND rm -f *.whl
   COMMAND TTMLIR_ENABLE_RUNTIME=${TTMLIR_ENABLE_RUNTIME}
           TT_RUNTIME_ENABLE_TTNN=${TT_RUNTIME_ENABLE_TTNN}
           TT_RUNTIME_ENABLE_TTMETAL=${TT_RUNTIME_ENABLE_TTMETAL}