From 6c26ae0544fa12d0b8a584206aa67628c058f4c6 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Thu, 12 Dec 2024 00:37:55 +0000 Subject: [PATCH 1/2] Update README, add iommu_detect.sh --- README.md | 44 +++++++++++++++++++++------- scripts/iommu_detect.sh | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 11 deletions(-) create mode 100755 scripts/iommu_detect.sh diff --git a/README.md b/README.md index 3e6ecc61..9325f773 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# UMD -## About -Usermode Driver for Tenstorrent AI Accelerators +# Tenstorrent AI User-Mode Driver +## Official Repository +https://github.com/tenstorrent/tt-umd + +## Software Dependencies +UMD requires Tenstorrent's [kernel-mode driver](https://github.com/tenstorrent/tt-kmd) -## Dependencies Required Ubuntu dependencies: ``` sudo apt install -y libhwloc-dev cmake ninja-build @@ -15,9 +17,29 @@ chmod u+x llvm.sh sudo ./llvm.sh 17 ``` + +## IOMMU and Hugepage requirements +To determine whether your system requires hugepage configuration, run the provided script: + +```bash +./scripts/iommu_detect.sh +``` + +#### Grayskull and Wormhole +[1G hugepages](https://www.kernel.org/doc/Documentation/admin-guide/mm/hugetlbpage.rst) are required for shared device/host memory. Techniques for setup: + * Recommended: the [tt-system-tools](https://github.com/tenstorrent/tt-system-tools) repository contains a .deb package which will configure your system + * `sudo dpkg -i tenstorrent-tools_1.1-5_all.deb` + * Alternative: Metal project provides instructions and a [script](https://github.com/tenstorrent/tt-metal/blob/main/INSTALLING.md#step-3-hugepages). + * For experts: + * Put system IOMMU in passthrough mode or disable it + * Allocate 1 or more 1G hugepages + * Mount the hugetlbfs at /dev/hugepages-1G (e.g. `mount -t hugetlbfs hugetlbfs /dev/hugepages-1G -o mode=777,pagesize=1024M`) +#### Blackhole +If your system IOMMU is enabled, no hugepage setup is required. + ## Build flow -To build `libdevice.so`: +To build `libdevice.so`: ``` cmake -B build -G Ninja cmake --build build @@ -70,7 +92,7 @@ add_subdirectory() ### Ubuntu ``` -apt install ./umd-dev-x.y.z-Linux.deb +apt install ./umd-dev-x.y.z-Linux.deb ``` ## Simulator Integration @@ -100,7 +122,7 @@ To set up pre-commit on your local machine, follow these steps: Ensure you have Python installed, then run: ```bash pip install pre-commit - ``` + ``` 2. **Install the Git Hook Scripts**: In your local repository, run the following command to install the pre-commit hooks: ```bash @@ -113,10 +135,10 @@ To set up pre-commit on your local machine, follow these steps: pre-commit run --all-files ``` ## Why You Should Use Pre-commit -By setting up pre-commit locally, you can help maintain the quality of the codebase and ensure that commits consistently meet the project's formatting standards. This saves time during code reviews and reduces the likelihood of code formatting issues slipping into the repository. - -Since the hooks run automatically before each commit, you don't need to remember to manually format or check your code, making it easier to maintain consistency. - +By setting up pre-commit locally, you can help maintain the quality of the codebase and ensure that commits consistently meet the project's formatting standards. This saves time during code reviews and reduces the likelihood of code formatting issues slipping into the repository. + +Since the hooks run automatically before each commit, you don't need to remember to manually format or check your code, making it easier to maintain consistency. + We strongly encourage all developers to integrate pre-commit into their workflow. # Formatting C++ code diff --git a/scripts/iommu_detect.sh b/scripts/iommu_detect.sh new file mode 100755 index 00000000..c1525b3a --- /dev/null +++ b/scripts/iommu_detect.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Device IDs +TT_VID="1e52" +GS_PID="faca" +WH_PID="401e" +BH_PID="b140" + +# Find all Tenstorrent PCI devices with full domain info +tt_devices=$(lspci -D -d ${TT_VID}: | cut -d' ' -f1) + +if [ -z "$tt_devices" ]; then + echo "No Tenstorrent devices found" + exit 1 +fi + +found_gs_wh=false +found_bh=false + +for dev in $tt_devices; do + device_id=$(lspci -D -n -s "$dev" | cut -d' ' -f3 | cut -d: -f2) + echo "Checking device $dev (ID: $device_id):" + + case $device_id in + $GS_PID|$WH_PID) + found_gs_wh=true + if [ -f "/sys/bus/pci/devices/${dev}/iommu_group/type" ]; then + iommu_type=$(cat "/sys/bus/pci/devices/${dev}/iommu_group/type") + if [[ "$iommu_type" == *"DMA"* ]]; then + echo " WARNING: Grayskull/Wormhole device with IOMMU enabled (type: $iommu_type) - this configuration is not supported" + else + echo " Grayskull/Wormhole device detected - hugepages required" + fi + else + echo " Grayskull/Wormhole device detected - hugepages required" + fi + ;; + $BH_PID) + found_bh=true + if [ -f "/sys/bus/pci/devices/${dev}/iommu_group/type" ]; then + iommu_type=$(cat "/sys/bus/pci/devices/${dev}/iommu_group/type") + if [[ "$iommu_type" == *"DMA"* ]]; then + echo " Blackhole device with IOMMU enabled (type: $iommu_type) - hugepages optional" + else + echo " Blackhole device with IOMMU in passthrough mode (type: $iommu_type) - hugepages required" + fi + else + echo " Blackhole device with no IOMMU configuration - hugepages required" + fi + ;; + *) + echo " Unknown device ID: $device_id" + ;; + esac +done + +echo -e "\nSummary:" +if [ "$found_gs_wh" = true ]; then + echo "- System has Grayskull/Wormhole devices - hugepages required" + echo "- IOMMU must be disabled or in passthrough mode" +elif [ "$found_bh" = true ]; then + echo "- System has Blackhole devices - check IOMMU status above to determine if hugepages are needed" +fi From 5c34e2686df56b4d7feff0b75ea454bb238bde14 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Fri, 13 Dec 2024 17:10:00 +0000 Subject: [PATCH 2/2] update script --- scripts/iommu_detect.sh | 65 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/scripts/iommu_detect.sh b/scripts/iommu_detect.sh index c1525b3a..2ff94f72 100755 --- a/scripts/iommu_detect.sh +++ b/scripts/iommu_detect.sh @@ -6,7 +6,6 @@ GS_PID="faca" WH_PID="401e" BH_PID="b140" -# Find all Tenstorrent PCI devices with full domain info tt_devices=$(lspci -D -d ${TT_VID}: | cut -d' ' -f1) if [ -z "$tt_devices" ]; then @@ -17,43 +16,45 @@ fi found_gs_wh=false found_bh=false +# First identify devices for dev in $tt_devices; do device_id=$(lspci -D -n -s "$dev" | cut -d' ' -f3 | cut -d: -f2) - echo "Checking device $dev (ID: $device_id):" - case $device_id in - $GS_PID|$WH_PID) - found_gs_wh=true - if [ -f "/sys/bus/pci/devices/${dev}/iommu_group/type" ]; then - iommu_type=$(cat "/sys/bus/pci/devices/${dev}/iommu_group/type") - if [[ "$iommu_type" == *"DMA"* ]]; then - echo " WARNING: Grayskull/Wormhole device with IOMMU enabled (type: $iommu_type) - this configuration is not supported" - else - echo " Grayskull/Wormhole device detected - hugepages required" - fi - else - echo " Grayskull/Wormhole device detected - hugepages required" - fi - ;; - $BH_PID) - found_bh=true - if [ -f "/sys/bus/pci/devices/${dev}/iommu_group/type" ]; then - iommu_type=$(cat "/sys/bus/pci/devices/${dev}/iommu_group/type") - if [[ "$iommu_type" == *"DMA"* ]]; then - echo " Blackhole device with IOMMU enabled (type: $iommu_type) - hugepages optional" - else - echo " Blackhole device with IOMMU in passthrough mode (type: $iommu_type) - hugepages required" - fi - else - echo " Blackhole device with no IOMMU configuration - hugepages required" - fi - ;; - *) - echo " Unknown device ID: $device_id" - ;; + $GS_PID|$WH_PID) found_gs_wh=true ;; + $BH_PID) found_bh=true ;; esac done +# Check and output for each device +for dev in $tt_devices; do + device_id=$(lspci -D -n -s "$dev" | cut -d' ' -f3 | cut -d: -f2) + echo "Checking device $dev (ID: $device_id):" + + # Check IOMMU status for this device + iommu_enabled=false + iommu_type="none" + if [ -f "/sys/bus/pci/devices/${dev}/iommu_group/type" ]; then + iommu_type=$(cat "/sys/bus/pci/devices/${dev}/iommu_group/type") + [[ "$iommu_type" == *"DMA"* ]] && iommu_enabled=true + fi + + if [[ "$device_id" == "$GS_PID" || "$device_id" == "$WH_PID" ]]; then + if [ "$iommu_enabled" = true ]; then + echo " WARNING: Grayskull/Wormhole device with IOMMU enabled (type: $iommu_type) - this configuration is not supported" + else + echo " Grayskull/Wormhole device detected - hugepages required" + fi + elif [[ "$device_id" == "$BH_PID" ]]; then + if [ "$iommu_enabled" = true ]; then + echo " Blackhole device with IOMMU enabled (type: $iommu_type) - hugepages optional" + else + echo " Blackhole device with no IOMMU/passthrough (type: $iommu_type) - hugepages required" + fi + else + echo " Unknown device ID: $device_id" + fi +done + echo -e "\nSummary:" if [ "$found_gs_wh" = true ]; then echo "- System has Grayskull/Wormhole devices - hugepages required"