From dfba4a7a336545e6f235d8dd2acb931a139976be Mon Sep 17 00:00:00 2001 From: pjanevski Date: Mon, 21 Oct 2024 14:30:31 +0000 Subject: [PATCH] Improve and refactor harvesting and coordinate management - Coordinate manager class for each arch - Write tests for the harvesting - Add functions to convert between coordinate systems --- device/CMakeLists.txt | 4 +- .../blackhole_coordinate_manager.cpp | 14 + .../blackhole/blackhole_coordinate_manager.h | 26 ++ device/coordinate_manager.cpp | 161 +++++++++ device/coordinate_manager.h | 67 ++++ .../grayskull/grayskull_coordinate_manager.h | 23 ++ device/tt_soc_descriptor.cpp | 42 ++- device/tt_soc_descriptor.h | 34 +- device/tt_xy_pair.h | 44 +++ .../wormhole/wormhole_coordinate_manager.cpp | 14 + device/wormhole/wormhole_coordinate_manager.h | 26 ++ tests/api/CMakeLists.txt | 1 + tests/api/test_soc_descriptor.cpp | 305 ++++++++++++++++++ 13 files changed, 745 insertions(+), 16 deletions(-) create mode 100644 device/blackhole/blackhole_coordinate_manager.cpp create mode 100644 device/blackhole/blackhole_coordinate_manager.h create mode 100644 device/coordinate_manager.cpp create mode 100644 device/coordinate_manager.h create mode 100644 device/grayskull/grayskull_coordinate_manager.h create mode 100644 device/wormhole/wormhole_coordinate_manager.cpp create mode 100644 device/wormhole/wormhole_coordinate_manager.h create mode 100644 tests/api/test_soc_descriptor.cpp diff --git a/device/CMakeLists.txt b/device/CMakeLists.txt index 750e9713..d196b362 100644 --- a/device/CMakeLists.txt +++ b/device/CMakeLists.txt @@ -1,4 +1,3 @@ - set(POSITION_INDEPENDENT_CODE ON) set(UMD_DEVICE_SRCS @@ -15,6 +14,9 @@ set(UMD_DEVICE_SRCS blackhole/blackhole_implementation.cpp grayskull/grayskull_implementation.cpp wormhole/wormhole_implementation.cpp + coordinate_manager.cpp + blackhole/blackhole_coordinate_manager.cpp + wormhole/wormhole_coordinate_manager.cpp pcie/pci_device.cpp ) diff --git a/device/blackhole/blackhole_coordinate_manager.cpp b/device/blackhole/blackhole_coordinate_manager.cpp new file mode 100644 index 00000000..0d73918e --- /dev/null +++ b/device/blackhole/blackhole_coordinate_manager.cpp @@ -0,0 +1,14 @@ +#include "blackhole_coordinate_manager.h" + +std::set BlackholeCoordinateManager::get_x_coordinates_to_harvest(std::size_t harvesting_mask) { + std::set x_to_harvest; + std::size_t logical_x = 0; + while (harvesting_mask > 0) { + if (harvesting_mask & 1) { + x_to_harvest.insert(logical_x); + } + logical_x++; + harvesting_mask >>= 1; + } + return x_to_harvest; +} diff --git a/device/blackhole/blackhole_coordinate_manager.h b/device/blackhole/blackhole_coordinate_manager.h new file mode 100644 index 00000000..19b12436 --- /dev/null +++ b/device/blackhole/blackhole_coordinate_manager.h @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: (c) 2023 Tenstorrent Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "device/coordinate_manager.h" + +class BlackholeCoordinateManager : public CoordinateManager { + +public: + BlackholeCoordinateManager(const tt_xy_pair& worker_grid_size, const std::vector& workers, std::size_t harvesting_mask) + : CoordinateManager(worker_grid_size, workers, harvesting_mask) {} + +protected: + std::set get_x_coordinates_to_harvest(std::size_t harvesting_mask) override; +}; \ No newline at end of file diff --git a/device/coordinate_manager.cpp b/device/coordinate_manager.cpp new file mode 100644 index 00000000..2ef3695b --- /dev/null +++ b/device/coordinate_manager.cpp @@ -0,0 +1,161 @@ +#include "device/coordinate_manager.h" +#include +#include "coordinate_manager.h" +#include "grayskull/grayskull_coordinate_manager.h" + +tt_physical_coords CoordinateManager::logical_to_physical_coords(tt_logical_coords logical_coords) { + // log_assert(logical_coords.x < logical_x_to_physical_x.size()); + // log_assert(logical_coords.y < logical_y_to_physical_y.size()); + return tt_physical_coords(logical_x_to_physical_x[logical_coords.x], logical_y_to_physical_y[logical_coords.y]); +} + +// TODO(pjanevski): is it enough just to add 18 to the logical coordinates +// in order to get the translated coordinates? +tt_translated_coords CoordinateManager::logical_to_translated_coords(tt_logical_coords logical_coords) { + static const std::size_t translated_offset = 18; + return tt_translated_coords(logical_coords.x + translated_offset, logical_coords.y + translated_offset); +} + +tt_logical_coords CoordinateManager::physical_to_logical_coords(tt_physical_coords physical_coords) { + return tt_logical_coords(physical_x_to_logical_x[physical_coords.x], physical_y_to_logical_y[physical_coords.y]); +} + +tt_translated_coords CoordinateManager::physical_to_translated_coords(tt_physical_coords physical_coords) { + return tt_translated_coords(0, 0); +} + +tt_virtual_coords CoordinateManager::logical_to_virtual_coords(tt_logical_coords logical_coords) { + return tt_virtual_coords(logical_x_to_virtual_x[logical_coords.x], logical_y_to_virtual_y[logical_coords.y]); +} + +tt_logical_coords CoordinateManager::virtual_to_logical_coords(tt_virtual_coords virtual_coords) { + return tt_logical_coords(virtual_x_to_logical_x[virtual_coords.x], virtual_y_to_logical_y[virtual_coords.y]); +} + +void CoordinateManager::clear_harvesting_structures() { + logical_x_to_physical_x.clear(); + logical_y_to_physical_y.clear(); + logical_x_to_virtual_x.clear(); + logical_y_to_virtual_y.clear(); + physical_x_to_logical_x.clear(); + physical_y_to_logical_y.clear(); + virtual_x_to_logical_x.clear(); + virtual_y_to_logical_y.clear(); +} + +std::set CoordinateManager::get_x_coordinates_to_harvest(std::size_t harvesting_mask) { + return {}; +} + +std::set CoordinateManager::get_y_coordinates_to_harvest(std::size_t harvesting_mask) { + return {}; +} + +void CoordinateManager::perform_harvesting(std::size_t harvesting_mask) { + clear_harvesting_structures(); + + std::set physical_x_unharvested; + std::set physical_y_unharvested; + for (auto core : workers) { + physical_x_unharvested.insert(core.x); + physical_y_unharvested.insert(core.y); + } + + std::set x_coordinates_to_harvest = get_x_coordinates_to_harvest(harvesting_mask); + std::set y_coordinates_to_harvest = get_y_coordinates_to_harvest(harvesting_mask); + + std::size_t num_harvested_y = y_coordinates_to_harvest.size(); + std::size_t num_harvested_x = x_coordinates_to_harvest.size(); + + std::size_t grid_size_x = worker_grid_size.x; + std::size_t grid_size_y = worker_grid_size.y; + + logical_x_to_physical_x.resize(grid_size_x - num_harvested_x); + logical_y_to_physical_y.resize(grid_size_y - num_harvested_y); + + logical_x_to_virtual_x.resize(grid_size_x - num_harvested_x); + logical_y_to_virtual_y.resize(grid_size_y - num_harvested_y); + + fill_logical_to_physical_mapping(x_coordinates_to_harvest, y_coordinates_to_harvest, physical_x_unharvested, physical_y_unharvested); + fill_logical_to_virtual_mapping(physical_x_unharvested, physical_y_unharvested); +} + +void CoordinateManager::fill_logical_to_physical_mapping( + const std::set& x_to_harvest, const std::set& y_to_harvest, + const std::set& physical_x_unharvested, const std::set& physical_y_unharvested) { + + auto physical_y_it = physical_y_unharvested.begin(); + std::size_t logical_y = 0; + for (size_t y = 0; y < worker_grid_size.y; y++) { + if (y_to_harvest.find(y) == y_to_harvest.end()) { + logical_y_to_physical_y[logical_y] = *physical_y_it; + if (physical_y_to_logical_y.find(*physical_y_it) != physical_y_to_logical_y.end()) { + throw std::runtime_error("Duplicate physical y coordinate found in the worker cores"); + } + physical_y_to_logical_y[*physical_y_it] = logical_y; + logical_y++; + physical_y_it++; + } else { + physical_y_it++; + } + } + + auto physical_x_it = physical_x_unharvested.begin(); + std::size_t logical_x = 0; + for(std::size_t x = 0; x < worker_grid_size.x; x++) { + if (x_to_harvest.find(x) == x_to_harvest.end()) { + logical_x_to_physical_x[logical_x] = *physical_x_it; + if (physical_x_to_logical_x.find(*physical_x_it) != physical_x_to_logical_x.end()) { + throw std::runtime_error("Duplicate physical x coordinate found in the worker cores"); + } + physical_x_to_logical_x[*physical_x_it] = logical_x; + logical_x++; + physical_x_it++; + } else { + physical_x_it++; + } + } +} + +void CoordinateManager::fill_logical_to_virtual_mapping(const std::set& physical_x_unharvested, const std::set& physical_y_unharvested) { + auto physical_y_it = physical_y_unharvested.begin(); + for (std::size_t y = 0; y < logical_y_to_virtual_y.size(); y++) { + logical_y_to_virtual_y[y] = *physical_y_it; + if (virtual_y_to_logical_y.find(*physical_y_it) != virtual_y_to_logical_y.end()) { + throw std::runtime_error("Duplicate virtual y coordinate found in the worker cores"); + } + virtual_y_to_logical_y[*physical_y_it] = y; + physical_y_it++; + } + + auto physical_x_it = physical_x_unharvested.begin(); + for (std::size_t x = 0; x < logical_x_to_virtual_x.size(); x++) { + logical_x_to_virtual_x[x] = *physical_x_it; + if (virtual_x_to_logical_x.find(*physical_x_it) != virtual_x_to_logical_x.end()) { + throw std::runtime_error("Duplicate virtual x coordinate found in the worker cores"); + } + virtual_x_to_logical_x[*physical_x_it] = x; + physical_x_it++; + } +} + +#include "device/blackhole/blackhole_coordinate_manager.h" +#include "device/grayskull/grayskull_coordinate_manager.h" +#include "device/wormhole/wormhole_coordinate_manager.h" + +CoordinateManager* CoordinateManager::get_coordinate_manager( + tt::ARCH arch, + const tt_xy_pair& worker_grid_size, + const std::vector& workers, + std::size_t harvesting_mask) { + + if (arch == tt::ARCH::GRAYSKULL) { + return new GrayskullCoordinateManager(worker_grid_size, workers, harvesting_mask); + } else if (arch == tt::ARCH::WORMHOLE_B0) { + return new WormholeCoordinateManager(worker_grid_size, workers, harvesting_mask); + } else if (arch == tt::ARCH::BLACKHOLE) { + return new BlackholeCoordinateManager(worker_grid_size, workers, harvesting_mask); + } else { + throw std::runtime_error("Invalid architecture for coordinate manager"); + } +} \ No newline at end of file diff --git a/device/coordinate_manager.h b/device/coordinate_manager.h new file mode 100644 index 00000000..86b2f1f5 --- /dev/null +++ b/device/coordinate_manager.h @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: (c) 2023 Tenstorrent Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "device/tt_xy_pair.h" +#include "device/tt_arch_types.h" +#include + +class CoordinateManager { + +public: + CoordinateManager(const tt_xy_pair& worker_grid_size, const std::vector& workers, std::size_t harvesting_mask) + : worker_grid_size(worker_grid_size), workers(workers), harvesting_mask(harvesting_mask) {} + + virtual void perform_harvesting(std::size_t harvesting_mask); + + virtual tt_physical_coords logical_to_physical_coords(tt_logical_coords logical_coords); + virtual tt_translated_coords logical_to_translated_coords(tt_logical_coords logical_coords); + virtual tt_logical_coords physical_to_logical_coords(tt_physical_coords physical_coords); + virtual tt_translated_coords physical_to_translated_coords(tt_physical_coords physical_coords); + virtual tt_virtual_coords logical_to_virtual_coords(tt_logical_coords logical_coords); + virtual tt_logical_coords virtual_to_logical_coords(tt_virtual_coords virtual_coords); + + static CoordinateManager* get_coordinate_manager( + tt::ARCH arch, + const tt_xy_pair& worker_grid_size, + const std::vector& workers, + std::size_t harvesting_mask); + +protected: + virtual void clear_harvesting_structures(); + + virtual std::set get_x_coordinates_to_harvest(std::size_t harvesting_mask); + virtual std::set get_y_coordinates_to_harvest(std::size_t harvesting_mask); + + virtual void fill_logical_to_physical_mapping( + const std::set& x_to_harvest, const std::set& y_to_harvest, + const std::set& physical_x_unharvested, const std::set& physical_y_unharvested); + virtual void fill_logical_to_virtual_mapping(const std::set& physical_x_unharvested, const std::set& physical_y_unharvested); + + std::map physical_y_to_logical_y; + std::map physical_x_to_logical_x; + + std::vector logical_y_to_physical_y; + std::vector logical_x_to_physical_x; + + std::vector logical_y_to_virtual_y; + std::vector logical_x_to_virtual_x; + + std::map virtual_y_to_logical_y; + std::map virtual_x_to_logical_x; + + const tt_xy_pair worker_grid_size; + const std::vector& workers; + const std::size_t harvesting_mask; +}; \ No newline at end of file diff --git a/device/grayskull/grayskull_coordinate_manager.h b/device/grayskull/grayskull_coordinate_manager.h new file mode 100644 index 00000000..240c2a1d --- /dev/null +++ b/device/grayskull/grayskull_coordinate_manager.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: (c) 2023 Tenstorrent Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "device/coordinate_manager.h" + +class GrayskullCoordinateManager : public CoordinateManager { + +public: + GrayskullCoordinateManager(const tt_xy_pair& worker_grid_size, const std::vector& workers, std::size_t harvesting_mask) + : CoordinateManager(worker_grid_size, workers, harvesting_mask) {} +}; \ No newline at end of file diff --git a/device/tt_soc_descriptor.cpp b/device/tt_soc_descriptor.cpp index d69a47c8..b8164ed9 100644 --- a/device/tt_soc_descriptor.cpp +++ b/device/tt_soc_descriptor.cpp @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 +#include "coordinate_manager.h" +#include "tt_xy_pair.h" #include "yaml-cpp/yaml.h" #include "tt_soc_descriptor.h" @@ -9,6 +11,7 @@ #include #include #include +#include #include #include @@ -58,10 +61,7 @@ inline std::string& trim(std::string& s, const char* t = ws) } void tt_SocDescriptor::load_soc_features_from_device_descriptor(YAML::Node &device_descriptor_yaml) { - overlay_version = device_descriptor_yaml["features"]["overlay"]["version"].as(); noc_translation_id_enabled = device_descriptor_yaml["features"]["noc"] && device_descriptor_yaml["features"]["noc"]["translation_id_enabled"] ? device_descriptor_yaml["features"]["noc"]["translation_id_enabled"].as() : false; - packer_version = device_descriptor_yaml["features"]["packer"]["version"].as(); - unpacker_version = device_descriptor_yaml["features"]["unpacker"]["version"].as(); dst_size_alignment = device_descriptor_yaml["features"]["math"]["dst_size_alignment"].as(); worker_l1_size = device_descriptor_yaml["worker_l1_size"].as(); eth_l1_size = device_descriptor_yaml["eth_l1_size"].as(); @@ -166,7 +166,39 @@ void tt_SocDescriptor::load_core_descriptors_from_device_descriptor(YAML::Node & } } -tt_SocDescriptor::tt_SocDescriptor(std::string device_descriptor_path) { +void tt_SocDescriptor::create_coordinate_manager(std::size_t harvesting_mask) { + coordinate_manager = CoordinateManager::get_coordinate_manager(arch, worker_grid_size, workers, harvesting_mask); +} + +void tt_SocDescriptor::perform_harvesting(std::size_t harvesting_mask) { + coordinate_manager->perform_harvesting(harvesting_mask); +} + +tt_physical_coords tt_SocDescriptor::logical_to_physical_coords(tt_logical_coords logical_coords) { + return coordinate_manager->logical_to_physical_coords(logical_coords); +} + +tt_translated_coords tt_SocDescriptor::logical_to_translated_coords(tt_logical_coords logical_coords) { + return coordinate_manager->logical_to_translated_coords(logical_coords); +} + +tt_logical_coords tt_SocDescriptor::physical_to_logical_coords(tt_physical_coords physical_coords) { + return coordinate_manager->physical_to_logical_coords(physical_coords); +} + +tt_translated_coords tt_SocDescriptor::physical_to_translated_coords(tt_physical_coords physical_coords) { + return coordinate_manager->physical_to_translated_coords(physical_coords); +} + +tt_virtual_coords tt_SocDescriptor::logical_to_virtual_coords(tt_logical_coords logical_coords) { + return coordinate_manager->logical_to_virtual_coords(logical_coords); +} + +tt_logical_coords tt_SocDescriptor::virtual_to_logical_coords(tt_virtual_coords virtual_coords) { + return coordinate_manager->virtual_to_logical_coords(virtual_coords); +} + +tt_SocDescriptor::tt_SocDescriptor(std::string device_descriptor_path, std::size_t harvesting_mask) { std::ifstream fdesc(device_descriptor_path); if (fdesc.fail()) { throw std::runtime_error(fmt::format("Error: device descriptor file {} does not exist!", device_descriptor_path)); @@ -189,6 +221,8 @@ tt_SocDescriptor::tt_SocDescriptor(std::string device_descriptor_path) { arch_name_value = trim(arch_name_value); arch = get_arch_name(arch_name_value); load_soc_features_from_device_descriptor(device_descriptor_yaml); + create_coordinate_manager(harvesting_mask); + perform_harvesting(harvesting_mask); } int tt_SocDescriptor::get_num_dram_channels() const { diff --git a/device/tt_soc_descriptor.h b/device/tt_soc_descriptor.h index a34b8e6e..f136111e 100644 --- a/device/tt_soc_descriptor.h +++ b/device/tt_soc_descriptor.h @@ -16,9 +16,11 @@ #include #include +#include "coordinate_manager.h" #include "tt_xy_pair.h" #include "device/tt_arch_types.h" +#include "device/coordinate_manager.h" #include "fmt/core.h" @@ -102,7 +104,8 @@ struct CoreDescriptor { Should only contain relevant configuration for SOC */ class tt_SocDescriptor { - public: + +public: tt::ARCH arch; tt_xy_pair grid_size; tt_xy_pair physical_grid_size; @@ -123,10 +126,7 @@ class tt_SocDescriptor { std::vector trisc_sizes; // Most of software stack assumes same trisc size for whole chip.. std::string device_descriptor_file_path = std::string(""); bool has(tt_xy_pair input) { return cores.find(input) != cores.end(); } - int overlay_version; - int unpacker_version; int dst_size_alignment; - int packer_version; int worker_l1_size; int eth_l1_size; bool noc_translation_id_enabled; @@ -139,8 +139,9 @@ class tt_SocDescriptor { // Default constructor. Creates uninitialized object with public access to all of its attributes. tt_SocDescriptor() = default; - // Constructor used to build object from device descriptor file. - tt_SocDescriptor(std::string device_descriptor_path); + // Constructor used to build object from device descriptor file. + tt_SocDescriptor(std::string device_descriptor_path, std::size_t harvesting_mask = 0); + // Copy constructor tt_SocDescriptor(const tt_SocDescriptor& other) : arch(other.arch), @@ -162,16 +163,27 @@ class tt_SocDescriptor { ethernet_core_channel_map(other.ethernet_core_channel_map), trisc_sizes(other.trisc_sizes), device_descriptor_file_path(other.device_descriptor_file_path), - overlay_version(other.overlay_version), - unpacker_version(other.unpacker_version), dst_size_alignment(other.dst_size_alignment), - packer_version(other.packer_version), worker_l1_size(other.worker_l1_size), eth_l1_size(other.eth_l1_size), noc_translation_id_enabled(other.noc_translation_id_enabled), - dram_bank_size(other.dram_bank_size) { + dram_bank_size(other.dram_bank_size), + coordinate_manager(other.coordinate_manager) { } - private: + + // Coordinate converters. + tt_physical_coords logical_to_physical_coords(tt_logical_coords logical_coords); + tt_translated_coords logical_to_translated_coords(tt_logical_coords logical_coords); + tt_logical_coords physical_to_logical_coords(tt_physical_coords physical_coords); + tt_translated_coords physical_to_translated_coords(tt_physical_coords physical_coords); + tt_virtual_coords logical_to_virtual_coords(tt_logical_coords logical_coords); + tt_logical_coords virtual_to_logical_coords(tt_virtual_coords virtual_coords); + + void perform_harvesting(std::size_t harvesting_mask); + +private: + CoordinateManager* coordinate_manager = nullptr; + void create_coordinate_manager(std::size_t harvesting_mask); void load_core_descriptors_from_device_descriptor(YAML::Node &device_descriptor_yaml); void load_soc_features_from_device_descriptor(YAML::Node &device_descriptor_yaml); }; diff --git a/device/tt_xy_pair.h b/device/tt_xy_pair.h index f75d9033..052b6130 100644 --- a/device/tt_xy_pair.h +++ b/device/tt_xy_pair.h @@ -12,3 +12,47 @@ using tt_xy_pair = tt::umd::xy_pair; using tt_cxy_pair = tt::umd::cxy_pair; + +struct tt_physical_coords : public tt_xy_pair { + tt_physical_coords() : tt_xy_pair() {} + tt_physical_coords(std::size_t x, std::size_t y) : tt_xy_pair(x, y) {} +}; + +struct tt_chip_physical_coords : public tt_cxy_pair { + tt_chip_physical_coords() : tt_cxy_pair() {} + tt_chip_physical_coords(std::size_t ichip, xy_pair pair) : tt_cxy_pair(ichip, pair) {} + tt_chip_physical_coords(std::size_t ichip, std::size_t x, std::size_t y) : tt_cxy_pair(ichip, x, y) {} +}; + +struct tt_logical_coords : public tt_xy_pair { + tt_logical_coords() : tt_xy_pair() {} + tt_logical_coords(std::size_t x, std::size_t y) : tt_xy_pair(x, y) {} +}; + +struct tt_chip_logical_coords : public tt_cxy_pair { + tt_chip_logical_coords() : tt_cxy_pair() {} + tt_chip_logical_coords(std::size_t ichip, xy_pair pair) : tt_cxy_pair(ichip, pair) {} + tt_chip_logical_coords(std::size_t ichip, std::size_t x, std::size_t y) : tt_cxy_pair(ichip, x, y) {} +}; + +struct tt_virtual_coords : public tt_xy_pair { + tt_virtual_coords() : tt_xy_pair() {} + tt_virtual_coords(std::size_t x, std::size_t y) : tt_xy_pair(x, y) {} +}; + +struct tt_chip_virtual_coords : public tt_cxy_pair { + tt_chip_virtual_coords() : tt_cxy_pair() {} + tt_chip_virtual_coords(std::size_t ichip, xy_pair pair) : tt_cxy_pair(ichip, pair) {} + tt_chip_virtual_coords(std::size_t ichip, std::size_t x, std::size_t y) : tt_cxy_pair(ichip, x, y) {} +}; + +struct tt_translated_coords : public tt_xy_pair { + tt_translated_coords() : tt_xy_pair() {} + tt_translated_coords(std::size_t x, std::size_t y) : tt_xy_pair(x, y) {} +}; + +struct tt_chip_translated_coords : public tt_cxy_pair { + tt_chip_translated_coords() : tt_cxy_pair() {} + tt_chip_translated_coords(std::size_t ichip, xy_pair pair) : tt_cxy_pair(ichip, pair) {} + tt_chip_translated_coords(std::size_t ichip, std::size_t x, std::size_t y) : tt_cxy_pair(ichip, x, y) {} +}; \ No newline at end of file diff --git a/device/wormhole/wormhole_coordinate_manager.cpp b/device/wormhole/wormhole_coordinate_manager.cpp new file mode 100644 index 00000000..016e3a6b --- /dev/null +++ b/device/wormhole/wormhole_coordinate_manager.cpp @@ -0,0 +1,14 @@ +#include "wormhole_coordinate_manager.h" + +std::set WormholeCoordinateManager::get_y_coordinates_to_harvest(std::size_t harvesting_mask) { + std::set y_to_harvest; + std::size_t logical_y = 0; + while (harvesting_mask > 0) { + if (harvesting_mask & 1) { + y_to_harvest.insert(logical_y); + } + logical_y++; + harvesting_mask >>= 1; + } + return y_to_harvest; +} diff --git a/device/wormhole/wormhole_coordinate_manager.h b/device/wormhole/wormhole_coordinate_manager.h new file mode 100644 index 00000000..a9c30b6e --- /dev/null +++ b/device/wormhole/wormhole_coordinate_manager.h @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: (c) 2023 Tenstorrent Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "device/coordinate_manager.h" + +class WormholeCoordinateManager : public CoordinateManager { + +public: + WormholeCoordinateManager(const tt_xy_pair& worker_grid_size, const std::vector& workers, std::size_t harvesting_mask) + : CoordinateManager(worker_grid_size, workers, harvesting_mask) {} + +protected: + std::set get_y_coordinates_to_harvest(std::size_t harvesting_mask) override; +}; \ No newline at end of file diff --git a/tests/api/CMakeLists.txt b/tests/api/CMakeLists.txt index 08c11ffe..7c51f7f9 100644 --- a/tests/api/CMakeLists.txt +++ b/tests/api/CMakeLists.txt @@ -1,6 +1,7 @@ set(API_TESTS_SRCS test_cluster.cpp test_cluster_descriptor.cpp + test_soc_descriptor.cpp ) add_executable(api_tests ${API_TESTS_SRCS}) diff --git a/tests/api/test_soc_descriptor.cpp b/tests/api/test_soc_descriptor.cpp new file mode 100644 index 00000000..da1768f8 --- /dev/null +++ b/tests/api/test_soc_descriptor.cpp @@ -0,0 +1,305 @@ +#include +#include +#include + +#include "gtest/gtest.h" +#include "tt_device.h" +#include "eth_l1_address_map.h" +#include "l1_address_map.h" +#include "eth_l1_address_map.h" +#include "eth_interface.h" +#include "host_mem_address_map.h" + +#include "device/tt_cluster_descriptor.h" +#include "device/wormhole/wormhole_implementation.h" +#include "tests/test_utils/generate_cluster_desc.hpp" +#include "tests/test_utils/device_test_utils.hpp" + +// Wormhole workers - x-y annotation +// functional_workers: +// [ +// 1-1, 2-1, 3-1, 4-1, 6-1, 7-1, 8-1, 9-1, +// 1-2, 2-2, 3-2, 4-2, 6-2, 7-2, 8-2, 9-2, +// 1-3, 2-3, 3-3, 4-3, 6-3, 7-3, 8-3, 9-3, +// 1-4, 2-4, 3-4, 4-4, 6-4, 7-4, 8-4, 9-4, +// 1-5, 2-5, 3-5, 4-5, 6-5, 7-5, 8-5, 9-5, +// 1-7, 2-7, 3-7, 4-7, 6-7, 7-7, 8-7, 9-7, +// 1-8, 2-8, 3-8, 4-8, 6-8, 7-8, 8-8, 9-8, +// 1-9, 2-9, 3-9, 4-9, 6-9, 7-9, 8-9, 9-9, +// 1-10, 2-10, 3-10, 4-10, 6-10, 7-10, 8-10, 9-10, +// 1-11, 2-11, 3-11, 4-11, 6-11, 7-11, 8-11, 9-11, +// ] + +// Blackhole workers - x-y annotation +// functional_workers: +// [ +// 1-2, 2-2, 3-2, 4-2, 5-2, 6-2, 7-2, 10-2, 11-2, 12-2, 13-2, 14-2, 15-2, 16-2, +// 1-3, 2-3, 3-3, 4-3, 5-3, 6-3, 7-3, 10-3, 11-3, 12-3, 13-3, 14-3, 15-3, 16-3, +// 1-4, 2-4, 3-4, 4-4, 5-4, 6-4, 7-4, 10-4, 11-4, 12-4, 13-4, 14-4, 15-4, 16-4, +// 1-5, 2-5, 3-5, 4-5, 5-5, 6-5, 7-5, 10-5, 11-5, 12-5, 13-5, 14-5, 15-5, 16-5, +// 1-6, 2-6, 3-6, 4-6, 5-6, 6-6, 7-6, 10-6, 11-6, 12-6, 13-6, 14-6, 15-6, 16-6, +// 1-7, 2-7, 3-7, 4-7, 5-7, 6-7, 7-7, 10-7, 11-7, 12-7, 13-7, 14-7, 15-7, 16-7, +// 1-8, 2-8, 3-8, 4-8, 5-8, 6-8, 7-8, 10-8, 11-8, 12-8, 13-8, 14-8, 15-8, 16-8, +// 1-9, 2-9, 3-9, 4-9, 5-9, 6-9, 7-9, 10-9, 11-9, 12-9, 13-9, 14-9, 15-9, 16-9, +// 1-10, 2-10, 3-10, 4-10, 5-10, 6-10, 7-10, 10-10, 11-10, 12-10, 13-10, 14-10, 15-10, 16-10, +// 1-11, 2-11, 3-11, 4-11, 5-11, 6-11, 7-11, 10-11, 11-11, 12-11, 13-11, 14-11, 15-11, 16-11, +// ] + +std::size_t get_num_harvested(std::size_t harvesting_mask) { + std::size_t num_harvested = 0; + while (harvesting_mask > 0) { + if (harvesting_mask & 1) { + num_harvested++; + } + harvesting_mask >>= 1; + } + return num_harvested; +} + +// Tests that all physical coordinates are same as all virtual coordinates +// when there is no harvesting. +TEST(SocDescriptor, SocDescriptorWHNoHarvesting) { + + const std::size_t harvesting_mask = 0; + + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/wormhole_b0_8x10.yaml"), harvesting_mask); + + // We expect full grid size since there is no harvesting. + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + for (size_t x = 0; x < worker_grid_size.x; x++) { + for (size_t y = 0; y < worker_grid_size.y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_virtual_coords virtual_coords = soc_desc.logical_to_virtual_coords(logical_coords); + tt_physical_coords physical_coords = soc_desc.logical_to_physical_coords(logical_coords); + + // Virtual and physical coordinates should be the same. + EXPECT_EQ(physical_coords, virtual_coords); + } + } +} + +// Tests that all physical coordinates are same as all virtual coordinates +// when there is no harvesting. +TEST(SocDescriptor, SocDescriptorBHNoHarvesting) { + + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), 0); + + // We expect full grid size since there is no harvesting. + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + for (size_t x = 0; x < worker_grid_size.x; x++) { + for (size_t y = 0; y < worker_grid_size.y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_virtual_coords virtual_coords = soc_desc.logical_to_virtual_coords(logical_coords); + tt_physical_coords physical_coords = soc_desc.logical_to_physical_coords(logical_coords); + + // Virtual and physical coordinates should be the same. + EXPECT_EQ(physical_coords, virtual_coords); + } + } +} + +// Test basic translation to virtual and physical noc coordinates. +// We expect that the top left core will have virtual and physical coordinates (1, 1) and (1, 2) for +// the logical coordinates if the first row is harvested. +TEST(SocDescriptor, SocDescriptorWHTopLeftCore) { + + const std::size_t harvesting_mask = 1; + + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/wormhole_b0_8x10.yaml"), harvesting_mask); + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + tt_logical_coords logical_coords = tt_logical_coords(0, 0); + + // Always expect same virtual coordinate for (0, 0) logical coordinate. + tt_virtual_coords virtual_cords = soc_desc.logical_to_virtual_coords(logical_coords); + EXPECT_EQ(virtual_cords, tt_virtual_coords(1, 1)); + + // This depends on harvesting mask. So expected physical coord is specific to this test and Wormhole arch. + tt_physical_coords physical_cords = soc_desc.logical_to_physical_coords(logical_coords); + EXPECT_EQ(physical_cords, tt_physical_coords(1, 2)); +} + +// Test basic translation to virtual and physical noc coordinates. +// We expect that the top left core will have virtual and physical coordinates (1, 2) and (2, 2) for +// the logical coordinates if the first row is harvested. +TEST(SocDescriptor, SocDescriptorBHTopLeftCore) { + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), 1); + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + tt_logical_coords logical_coords = tt_logical_coords(0, 0); + + // Always expect same virtual coordinate for (0, 0) logical coordinate. + tt_virtual_coords virtual_cords = soc_desc.logical_to_virtual_coords(logical_coords); + EXPECT_EQ(virtual_cords, tt_virtual_coords(1, 2)); + + // This depends on harvesting mask. So expected physical coord is specific to this test and Blackhole arch. + tt_physical_coords physical_cords = soc_desc.logical_to_physical_coords(logical_coords); + EXPECT_EQ(physical_cords, tt_physical_coords(2, 2)); +} + +// Test logical to physical coordinate translation. +// For the full grid of logical coordinates we expect that there are no duplicates of physical coordinates. +// For the reverse mapping back of physical to logical coordinates we expect that same logical coordinates are returned as from original mapping. +TEST(SocDescriptor, SocDescriptorWHLogicalPhysicalMapping) { + + const std::size_t max_num_harvested_y = 10; + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/wormhole_b0_8x10.yaml")); + for (std::size_t harvesting_mask = 0; harvesting_mask < (1 << max_num_harvested_y); harvesting_mask++) { + + soc_desc.perform_harvesting(harvesting_mask); + + std::map logical_to_physical; + std::set physical_coords_set; + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + std::size_t num_harvested_y = get_num_harvested(harvesting_mask); + + for (size_t x = 0; x < worker_grid_size.x; x++) { + for (size_t y = 0; y < worker_grid_size.y - num_harvested_y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_physical_coords physical_coords = soc_desc.logical_to_physical_coords(logical_coords); + logical_to_physical[logical_coords] = physical_coords; + + // Expect that logical to physical translation is 1-1 mapping. No duplicates for physical coordinates. + EXPECT_EQ(physical_coords_set.count(physical_coords), 0); + physical_coords_set.insert(physical_coords); + } + } + + // Expect that the number of physical coordinates is equal to the number of workers minus the number of harvested rows. + EXPECT_EQ(physical_coords_set.size(), worker_grid_size.x * (worker_grid_size.y - num_harvested_y)); + + for (auto it : logical_to_physical) { + tt_physical_coords physical_coords = it.second; + tt_logical_coords logical_coords = soc_desc.physical_to_logical_coords(physical_coords); + + // Expect that reverse mapping of physical coordinates gives the same logical coordinates + // using which we got the physical coordinates. + EXPECT_EQ(it.first, logical_coords); + } + } +} + +// Test logical to physical coordinate translation. +// For the full grid of logical coordinates we expect that there are no duplicates of physical coordinates. +// For the reverse mapping back of physical to logical coordinates we expect that same logical coordinates are returned as from original mapping. +TEST(SocDescriptor, SocDescriptorBHLogicalPhysicalMapping) { + + const std::size_t max_num_harvested_x = 14; + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch.yaml")); + for (std::size_t harvesting_mask = 0; harvesting_mask < (1 << max_num_harvested_x); harvesting_mask++) { + + soc_desc.perform_harvesting(harvesting_mask); + + std::map logical_to_physical; + std::set physical_coords_set; + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + std::size_t num_harvested_x = get_num_harvested(harvesting_mask); + + for (size_t x = 0; x < worker_grid_size.x - num_harvested_x; x++) { + for (size_t y = 0; y < worker_grid_size.y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_physical_coords physical_coords = soc_desc.logical_to_physical_coords(logical_coords); + logical_to_physical[logical_coords] = physical_coords; + + // Expect that logical to physical translation is 1-1 mapping. No duplicates for physical coordinates. + EXPECT_EQ(physical_coords_set.count(physical_coords), 0); + physical_coords_set.insert(physical_coords); + } + } + + EXPECT_EQ(physical_coords_set.size(), worker_grid_size.y * (worker_grid_size.x - num_harvested_x)); + + for (auto it : logical_to_physical) { + tt_physical_coords physical_coords = it.second; + tt_logical_coords logical_coords = soc_desc.physical_to_logical_coords(physical_coords); + + // Expect that reverse mapping of physical coordinates gives the same logical coordinates + // using which we got the physical coordinates. + EXPECT_EQ(it.first, logical_coords); + } + } +} + +// Test logical to virtual coordinate translation. +// For the full grid of logical coordinates we expect that there are no duplicates of virtual coordinates. +// For the reverse mapping back of virtual to logical coordinates we expect that same logical coordinates are returned as from original mapping. +TEST(SocDescriptor, SocDescriptorWHLogicalVirtualMapping) { + + const std::size_t max_num_harvested_y = 10; + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/wormhole_b0_8x10.yaml")); + for (std::size_t harvesting_mask = 0; harvesting_mask < (1 << max_num_harvested_y); harvesting_mask++) { + + soc_desc.perform_harvesting(harvesting_mask); + + std::map logical_to_virtual; + std::set virtual_coords_set; + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + std::size_t num_harvested_y = get_num_harvested(harvesting_mask); + + for (size_t x = 0; x < worker_grid_size.x; x++) { + for (size_t y = 0; y < worker_grid_size.y - num_harvested_y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_virtual_coords virtual_coords = soc_desc.logical_to_virtual_coords(logical_coords); + logical_to_virtual[logical_coords] = virtual_coords; + + // Expect that logical to virtual translation is 1-1 mapping. No duplicates for virtual coordinates. + EXPECT_EQ(virtual_coords_set.count(virtual_coords), 0); + virtual_coords_set.insert(virtual_coords); + } + } + + for (auto it : logical_to_virtual) { + tt_virtual_coords virtual_coords = it.second; + tt_logical_coords logical_coords = soc_desc.virtual_to_logical_coords(virtual_coords); + + // Expect that reverse mapping of virtual coordinates gives the same logical coordinates + // using which we got the virtual coordinates. + EXPECT_EQ(it.first, logical_coords); + } + } +} + +// Test logical to virtual coordinate translation. +// For the full grid of logical coordinates we expect that there are no duplicates of virtual coordinates. +// For the reverse mapping back of virtual to logical coordinates we expect that same logical coordinates are returned as from original mapping. +TEST(SocDescriptor, SocDescriptorBHLogicalVirtualMapping) { + + const std::size_t max_num_harvested_x = 14; + tt_SocDescriptor soc_desc = tt_SocDescriptor(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch.yaml")); + for (std::size_t harvesting_mask = 0; harvesting_mask < (1 << max_num_harvested_x); harvesting_mask++) { + + soc_desc.perform_harvesting(harvesting_mask); + + std::map logical_to_virtual; + std::set virtual_coords_set; + tt_xy_pair worker_grid_size = soc_desc.worker_grid_size; + + std::size_t num_harvested_x = get_num_harvested(harvesting_mask); + + for (size_t x = 0; x < worker_grid_size.x - num_harvested_x; x++) { + for (size_t y = 0; y < worker_grid_size.y; y++) { + tt_logical_coords logical_coords = tt_logical_coords(x, y); + tt_virtual_coords virtual_coords = soc_desc.logical_to_virtual_coords(logical_coords); + logical_to_virtual[logical_coords] = virtual_coords; + + // Expect that logical to virtual translation is 1-1 mapping. No duplicates for virtual coordinates. + EXPECT_EQ(virtual_coords_set.count(virtual_coords), 0); + virtual_coords_set.insert(virtual_coords); + } + } + + EXPECT_EQ(virtual_coords_set.size(), worker_grid_size.y * (worker_grid_size.x - num_harvested_x)); + + for (auto it : logical_to_virtual) { + tt_virtual_coords virtual_coords = it.second; + tt_logical_coords logical_coords = soc_desc.virtual_to_logical_coords(virtual_coords); + + // Expect that reverse mapping of virtual coordinates gives the same logical coordinates + // using which we got the virtual coordinates. + EXPECT_EQ(it.first, logical_coords); + } + } +}