barrier params

tenstorrent · Dec 13, 2024 · 6e2d0bb · 6e2d0bb
1 parent 62d37ac
commit 6e2d0bb
Show file tree

Hide file tree

Showing 19 changed files with 101 additions and 19 deletions.
diff --git a/device/api/umd/device/architecture_implementation.h b/device/api/umd/device/architecture_implementation.h
@@ -73,7 +73,6 @@ class architecture_implementation {
         std::uint32_t tlb_index, const tlb_data& data) const = 0;
 
     virtual tt_device_l1_address_params get_l1_address_params() const = 0;
-    virtual tt_device_dram_address_params get_dram_address_params() const = 0;
     virtual tt_driver_host_address_params get_host_address_params() const = 0;
     virtual tt_driver_eth_interface_params get_eth_interface_params() const = 0;
     virtual tt_driver_noc_params get_noc_params() const = 0;

diff --git a/device/api/umd/device/blackhole_implementation.h b/device/api/umd/device/blackhole_implementation.h
@@ -310,7 +310,6 @@ class blackhole_implementation : public architecture_implementation {
     std::pair<std::uint64_t, std::uint64_t> get_tlb_data(std::uint32_t tlb_index, const tlb_data& data) const override;
 
     tt_device_l1_address_params get_l1_address_params() const override;
-    tt_device_dram_address_params get_dram_address_params() const override;
     tt_driver_host_address_params get_host_address_params() const override;
     tt_driver_eth_interface_params get_eth_interface_params() const override;
     tt_driver_noc_params get_noc_params() const override;

diff --git a/device/api/umd/device/cluster.h b/device/api/umd/device/cluster.h
@@ -46,6 +46,9 @@ class tt_device {
     virtual ~tt_device(){};
 
     // Setup/Teardown Functions
+    virtual void set_barrier_address_params(const barrier_address_params& barrier_address_params_) {
+        throw std::runtime_error("---- tt_device::set_device_l1_address_params is not implemented\n");
+    }
 
     /**
      * Configure a TLB to point to a specific core and an address within that core. Should be done for Static TLBs.
@@ -529,6 +532,7 @@ class Cluster : public tt_device {
     static std::unique_ptr<Cluster> create_mock_cluster();
 
     // Setup/Teardown Functions
+    virtual void set_barrier_address_params(const barrier_address_params& barrier_address_params_);
     virtual void configure_tlb(
         chip_id_t logical_device_id,
         tt_xy_pair core,

diff --git a/device/api/umd/device/grayskull_implementation.h b/device/api/umd/device/grayskull_implementation.h
@@ -300,7 +300,6 @@ class grayskull_implementation : public architecture_implementation {
     std::pair<std::uint64_t, std::uint64_t> get_tlb_data(std::uint32_t tlb_index, const tlb_data& data) const override;
 
     tt_device_l1_address_params get_l1_address_params() const override;
-    tt_device_dram_address_params get_dram_address_params() const override;
     tt_driver_host_address_params get_host_address_params() const override;
     tt_driver_eth_interface_params get_eth_interface_params() const override;
     tt_driver_noc_params get_noc_params() const override;

diff --git a/device/api/umd/device/tt_simulation_device.h b/device/api/umd/device/tt_simulation_device.h
@@ -20,6 +20,7 @@ class tt_SimulationDevice : public tt_device {
 
     tt_SimulationHost host;
 
+    virtual void set_barrier_address_params(const barrier_address_params& barrier_address_params_);
     virtual void start_device(const tt_device_params& device_params);
     virtual void assert_risc_reset();
     virtual void deassert_risc_reset();

diff --git a/device/api/umd/device/types/cluster_types.h b/device/api/umd/device/types/cluster_types.h
@@ -35,6 +35,12 @@ inline std::ostream& operator<<(std::ostream& os, const tt_DevicePowerState powe
     return os;
 }
 
+struct barrier_address_params {
+    std::uint32_t tensix_l1_barrier_base = 0;
+    std::uint32_t eth_l1_barrier_base = 0;
+    std::uint32_t dram_barrier_base = 0;
+};
+
 struct tt_device_dram_address_params {
     std::uint32_t DRAM_BARRIER_BASE = 0;
 };

diff --git a/device/api/umd/device/wormhole_implementation.h b/device/api/umd/device/wormhole_implementation.h
@@ -334,7 +334,6 @@ class wormhole_implementation : public architecture_implementation {
     std::pair<std::uint64_t, std::uint64_t> get_tlb_data(std::uint32_t tlb_index, const tlb_data& data) const override;
 
     tt_device_l1_address_params get_l1_address_params() const override;
-    tt_device_dram_address_params get_dram_address_params() const override;
     tt_driver_host_address_params get_host_address_params() const override;
     tt_driver_eth_interface_params get_eth_interface_params() const override;
     tt_driver_noc_params get_noc_params() const override;

diff --git a/device/blackhole/blackhole_implementation.cpp b/device/blackhole/blackhole_implementation.cpp
@@ -13,8 +13,6 @@
 constexpr std::uint32_t NOC_ADDR_LOCAL_BITS = 36;   // source: noc_parameters.h, common for WH && BH
 constexpr std::uint32_t NOC_ADDR_NODE_ID_BITS = 6;  // source: noc_parameters.h, common for WH && BH
 
-constexpr std::uint32_t DRAM_BARRIER_BASE = 0;
-
 namespace tt::umd {
 
 std::tuple<xy_pair, xy_pair> blackhole_implementation::multicast_workaround(xy_pair start, xy_pair end) const {
@@ -83,14 +81,14 @@ std::pair<std::uint64_t, std::uint64_t> blackhole_implementation::get_tlb_data(
 }
 
 tt_device_l1_address_params blackhole_implementation::get_l1_address_params() const {
+    // L1 barrier base and erisc barrier base should be explicitly set by the client.
+    // Setting some default values here, but it should be ultimately overridden by the client.
     return {
         ::l1_mem::address_map::L1_BARRIER_BASE,
         ::eth_l1_mem::address_map::ERISC_BARRIER_BASE,
         ::eth_l1_mem::address_map::FW_VERSION_ADDR};
 }
 
-tt_device_dram_address_params blackhole_implementation::get_dram_address_params() const { return {DRAM_BARRIER_BASE}; }
-
 tt_driver_host_address_params blackhole_implementation::get_host_address_params() const {
     return {
         ::blackhole::host_mem::address_map::ETH_ROUTING_BLOCK_SIZE,

diff --git a/device/cluster.cpp b/device/cluster.cpp
@@ -475,9 +475,6 @@ void Cluster::construct_cluster(
     // Default initialize l1_address_params based on detected arch
     l1_address_params = architecture_implementation->get_l1_address_params();
 
-    // Default initialize dram_address_params based on detected arch
-    dram_address_params = architecture_implementation->get_dram_address_params();
-
     // Default initialize host_address_params based on detected arch
     host_address_params = architecture_implementation->get_host_address_params();
 
@@ -3369,4 +3366,10 @@ tt_version Cluster::get_ethernet_fw_version() const {
     return eth_fw_version;
 }
 
+void Cluster::set_barrier_address_params(const barrier_address_params& barrier_address_params_) {
+    l1_address_params.tensix_l1_barrier_base = barrier_address_params_.tensix_l1_barrier_base;
+    l1_address_params.eth_l1_barrier_base = barrier_address_params_.eth_l1_barrier_base;
+    dram_address_params.DRAM_BARRIER_BASE = barrier_address_params_.dram_barrier_base;
+}
+
 }  // namespace tt::umd
diff --git a/device/grayskull/grayskull_implementation.cpp b/device/grayskull/grayskull_implementation.cpp
@@ -12,8 +12,6 @@
 constexpr std::uint32_t NOC_ADDR_LOCAL_BITS = 32;   // source: noc_parameters.h, unique for GS
 constexpr std::uint32_t NOC_ADDR_NODE_ID_BITS = 6;  // source: noc_parameters.h, common for GS && WH && BH
 
-constexpr std::uint32_t DRAM_BARRIER_BASE = 0;
-
 namespace tt::umd {
 
 std::tuple<xy_pair, xy_pair> grayskull_implementation::multicast_workaround(xy_pair start, xy_pair end) const {
@@ -92,11 +90,12 @@ std::pair<std::uint64_t, std::uint64_t> grayskull_implementation::get_tlb_data(
 }
 
 tt_device_l1_address_params grayskull_implementation::get_l1_address_params() const {
+    // L1 barrier base should be explicitly set by the client.
+    // Setting some default value here, but it should be ultimately overridden by the client.
+    // Grayskull doesn't have ethernet cores, so no eth params are set here.
     return {::l1_mem::address_map::L1_BARRIER_BASE, 0, 0};
 }
 
-tt_device_dram_address_params grayskull_implementation::get_dram_address_params() const { return {DRAM_BARRIER_BASE}; }
-
 tt_driver_host_address_params grayskull_implementation::get_host_address_params() const {
     return {
         ::grayskull::host_mem::address_map::ETH_ROUTING_BLOCK_SIZE,

diff --git a/device/mockup/tt_mockup_device.hpp b/device/mockup/tt_mockup_device.hpp
@@ -21,6 +21,8 @@ class tt_MockupDevice : public tt_device {
 
     virtual ~tt_MockupDevice() {}
 
+    void set_barrier_address_params(const barrier_address_params& barrier_address_params_) override {}
+
     void start_device(const tt_device_params& device_params) override {}
 
     void assert_risc_reset() override {}

diff --git a/device/simulation/tt_simulation_device.cpp b/device/simulation/tt_simulation_device.cpp
@@ -79,6 +79,8 @@ tt_SimulationDevice::tt_SimulationDevice(const std::string& sdesc_path) : tt_dev
 
 tt_SimulationDevice::~tt_SimulationDevice() { close_device(); }
 
+void tt_SimulationDevice::set_barrier_address_params(const barrier_address_params& barrier_address_params_) {}
+
 void tt_SimulationDevice::start_device(const tt_device_params& device_params) {
     void* buf_ptr = nullptr;
 

diff --git a/device/wormhole/wormhole_implementation.cpp b/device/wormhole/wormhole_implementation.cpp
@@ -13,8 +13,6 @@
 constexpr std::uint32_t NOC_ADDR_LOCAL_BITS = 36;   // source: noc_parameters.h, common for WH && BH
 constexpr std::uint32_t NOC_ADDR_NODE_ID_BITS = 6;  // source: noc_parameters.h, common for WH && BH
 
-constexpr std::uint32_t DRAM_BARRIER_BASE = 0;
-
 namespace tt::umd {
 
 std::tuple<xy_pair, xy_pair> wormhole_implementation::multicast_workaround(xy_pair start, xy_pair end) const {
@@ -101,14 +99,14 @@ std::pair<std::uint64_t, std::uint64_t> wormhole_implementation::get_tlb_data(
 }
 
 tt_device_l1_address_params wormhole_implementation::get_l1_address_params() const {
+    // L1 barrier base and erisc barrier base should be explicitly set by the client.
+    // Setting some default values here, but it should be ultimately overridden by the client.
     return {
         ::l1_mem::address_map::L1_BARRIER_BASE,
         ::eth_l1_mem::address_map::ERISC_BARRIER_BASE,
         ::eth_l1_mem::address_map::FW_VERSION_ADDR};
 }
 
-tt_device_dram_address_params wormhole_implementation::get_dram_address_params() const { return {DRAM_BARRIER_BASE}; }
-
 tt_driver_host_address_params wormhole_implementation::get_host_address_params() const {
     return {
         ::wormhole::host_mem::address_map::ETH_ROUTING_BLOCK_SIZE,

diff --git a/tests/blackhole/test_bh_common.h b/tests/blackhole/test_bh_common.h
@@ -11,8 +11,16 @@
 
 using namespace tt::umd;
 
+constexpr std::uint32_t DRAM_BARRIER_BASE = 0;
+
 namespace tt::umd::test::utils {
 
+static void set_params_for_remote_txn(Cluster& device) {
+    // Populate address map and NOC parameters that the driver needs for remote transactions
+    device.set_barrier_address_params(
+        {l1_mem::address_map::L1_BARRIER_BASE, eth_l1_mem::address_map::ERISC_BARRIER_BASE, DRAM_BARRIER_BASE});
+}
+
 class BlackholeTestFixture : public ::testing::Test {
 protected:
     // You can remove any or all of the following functions if their bodies would
@@ -50,6 +58,8 @@ class BlackholeTestFixture : public ::testing::Test {
         assert(device != nullptr);
         assert(device->get_cluster_description()->get_number_of_chips() == get_detected_num_chips());
 
+        set_params_for_remote_txn(*device);
+
         tt_device_params default_params;
         device->start_device(default_params);
 

diff --git a/tests/blackhole/test_silicon_driver_bh.cpp b/tests/blackhole/test_silicon_driver_bh.cpp
@@ -18,6 +18,14 @@
 
 using namespace tt::umd;
 
+constexpr std::uint32_t DRAM_BARRIER_BASE = 0;
+
+static void set_params_for_remote_txn(Cluster& device) {
+    // Populate address map and NOC parameters that the driver needs for remote transactions
+    device.set_barrier_address_params(
+        {l1_mem::address_map::L1_BARRIER_BASE, eth_l1_mem::address_map::ERISC_BARRIER_BASE, DRAM_BARRIER_BASE});
+}
+
 std::int32_t get_static_tlb_index(tt_xy_pair target) {
     bool is_eth_location =
         std::find(std::begin(tt::umd::blackhole::ETH_LOCATIONS), std::end(tt::umd::blackhole::ETH_LOCATIONS), target) !=
@@ -84,6 +92,7 @@ TEST(SiliconDriverBH, CreateDestroy) {
             false,
             true,
             false);
+        set_params_for_remote_txn(device);
         device.start_device(default_params);
         device.deassert_risc_reset();
         device.close_device();
@@ -181,6 +190,7 @@ TEST(SiliconDriverBH, CreateDestroy) {
 //         true,
 //         true,
 //         simulated_harvesting_masks);
+//     set_params_for_remote_txn(device);
 //     auto mmio_devices = device.get_target_mmio_device_ids();
 
 //     for (int i = 0; i < target_devices.size(); i++) {
@@ -270,6 +280,7 @@ TEST(SiliconDriverBH, UnalignedStaticTLB_RW) {
     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
+    set_params_for_remote_txn(device);
     auto mmio_devices = device.get_target_mmio_device_ids();
 
     for (int i = 0; i < target_devices.size(); i++) {
@@ -326,6 +337,7 @@ TEST(SiliconDriverBH, StaticTLB_RW) {
     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
+    set_params_for_remote_txn(device);
     auto mmio_devices = device.get_target_mmio_device_ids();
 
     for (int i = 0; i < target_devices.size(); i++) {
@@ -390,6 +402,8 @@ TEST(SiliconDriverBH, DynamicTLB_RW) {
     uint32_t num_host_mem_ch_per_mmio_device = 1;
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
 
+    set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     // MT: Don't deassert risc resets since there's no loaded FW
@@ -478,6 +492,8 @@ TEST(SiliconDriverBH, MultiThreadedDevice) {
     uint32_t num_host_mem_ch_per_mmio_device = 1;
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
 
+    set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     device.deassert_risc_reset();
@@ -546,6 +562,7 @@ TEST(SiliconDriverBH, MultiThreadedMemBar) {
     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
+    set_params_for_remote_txn(device);
     for (int i = 0; i < target_devices.size(); i++) {
         // Iterate over devices and only setup static TLBs for functional worker cores
         auto& sdesc = device.get_soc_descriptor(i);
@@ -685,6 +702,7 @@ TEST(SiliconDriverBH, DISABLED_BroadcastWrite) {  // Cannot broadcast to tensix/
     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
+    set_params_for_remote_txn(device);
     auto mmio_devices = device.get_target_mmio_device_ids();
 
     tt_device_params default_params;
@@ -772,6 +790,7 @@ TEST(SiliconDriverBH, DISABLED_VirtualCoordinateBroadcast) {  // same problem as
     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
     Cluster device = Cluster(num_host_mem_ch_per_mmio_device, false, true, true);
+    set_params_for_remote_txn(device);
     auto mmio_devices = device.get_target_mmio_device_ids();
 
     tt_device_params default_params;

diff --git a/tests/galaxy/test_umd_concurrent_threads.cpp b/tests/galaxy/test_umd_concurrent_threads.cpp
@@ -53,6 +53,8 @@ TEST(GalaxyConcurrentThreads, WriteToAllChipsL1) {
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), all_devices, num_host_mem_ch_per_mmio_device, false, true);
     const auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
 
@@ -148,6 +150,8 @@ TEST(GalaxyConcurrentThreads, WriteToAllChipsDram) {
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), all_devices, num_host_mem_ch_per_mmio_device, false, true);
     const auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
 
@@ -232,6 +236,8 @@ TEST(GalaxyConcurrentThreads, PushInputsWhileSignalingCluster) {
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), target_devices, num_host_mem_ch_per_mmio_device, false, true);
     const auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     device.deassert_risc_reset();

diff --git a/tests/galaxy/test_umd_remote_api.cpp b/tests/galaxy/test_umd_remote_api.cpp
@@ -33,6 +33,8 @@ void run_remote_read_write_test(uint32_t vector_size, bool dram_write) {
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), target_devices, num_host_mem_ch_per_mmio_device, false, true);
     const auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     device.deassert_risc_reset();
@@ -148,6 +150,8 @@ void run_data_mover_test(
     Cluster device =
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), target_devices, num_host_mem_ch_per_mmio_device, false, true);
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     device.deassert_risc_reset();
@@ -265,6 +269,8 @@ void run_data_broadcast_test(
     Cluster device =
         Cluster(test_utils::GetAbsPath(SOC_DESC_PATH), target_devices, num_host_mem_ch_per_mmio_device, false, true);
 
+    tt::umd::test::utils::set_params_for_remote_txn(device);
+
     tt_device_params default_params;
     device.start_device(default_params);
     device.deassert_risc_reset();