Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#3764: Device side opID support #9830

Closed
wants to merge 2 commits into from
Closed

#3764: Device side opID support #9830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83d596e
#10107: Fix hangs w/ launch_msg size >32bytes
pgkeller Jul 10, 2024
374d29e
#3764: Device side opID support
tt-aho Jun 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 21 additions & 22 deletions tests/scripts/run_profiler_regressions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,6 @@ source scripts/tools_setup_common.sh

set -eo pipefail

run_additional_T3000_test(){
remove_default_log_locations
mkdir -p $PROFILER_ARTIFACTS_DIR

./tt_metal/tools/profiler/profile_this.py -c "pytest tests/tt_eager/python_api_testing/unit_testing/misc/test_all_gather.py::test_all_gather_on_t3000_post_commit[mem_config0-input_dtype0-8-1-input_shape1-0-layout1]" > $PROFILER_ARTIFACTS_DIR/test_out.log

cat $PROFILER_ARTIFACTS_DIR/test_out.log

if cat $PROFILER_ARTIFACTS_DIR/test_out.log | grep "SKIPPED"
then
echo "No verification as test was skipped"
else
echo "Verifying test results"
runDate=$(ls $PROFILER_OUTPUT_DIR/)
LINE_COUNT=9 #1 header + 8 devices
res=$(verify_perf_line_count "$PROFILER_OUTPUT_DIR/$runDate/ops_perf_results_$runDate.csv" "$LINE_COUNT")
echo $res
fi
}

run_async_mode_T3000_test(){
#Some tests here do not skip grayskull
if [ "$ARCH_NAME" != "grayskull" ]; then
Expand All @@ -47,6 +27,27 @@ run_async_mode_T3000_test(){
fi
}

run_additional_T3000_test(){
remove_default_log_locations
mkdir -p $PROFILER_ARTIFACTS_DIR

./tt_metal/tools/profiler/profile_this.py -c "pytest tests/tt_eager/python_api_testing/unit_testing/misc/test_all_gather.py::test_all_gather_on_t3000_post_commit[mem_config0-input_dtype0-8-1-input_shape1-0-layout1]" > $PROFILER_ARTIFACTS_DIR/test_out.log

cat $PROFILER_ARTIFACTS_DIR/test_out.log

if cat $PROFILER_ARTIFACTS_DIR/test_out.log | grep "SKIPPED"
then
echo "No verification as test was skipped"
else
echo "Verifying test results"
runDate=$(ls $PROFILER_OUTPUT_DIR/)
LINE_COUNT=9 #1 header + 8 devices
res=$(verify_perf_line_count "$PROFILER_OUTPUT_DIR/$runDate/ops_perf_results_$runDate.csv" "$LINE_COUNT")
echo $res
run_async_mode_T3000_test
fi
}

run_profiling_test(){
if [[ -z "$ARCH_NAME" ]]; then
echo "Must provide ARCH_NAME in environment" 1>&2
Expand All @@ -60,8 +61,6 @@ run_profiling_test(){

run_additional_T3000_test

run_async_mode_T3000_test

TT_METAL_DEVICE_PROFILER=1 pytest $PROFILER_TEST_SCRIPTS_ROOT/test_device_profiler.py

remove_default_log_locations
Expand Down
3 changes: 2 additions & 1 deletion tests/tt_metal/tt_metal/test_kernels/dataflow/dram_copy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ void kernel_main() {
#else
tt_l1_ptr mailboxes_t* const mailboxes = (tt_l1_ptr mailboxes_t*)(MEM_MAILBOX_BASE);
#endif
uint64_t dispatch_addr = NOC_XY_ADDR(NOC_X(mailboxes->launch.dispatch_core_x), NOC_Y(mailboxes->launch.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
uint64_t dispatch_addr = NOC_XY_ADDR(NOC_X(mailboxes->launch.kernel_config.dispatch_core_x),
NOC_Y(mailboxes->launch.kernel_config.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
noc_fast_atomic_increment(noc_index, NCRISC_AT_CMD_BUF, dispatch_addr, NOC_UNICAST_WRITE_VC, 1, 31, false);
#endif

Expand Down
4 changes: 3 additions & 1 deletion tests/tt_metal/tt_metal/test_kernels/misc/watcher_asserts.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ void MAIN {
#else
tt_l1_ptr mailboxes_t* const mailboxes = (tt_l1_ptr mailboxes_t*)(MEM_MAILBOX_BASE);
#endif
uint64_t dispatch_addr = NOC_XY_ADDR(NOC_X(mailboxes->launch.dispatch_core_x), NOC_Y(mailboxes->launch.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
uint64_t dispatch_addr =
NOC_XY_ADDR(NOC_X(mailboxes->launch.kernel_config.dispatch_core_x),
NOC_Y(mailboxes->launch.kernel_config.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
noc_fast_atomic_increment(noc_index, NCRISC_AT_CMD_BUF, dispatch_addr, NOC_UNICAST_WRITE_VC, 1, 31 /*wrap*/, false /*linked*/);
}
#else
Expand Down
2 changes: 1 addition & 1 deletion tests/tt_metal/tt_metal/unit_tests_common/watcher/test_assert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ static void RunTest(WatcherFixture *fixture, Device *device, riscv_id_t riscv_ty
// We should be able to find the expected watcher error in the log as well,
// expected error message depends on the risc we're running on.
string kernel = "tests/tt_metal/tt_metal/test_kernels/misc/watcher_asserts.cpp";
int line_num = 55;
int line_num = 57;

string expected = fmt::format(
"Device {} {} core(x={:2},y={:2}) phys(x={:2},y={:2}): {} tripped an assert on line {}. Current kernel: {}.",
Expand Down
9 changes: 9 additions & 0 deletions tt_eager/tt_dnn/op_library/run_operation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,15 @@ OutputTensors run_device_operation(
operation, input_tensors, optional_input_tensors, output_tensors, optional_output_tensors);
uint32_t device_id = detail::get_device(input_tensors, optional_input_tensors)->id();

if (std::holds_alternative<std::reference_wrapper<Program>>(program))
{
std::get<std::reference_wrapper<Program>>(program).get().set_global_id(op_id);
}
else
{
std::get<std::shared_ptr<Program>>(program)->set_global_id(op_id);
}

// Enqueue or Launch Program
std::visit(
[&operation, &input_tensors, &optional_input_tensors, &output_tensors, queue](auto&& program) {
Expand Down
30 changes: 17 additions & 13 deletions tt_metal/hw/firmware/src/brisc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ inline void deassert_ncrisc_trisc() {
// Below sets ncrisc to go so we can wait until it is cleared on first iteration
mailboxes->slave_sync.all = RUN_SYNC_MSG_ALL_SLAVES_DONE;

uint16_t fw_size16 = mailboxes->launch.ncrisc_kernel_size16;
uint16_t fw_size16 = mailboxes->launch.kernel_config.ncrisc_kernel_size16;
ncrisc_kernel_start_offset16 = fw_size16;

// Copies from L1 to IRAM on chips where NCRISC has IRAM
Expand Down Expand Up @@ -354,42 +354,45 @@ int main() {
// Wait for ncrisc to halt
wait_for_ncrisc_to_halt();

mailboxes->launch.run = RUN_MSG_DONE;
mailboxes->launch.go.run = RUN_MSG_DONE;

while (1) {
init_sync_registers();
reset_ncrisc_with_iram();

DEBUG_STATUS("GW");
while (mailboxes->launch.run != RUN_MSG_GO);
while (mailboxes->launch.go.run != RUN_MSG_GO);
DEBUG_STATUS("GD");

{
DeviceZoneScopedMainN("BRISC-FW");
DeviceZoneSetCounter(mailboxes->launch.kernel_config.host_assigned_op_id);

// Copies from L1 to IRAM on chips where NCRISC has IRAM
l1_to_ncrisc_iram_copy(mailboxes->launch.ncrisc_kernel_size16, ncrisc_kernel_start_offset16);
l1_to_ncrisc_iram_copy(mailboxes->launch.kernel_config.ncrisc_kernel_size16, ncrisc_kernel_start_offset16);

// Invalidate the i$ now the kernels have loaded and before running
volatile tt_reg_ptr uint32_t* cfg_regs = core.cfg_regs_base(0);
cfg_regs[RISCV_IC_INVALIDATE_InvalidateAll_ADDR32] = RISCV_IC_BRISC_MASK | RISCV_IC_TRISC_ALL_MASK | RISCV_IC_NCRISC_MASK;

enum dispatch_core_processor_masks enables = (enum dispatch_core_processor_masks)mailboxes->launch.enables;
enum dispatch_core_processor_masks enables = (enum dispatch_core_processor_masks)mailboxes->launch.kernel_config.enables;
run_triscs(enables);

noc_index = mailboxes->launch.brisc_noc_id;
noc_index = mailboxes->launch.kernel_config.brisc_noc_id;

setup_cb_read_write_interfaces(0, num_cbs_to_early_init, true, true);
finish_ncrisc_copy_and_run(enables);

// Run the BRISC kernel
DEBUG_STATUS("R");
uint32_t kernel_config_base = mailboxes->launch.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_DM0].crta_offset);
uint32_t kernel_config_base = mailboxes->launch.kernel_config.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_DM0].crta_offset);

if (enables & DISPATCH_CLASS_MASK_TENSIX_ENABLE_DM0) {
setup_cb_read_write_interfaces(num_cbs_to_early_init, mailboxes->launch.max_cb_index, true, true);
setup_cb_read_write_interfaces(num_cbs_to_early_init, mailboxes->launch.kernel_config.max_cb_index, true, true);
kernel_init();
} else {
// This was not initialized in kernel_init
Expand All @@ -399,12 +402,13 @@ int main() {

wait_ncrisc_trisc();

mailboxes->launch.run = RUN_MSG_DONE;
mailboxes->launch.go.run = RUN_MSG_DONE;

// Notify dispatcher core that it has completed
if (mailboxes->launch.mode == DISPATCH_MODE_DEV) {
if (mailboxes->launch.kernel_config.mode == DISPATCH_MODE_DEV) {
uint64_t dispatch_addr =
NOC_XY_ADDR(NOC_X(mailboxes->launch.dispatch_core_x), NOC_Y(mailboxes->launch.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
NOC_XY_ADDR(NOC_X(mailboxes->launch.kernel_config.dispatch_core_x),
NOC_Y(mailboxes->launch.kernel_config.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
DEBUG_SANITIZE_NOC_ADDR(dispatch_addr, 4);
noc_fast_atomic_increment(
noc_index,
Expand Down
11 changes: 7 additions & 4 deletions tt_metal/hw/firmware/src/erisc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,15 @@ void __attribute__((section("erisc_l1_code.1"), noinline)) Application(void) {

while (routing_info->routing_enabled) {
// FD: assume that no more host -> remote writes are pending
if (mailboxes->launch.run == RUN_MSG_GO) {
if (mailboxes->launch.go.run == RUN_MSG_GO) {
DeviceZoneScopedMainN("ERISC-FW");
DeviceZoneSetCounter(mailboxes->launch.kernel_config.host_assigned_op_id);
DEBUG_STATUS("R");
uint32_t kernel_config_base = mailboxes->launch.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_ETH_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_ETH_DM0].crta_offset);
uint32_t kernel_config_base = mailboxes->launch.kernel_config.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_ETH_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_ETH_DM0].crta_offset);

kernel_init();
} else {
Expand Down
8 changes: 5 additions & 3 deletions tt_metal/hw/firmware/src/erisck.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@ void __attribute__((section("erisc_l1_code"))) kernel_launch() {
rtos_context_switch_ptr = (void (*)())RtosTable[0];

kernel_main();
mailboxes->launch.run = RUN_MSG_DONE;
uint64_t dispatch_addr = NOC_XY_ADDR(NOC_X(mailboxes->launch.dispatch_core_x), NOC_Y(mailboxes->launch.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
if (routing_info->routing_enabled and mailboxes->launch.mode == DISPATCH_MODE_DEV) {
mailboxes->launch.go.run = RUN_MSG_DONE;
uint64_t dispatch_addr =
NOC_XY_ADDR(NOC_X(mailboxes->launch.kernel_config.dispatch_core_x),
NOC_Y(mailboxes->launch.kernel_config.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
if (routing_info->routing_enabled and mailboxes->launch.kernel_config.mode == DISPATCH_MODE_DEV) {
internal_::notify_dispatch_core_done(dispatch_addr);
}
}
25 changes: 15 additions & 10 deletions tt_metal/hw/firmware/src/idle_erisc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,24 +92,25 @@ int main() {
//device_setup();
noc_init();

mailboxes->launch.run = RUN_MSG_DONE;
mailboxes->launch.go.run = RUN_MSG_DONE;

// Cleanup profiler buffer incase we never get the go message
while (1) {

init_sync_registers();
// Wait...
DEBUG_STATUS("GW");
while (mailboxes->launch.run != RUN_MSG_GO)
while (mailboxes->launch.go.run != RUN_MSG_GO)
{
RISC_POST_HEARTBEAT(heartbeat);
};
DEBUG_STATUS("GD");

{
DeviceZoneScopedMainN("ERISC-IDLE-FW");
DeviceZoneSetCounter(mailboxes->launch.kernel_config.host_assigned_op_id);

noc_index = mailboxes->launch.brisc_noc_id;
noc_index = mailboxes->launch.kernel_config.brisc_noc_id;

//UC FIXME: do i need this?
setup_cb_read_write_interfaces(0, num_cbs_to_early_init, true, true);
Expand All @@ -118,10 +119,12 @@ int main() {
DEBUG_STATUS("R");
//if (mailboxes->launch.enable_brisc) {
//UC FIXME: do i need this?
setup_cb_read_write_interfaces(num_cbs_to_early_init, mailboxes->launch.max_cb_index, true, true);
uint32_t kernel_config_base = mailboxes->launch.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_ETH_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_ETH_DM0].crta_offset);
setup_cb_read_write_interfaces(num_cbs_to_early_init, mailboxes->launch.kernel_config.max_cb_index, true, true);
uint32_t kernel_config_base = mailboxes->launch.kernel_config.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_ETH_DM0].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_ETH_DM0].crta_offset);

kernel_init();
//} else {
Expand All @@ -130,12 +133,14 @@ int main() {
//}
DEBUG_STATUS("D");

mailboxes->launch.run = RUN_MSG_DONE;
mailboxes->launch.go.run = RUN_MSG_DONE;


// Notify dispatcher core that it has completed
if (mailboxes->launch.mode == DISPATCH_MODE_DEV) {
uint64_t dispatch_addr = NOC_XY_ADDR(NOC_X(mailboxes->launch.dispatch_core_x), NOC_Y(mailboxes->launch.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
if (mailboxes->launch.kernel_config.mode == DISPATCH_MODE_DEV) {
uint64_t dispatch_addr =
NOC_XY_ADDR(NOC_X(mailboxes->launch.kernel_config.dispatch_core_x),
NOC_Y(mailboxes->launch.kernel_config.dispatch_core_y), DISPATCH_MESSAGE_ADDR);
DEBUG_SANITIZE_NOC_ADDR(dispatch_addr, 4);
noc_fast_atomic_increment(noc_index, NCRISC_AT_CMD_BUF, dispatch_addr, NOC_UNICAST_WRITE_VC, 1, 31 /*wrap*/, false /*linked*/);
}
Expand Down
10 changes: 6 additions & 4 deletions tt_metal/hw/firmware/src/ncrisc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,13 @@ int main(int argc, char *argv[]) {
notify_brisc_and_wait();
DeviceZoneScopedMainN("NCRISC-FW");

setup_cb_read_write_interfaces(0, mailboxes->launch.max_cb_index, true, true);
setup_cb_read_write_interfaces(0, mailboxes->launch.kernel_config.max_cb_index, true, true);

uint32_t kernel_config_base = mailboxes->launch.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_DM1].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_DM1].crta_offset);
uint32_t kernel_config_base = mailboxes->launch.kernel_config.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_DM1].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_DM1].crta_offset);

DEBUG_STATUS("R");
kernel_init();
Expand Down
10 changes: 6 additions & 4 deletions tt_metal/hw/firmware/src/trisc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,14 @@ int main(int argc, char *argv[]) {
DeviceZoneScopedMainN("TRISC-FW");

#if !defined(UCK_CHLKC_MATH)
setup_cb_read_write_interfaces(0, mailboxes->launch.max_cb_index, cb_init_read, cb_init_write);
setup_cb_read_write_interfaces(0, mailboxes->launch.kernel_config.max_cb_index, cb_init_read, cb_init_write);
#endif

uint32_t kernel_config_base = mailboxes->launch.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_COMPUTE].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base + mailboxes->launch.mem_map[DISPATCH_CLASS_TENSIX_COMPUTE].crta_offset);
uint32_t kernel_config_base = mailboxes->launch.kernel_config.kernel_config_base;
rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_COMPUTE].rta_offset);
crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
mailboxes->launch.kernel_config.mem_map[DISPATCH_CLASS_TENSIX_COMPUTE].crta_offset);

DEBUG_STATUS("R");
kernel_init();
Expand Down
2 changes: 1 addition & 1 deletion tt_metal/hw/inc/debug/assert.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ void assert_and_hang(uint32_t line_num) {

// Update launch msg to show that we've exited.
tt_l1_ptr launch_msg_t *launch_msg = GET_MAILBOX_ADDRESS_DEV(launch);
launch_msg->run = RUN_MSG_DONE;
launch_msg->go.run = RUN_MSG_DONE;

// Hang, or in the case of erisc, early exit.
#if defined(COMPILE_FOR_ERISC)
Expand Down
2 changes: 1 addition & 1 deletion tt_metal/hw/inc/debug/sanitize_noc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ inline void debug_sanitize_post_noc_addr_and_hang(

// Update launch msg to show that we've exited.
tt_l1_ptr launch_msg_t *launch_msg = GET_MAILBOX_ADDRESS_DEV(launch);
launch_msg->run = RUN_MSG_DONE;
launch_msg->go.run = RUN_MSG_DONE;

#if defined(COMPILE_FOR_ERISC)
// For erisc, we can't hang the kernel/fw, because the core doesn't get restarted when a new
Expand Down
18 changes: 15 additions & 3 deletions tt_metal/hw/inc/dev_msgs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,12 @@ struct dyn_mem_map_t {
volatile uint16_t crta_offset;
};

struct launch_msg_t { // must be cacheline aligned
struct kernel_config_msg_t {
volatile uint16_t watcher_kernel_ids[DISPATCH_CLASS_MAX];
volatile uint16_t ncrisc_kernel_size16; // size in 16 byte units

volatile uint16_t host_assigned_op_id;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pgkeller was changing launch_msg_t in his PR here: https://github.com/tenstorrent/tt-metal/pull/9781/files#diff-d606266ae23fd80f4fa4f80ea161f8b17ff937bbf94cce50cfd731755d0a491b

He suspects that moving run field around is causing some issues.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm interesting, I did not see any hangs on CI with the addition of the two bytes. I will coordinate before merging this in.


// Ring buffer of kernel configuration data
volatile uint32_t kernel_config_base;
dyn_mem_map_t mem_map[DISPATCH_CLASS_MAX];
Expand All @@ -91,7 +93,17 @@ struct launch_msg_t { // must be cacheline aligned
volatile uint8_t dispatch_core_x;
volatile uint8_t dispatch_core_y;
volatile uint8_t exit_erisc_kernel;
volatile uint8_t run; // must be in last cacheline of this msg
volatile uint8_t pad1;
volatile uint16_t pad2;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need this 2 byte padding?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to be 32-bit aligned on this struct, cache aligned.

} __attribute__((packed));

struct go_msg_t {
volatile uint32_t run; // must be in last cacheline of this msg
} __attribute__((packed));

struct launch_msg_t { // must be cacheline aligned
kernel_config_msg_t kernel_config;
go_msg_t go;
} __attribute__((packed));

struct slave_sync_msg_t {
Expand Down Expand Up @@ -194,7 +206,7 @@ struct mailboxes_t {
struct debug_insert_delays_msg_t debug_insert_delays;
};

static_assert(sizeof(launch_msg_t) % sizeof(uint32_t) == 0);
static_assert(sizeof(kernel_config_msg_t) % sizeof(uint32_t) == 0);

#ifndef TENSIX_FIRMWARE
// Validate assumptions on mailbox layout on host compile
Expand Down
Loading
Loading