diff --git a/src/common/snippets/include/snippets/lowered/loop_port.hpp b/src/common/snippets/include/snippets/lowered/loop_port.hpp index 3804d0bedd2d6e..f9b74a4f042944 100644 --- a/src/common/snippets/include/snippets/lowered/loop_port.hpp +++ b/src/common/snippets/include/snippets/lowered/loop_port.hpp @@ -20,13 +20,13 @@ class LoopPort { enum {UNDEFINED_DIM_IDX = std::numeric_limits::max()}; enum class Type { Incremented, // Loop port which data ptr should be incremented after each Loop iteration - NotIncremented, // Loop port which data ptr should not be to avoid double increment + NotIncremented, // Loop port which data ptr should not be incremented (for example, to avoid double increment) NotProcessed, // LoopPort which doesn't process the dim by `dim_idx` (UNDEFINED_DIM_IDX) and is used only for Loop bound definition }; LoopPort() = default; - template::type = true> static LoopPort create(const ExpressionPort& port, size_t dim_idx = 0) { return LoopPort(port, dim_idx, T); @@ -46,12 +46,20 @@ class LoopPort { const std::shared_ptr& get_expr_port() const { return m_expr_port; } Type get_type() const { return m_type; } - size_t get_dim_idx() const { return m_dim_idx; } + size_t get_dim_idx() const; void set_expr_port(std::shared_ptr p); - void set_type(Type type); void set_dim_idx(size_t idx); + template::type = true> + void convert_to_type() { + m_type = T; + } + + bool is_processed() const; + bool is_incremented() const; + private: LoopPort(const ExpressionPort& port, size_t dim_idx, Type type); @@ -60,22 +68,7 @@ class LoopPort { Type m_type = Type::Incremented; }; -inline std::ostream& operator<<(std::ostream& out, const LoopPort::Type& type) { - switch (type) { - case LoopPort::Type::Incremented: - out << "Incremented"; - break; - case LoopPort::Type::NotIncremented: - out << "NotIncremented"; - break; - case LoopPort::Type::NotProcessed: - out << "NotProcessed"; - break; - default: - OPENVINO_THROW("Unknown LoopPort Type"); - } - return out; -} +std::ostream& operator<<(std::ostream& out, const LoopPort::Type& type); } // namespace lowered } // namespace snippets diff --git a/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp b/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp index 40a2611b80ef48..32b5e241ba4cf8 100644 --- a/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/insert_buffers.hpp @@ -32,8 +32,8 @@ class InsertBuffers : public RangedPass { const LinearIR::constExprIt& begin_it, const LinearIR::constExprIt& end_it, const LoopManagerPtr& loop_manager, - const std::vector& loop_entries, - const std::vector& loop_exits) const; + const std::vector& loop_entries, + const std::vector& loop_exits) const; static LinearIR::constExprIt insertion_position(const LinearIR& linear_ir, const LoopManagerPtr& loop_manager, diff --git a/src/common/snippets/src/lowered/expressions/buffer_expression.cpp b/src/common/snippets/src/lowered/expressions/buffer_expression.cpp index ad44fa48d24b2a..0a1b9758a6c371 100644 --- a/src/common/snippets/src/lowered/expressions/buffer_expression.cpp +++ b/src/common/snippets/src/lowered/expressions/buffer_expression.cpp @@ -109,8 +109,10 @@ void BufferExpression::init_allocation_size(const std::shared_ptr& OPENVINO_ASSERT(it != output_ports.end(), "compute_allocation_shape: output port of parent loop can not be found"); } const auto& loop_port = *it; + if (!loop_port.is_processed()) + continue; const auto& dim_idx = loop_port.get_dim_idx(); - if (loop_port.get_type() != LoopPort::Type::NotProcessed && dim_idx < rank) { + if (dim_idx < rank) { if (const auto& unified_loop_info = ov::as_type_ptr(loop_info)) m_allocation_size = utils::dynamic_safe_mul(m_allocation_size, unified_loop_info->get_work_amount()); else if (const auto& expanded_loop_info = ov::as_type_ptr(loop_info)) diff --git a/src/common/snippets/src/lowered/loop_info.cpp b/src/common/snippets/src/lowered/loop_info.cpp index dfd75f4cf4be97..3e9da0e15ada44 100644 --- a/src/common/snippets/src/lowered/loop_info.cpp +++ b/src/common/snippets/src/lowered/loop_info.cpp @@ -19,9 +19,9 @@ LoopInfo::LoopInfo(size_t work_amount, size_t increment, const std::vector(port)); for (const auto& port : exits) - m_output_ports.push_back(LoopPort::create(port)); + m_output_ports.push_back(LoopPort::create(port)); } bool LoopInfo::is_dynamic() const { @@ -30,12 +30,20 @@ bool LoopInfo::is_dynamic() const { size_t LoopInfo::get_dim_idx() const { OPENVINO_ASSERT(!m_input_ports.empty(), "Loop info must have at least one input port"); - auto equal_dim_idxes = [&](const LoopPort& p) { - return p.get_type() == LoopPort::Type::NotProcessed || p.get_dim_idx() == m_input_ports[0].get_dim_idx(); - }; + + auto is_processed = [](const LoopPort& p) { return p.is_processed(); }; + auto is_processed_it = std::find_if(m_input_ports.begin(), m_input_ports.end(), is_processed); + if (is_processed_it == m_input_ports.end()) { + is_processed_it = std::find_if(m_output_ports.begin(), m_output_ports.end(), is_processed); + if (is_processed_it == m_output_ports.end()) + return LoopPort::UNDEFINED_DIM_IDX; + } + const auto dim_idx = is_processed_it->get_dim_idx(); + + auto equal_dim_idxes = [&](const LoopPort& p) { return !p.is_processed() || p.get_dim_idx() == dim_idx; }; if (std::all_of(m_input_ports.begin(), m_input_ports.end(), equal_dim_idxes) && std::all_of(m_output_ports.begin(), m_output_ports.end(), equal_dim_idxes)) { - return m_input_ports[0].get_dim_idx(); + return dim_idx; } else { return LoopPort::UNDEFINED_DIM_IDX; } @@ -60,7 +68,7 @@ size_t LoopInfo::get_increment() const { std::vector LoopInfo::get_is_incremented() const { std::vector values; values.reserve(get_input_count() + get_output_count()); - iterate_through_ports([&values](const LoopPort& port) { values.push_back(port.get_type() == LoopPort::Type::Incremented); }); + iterate_through_ports([&values](const LoopPort& port) { values.push_back(port.is_incremented()); }); return values; } @@ -81,10 +89,7 @@ void LoopInfo::set_increment(size_t increment) { } void LoopInfo::set_dim_idx(size_t dim_idx) { - auto setter = [dim_idx](LoopPort& port) { - if (port.get_type() != LoopPort::Type::NotProcessed) - port.set_dim_idx(dim_idx); - }; + auto setter = [dim_idx](LoopPort& port) { if (port.is_processed()) port.set_dim_idx(dim_idx); }; std::for_each(m_input_ports.begin(), m_input_ports.end(), setter); std::for_each(m_output_ports.begin(), m_output_ports.end(), setter); } diff --git a/src/common/snippets/src/lowered/loop_port.cpp b/src/common/snippets/src/lowered/loop_port.cpp index 31f05f293efd3c..6168d77b0d52e2 100644 --- a/src/common/snippets/src/lowered/loop_port.cpp +++ b/src/common/snippets/src/lowered/loop_port.cpp @@ -23,18 +23,35 @@ std::shared_ptr LoopPort::clone_with_new_expr(const ExpressionPtr& new return new_loop_port; } +bool LoopPort::is_processed() const { + switch (m_type) { + case Type::Incremented: + case Type::NotIncremented: + return true; + case Type::NotProcessed: + return false; + default: + OPENVINO_THROW("Unknown LoopPort type"); + } +} + +bool LoopPort::is_incremented() const { + return m_type == Type::Incremented; +} + +size_t LoopPort::get_dim_idx() const { + OPENVINO_ASSERT(is_processed(), "NotProcessed LoopPort cannot call `get_dim_idx()`"); + return m_dim_idx; +} + void LoopPort::set_expr_port(std::shared_ptr p) { OPENVINO_ASSERT(p, "Expression port is missed"); m_expr_port = std::move(p); } -void LoopPort::set_type(Type type) { - m_type = type; -} - void LoopPort::set_dim_idx(size_t idx) { - if (get_type() == LoopPort::Type::NotProcessed) { - OPENVINO_ASSERT(idx == UNDEFINED_DIM_IDX, "NotProcessed LoopPort cah have only UNDEFINED_DIM_IDX"); + if (!is_processed()) { + OPENVINO_ASSERT(idx == UNDEFINED_DIM_IDX, "NotProcessed LoopPort can have only UNDEFINED_DIM_IDX"); } else { OPENVINO_ASSERT(idx < m_expr_port->get_descriptor_ptr()->get_shape().size(), "LoopPort dim_idx (", @@ -63,6 +80,24 @@ bool operator<(const LoopPort& lhs, const LoopPort& rhs) { (lhs.m_type == rhs.m_type && lhs.m_dim_idx < rhs.m_dim_idx))); } +std::ostream& operator<<(std::ostream& out, const LoopPort::Type& type) { + switch (type) { + case LoopPort::Type::Incremented: + out << "Incremented"; + break; + case LoopPort::Type::NotIncremented: + out << "NotIncremented"; + break; + case LoopPort::Type::NotProcessed: + out << "NotProcessed"; + break; + default: + OPENVINO_THROW("Unknown LoopPort Type"); + } + return out; +} + + } // namespace lowered } // namespace snippets } // namespace ov diff --git a/src/common/snippets/src/lowered/pass/brgemm_blocking.cpp b/src/common/snippets/src/lowered/pass/brgemm_blocking.cpp index c46a0cae1fd366..ba3bc93df991c4 100644 --- a/src/common/snippets/src/lowered/pass/brgemm_blocking.cpp +++ b/src/common/snippets/src/lowered/pass/brgemm_blocking.cpp @@ -146,21 +146,21 @@ bool BrgemmBlockingBase::mark_blocking_loops(snippets::lowered::LinearIR& linear const auto& loop_manager = linear_ir.get_loop_manager(); if (!ov::snippets::utils::is_full_dim_value(k_block)) { - const std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 0), - LoopPort::create(brgemm_expr->get_input_port(1), 1)}; + const std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 0), + LoopPort::create(brgemm_expr->get_input_port(1), 1)}; const std::vector exits{LoopPort::create(brgemm_expr->get_output_port(0))}; mark_k_blocking(loop_manager, brgemm_it, std::next(brgemm_it), entries, exits, k_block); } if (!ov::snippets::utils::is_full_dim_value(n_block)) { const std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0)), - LoopPort::create(brgemm_expr->get_input_port(1))}; - const std::vector exits{LoopPort::create(brgemm_expr->get_output_port(0))}; + LoopPort::create(brgemm_expr->get_input_port(1))}; + const std::vector exits{LoopPort::create(brgemm_expr->get_output_port(0))}; mark_n_blocking(loop_manager, brgemm_it, std::next(brgemm_it), entries, exits, n_block); } if (!ov::snippets::utils::is_full_dim_value(m_block)) { - const std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 1), + const std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 1), LoopPort::create(brgemm_expr->get_input_port(1))}; - const std::vector exits{LoopPort::create(brgemm_expr->get_output_port(0), 1)}; + const std::vector exits{LoopPort::create(brgemm_expr->get_output_port(0), 1)}; mark_m_blocking(loop_manager, brgemm_it, std::next(brgemm_it), entries, exits, m_block); } return true; diff --git a/src/common/snippets/src/lowered/pass/clean_repeated_ptr_shifts.cpp b/src/common/snippets/src/lowered/pass/clean_repeated_ptr_shifts.cpp index 4de50d5cd8b7fe..9090b0a0cc7ef7 100644 --- a/src/common/snippets/src/lowered/pass/clean_repeated_ptr_shifts.cpp +++ b/src/common/snippets/src/lowered/pass/clean_repeated_ptr_shifts.cpp @@ -96,10 +96,10 @@ bool CleanRepeatedDataPointerShifts::reuse_increments(const LoopManagerPtr& loop const auto loop_info = loop_manager->get_loop_info(loop_end->get_id()); size_t loop_port_idx = 0; loop_info->iterate_through_infos([&resetting_data_indexes, &loop_port_idx](LoopPort& loop_port, UnifiedLoopInfo::LoopPortDesc& shifts) { - if (resetting_data_indexes.count(loop_port_idx) && loop_port.get_type() != LoopPort::Type::NotProcessed) { + if (resetting_data_indexes.count(loop_port_idx) && loop_port.is_processed()) { shifts.ptr_increment = 0; shifts.finalization_offset = 0; - loop_port.set_type(LoopPort::Type::NotIncremented); + loop_port.convert_to_type(); } ++loop_port_idx; }); diff --git a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp index 62e446f116dc56..937babf4a19bf0 100644 --- a/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp +++ b/src/common/snippets/src/lowered/pass/define_buffer_clusters.cpp @@ -134,8 +134,8 @@ void DefineBufferClusters::parse_loop(const LoopManagerPtr& loop_manager, const const auto out_path = MarkInvariantShapePath::getInvariantPortShapePath(*output_buffer_port_info.port.get_expr_port()); // - Memory can be reused if there are the same loop pointer increments (data size, final offsets, ptr increments). // For that, loop ports with buffers should be on the same shape-path and have the same value of `is_incremented`. - const auto in_is_incremented = input_buffer_port_info.port.get_type() == LoopPort::Type::Incremented; - const auto out_is_incremented = output_buffer_port_info.port.get_type() == LoopPort::Type::Incremented; + const auto in_is_incremented = input_buffer_port_info.port.is_incremented(); + const auto out_is_incremented = output_buffer_port_info.port.is_incremented(); if (in_path != out_path || in_is_incremented != out_is_incremented) continue; @@ -176,8 +176,7 @@ void DefineBufferClusters::parse_nested_loops(const LoopManagerPtr& loop_manager auto can_be_data_ptr_proportionally_shifted = [](const LoopPortInfo& outer_port_info, const LoopPortInfo& inner_port_info) { // Outer Buffer ptr should be shifted to emulate "window" sliding const auto& outer_desc = outer_port_info.desc; - if (outer_port_info.port.get_type() != LoopPort::Type::Incremented || - (!utils::is_dynamic_value(outer_desc.ptr_increment) && outer_desc.ptr_increment == 0)) + if (!outer_port_info.port.is_incremented() || (!utils::is_dynamic_value(outer_desc.ptr_increment) && outer_desc.ptr_increment == 0)) return false; OPENVINO_ASSERT(inner_port_info.port.get_expr_port() && outer_port_info.port.get_expr_port(), "Expression ports are nullptr!"); diff --git a/src/common/snippets/src/lowered/pass/extract_loop_invariants.cpp b/src/common/snippets/src/lowered/pass/extract_loop_invariants.cpp index a9917435891f1a..f873a782b088b5 100644 --- a/src/common/snippets/src/lowered/pass/extract_loop_invariants.cpp +++ b/src/common/snippets/src/lowered/pass/extract_loop_invariants.cpp @@ -89,7 +89,7 @@ bool is_extraction_applicable(const ExpressionPtr& expr, const UnifiedLoopInfoPt if (is_loop_port) { // stride is not 1 after move to outside, then should not extract. const auto& loop_port = inner_loop_info->get_loop_port(expr_input_ports[i]); - if (loop_port.get_type() == LoopPort::Type::NotProcessed || get_stride_after_move_outer(loop_port) != 1) { + if (!loop_port.is_processed() || get_stride_after_move_outer(loop_port) != 1) { return false; } } diff --git a/src/common/snippets/src/lowered/pass/fuse_loops.cpp b/src/common/snippets/src/lowered/pass/fuse_loops.cpp index c41990f5bda754..e2dc1451aa9de0 100644 --- a/src/common/snippets/src/lowered/pass/fuse_loops.cpp +++ b/src/common/snippets/src/lowered/pass/fuse_loops.cpp @@ -35,7 +35,7 @@ bool FuseLoops::loop_ports_are_compatible(const LoopInfoPtr& loop_upper, const auto upper_exit_port_it = found_port(upper_exit_ports, src_port); if (upper_exit_port_it != upper_exit_ports.cend()) { const auto& upper_exit_port = *upper_exit_port_it; - if (!utils::everyone_is(LoopPort::Type::Incremented, lower_entry_port.get_type(), upper_exit_port.get_type())) + if (!lower_entry_port.is_incremented() || !upper_exit_port.is_incremented()) return false; if (lower_entry_port.get_dim_idx() != upper_exit_port.get_dim_idx()) return false; diff --git a/src/common/snippets/src/lowered/pass/init_loops.cpp b/src/common/snippets/src/lowered/pass/init_loops.cpp index e6b6cf7b15bdd6..07143ec8b59e6d 100644 --- a/src/common/snippets/src/lowered/pass/init_loops.cpp +++ b/src/common/snippets/src/lowered/pass/init_loops.cpp @@ -19,7 +19,7 @@ namespace { inline void init_is_incremented(LoopPort& port) { const auto& expr = port.get_expr_port()->get_expr(); if (!std::dynamic_pointer_cast(expr->get_node())) { - port.set_type(LoopPort::Type::NotIncremented); + port.convert_to_type(); } } diff --git a/src/common/snippets/src/lowered/pass/insert_buffers.cpp b/src/common/snippets/src/lowered/pass/insert_buffers.cpp index 16423c72148336..1014517a47c14f 100644 --- a/src/common/snippets/src/lowered/pass/insert_buffers.cpp +++ b/src/common/snippets/src/lowered/pass/insert_buffers.cpp @@ -72,12 +72,11 @@ void InsertBuffers::insertion(LinearIR& linear_ir, const LinearIR::constExprIt& begin_it, const LinearIR::constExprIt& end_it, const LoopManagerPtr& loop_manager, - const std::vector& loop_entries, - const std::vector& loop_exits) const { - for (const auto& input_port : loop_entries) { - const auto& entry_port = input_port.get_expr_port(); - const auto& expr = entry_port->get_expr(); - const auto port_idx = entry_port->get_index(); + const std::vector& loop_entries, + const std::vector& loop_exits) const { + for (const auto& entry_port : loop_entries) { + const auto& expr = entry_port.get_expr(); + const auto port_idx = entry_port.get_index(); const auto node = expr->get_node(); auto parent_expr_output = expr->get_input_port_connector(port_idx)->get_source(); auto parent_expr = parent_expr_output.get_expr(); @@ -116,17 +115,16 @@ void InsertBuffers::insertion(LinearIR& linear_ir, // Need to insert between 2nd and 4th Loops - after 2nd Loop const auto pos = insertion_position(linear_ir, loop_manager, parent_expr, expr); const auto buffer = std::make_shared(parent->output(parent_port)); - const auto buffer_consumer = has_shape_infer_parent ? top_shape_infer_expr->get_input_port(0) : *entry_port; + const auto buffer_consumer = has_shape_infer_parent ? top_shape_infer_expr->get_input_port(0) : entry_port; linear_ir.insert_node(buffer, std::vector{ parent_expr_output }, buffer_loop_ids, false, pos, { buffer_consumer }); } } - for (const auto& output_port : loop_exits) { - const auto& exit_port = output_port.get_expr_port(); - const auto& expr = exit_port->get_expr(); - const auto port_idx = exit_port->get_index(); + for (const auto& exit_port : loop_exits) { + const auto& expr = exit_port.get_expr(); + const auto port_idx = exit_port.get_index(); const auto node = expr->get_node(); - const auto output_connector = exit_port->get_port_connector_ptr(); + const auto output_connector = exit_port.get_port_connector_ptr(); const auto child_exprs_inputs = output_connector->get_consumers(); const auto& current_loops = expr->get_loop_ids(); @@ -200,7 +198,7 @@ void InsertBuffers::insertion(LinearIR& linear_ir, // | <- It should be new PortConnector // Relu // Output port connector is automatically filled from PortDescriptor - linear_ir.insert_node(buffer, std::vector{ *exit_port }, buffer_loop_ids, false, pos, { potential_consumers }); + linear_ir.insert_node(buffer, std::vector{ exit_port }, buffer_loop_ids, false, pos, { potential_consumers }); } } } @@ -213,8 +211,15 @@ bool InsertBuffers::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begi const auto loop_info = loop_data.second; const auto loop_entries = loop_info->get_input_ports(); const auto loop_exits = loop_info->get_output_ports(); + + auto cvt_to_expr_ports = [](const std::vector& loop_ports) { + std::vector expr_ports(loop_ports.size()); + std::transform(loop_ports.cbegin(), loop_ports.cend(), expr_ports.begin(), + [](const LoopPort& loop_port) { return *loop_port.get_expr_port(); }); + return expr_ports; + }; // using begin() as expr_it because we work with LoopInfo, not expressions in Linear IR - insertion(linear_ir, begin, end, loop_manager, loop_entries, loop_exits); + insertion(linear_ir, begin, end, loop_manager, cvt_to_expr_ports(loop_entries), cvt_to_expr_ports(loop_exits)); } for (auto expr_it = begin; expr_it != end; expr_it++) { @@ -226,12 +231,12 @@ bool InsertBuffers::run(LinearIR& linear_ir, lowered::LinearIR::constExprIt begi const auto input_ports = ma->get_memory_access_input_ports(); const auto output_ports = ma->get_memory_access_output_ports(); - std::vector loop_entries(input_ports.size()), loop_exits(output_ports.size()); + std::vector loop_entries(input_ports.size()), loop_exits(output_ports.size()); for (const auto& p : input_ports) { - loop_entries[p.first] = LoopPort::create(expr->get_input_port(p.first)); + loop_entries[p.first] = expr->get_input_port(p.first); } for (const auto& p : output_ports) { - loop_exits[p.first] = LoopPort::create(expr->get_output_port(p.first)); + loop_exits[p.first] = expr->get_output_port(p.first); } insertion(linear_ir, expr_it, end, loop_manager, loop_entries, loop_exits); diff --git a/src/common/snippets/src/lowered/pass/mha_parallel_wa_optimizer.cpp b/src/common/snippets/src/lowered/pass/mha_parallel_wa_optimizer.cpp index af956c0d4b3d70..c75d1e86abbfa5 100644 --- a/src/common/snippets/src/lowered/pass/mha_parallel_wa_optimizer.cpp +++ b/src/common/snippets/src/lowered/pass/mha_parallel_wa_optimizer.cpp @@ -105,7 +105,7 @@ std::unordered_set MHAParallelWAOptimizer::find_applicab return false; bool loop_by_m = true; outermost_loop->iterate_through_ports([&loop_by_m](const lowered::LoopPort& port) { - if (port.get_type() != LoopPort::Type::NotProcessed && port.get_dim_idx() != m_dim_M_idx) + if (port.is_processed() && port.get_dim_idx() != m_dim_M_idx) loop_by_m = false; }); return loop_by_m; diff --git a/src/common/snippets/src/lowered/pass/propagate_subtensors.cpp b/src/common/snippets/src/lowered/pass/propagate_subtensors.cpp index 00803967db8dd5..f53f23be6e1fe9 100644 --- a/src/common/snippets/src/lowered/pass/propagate_subtensors.cpp +++ b/src/common/snippets/src/lowered/pass/propagate_subtensors.cpp @@ -56,7 +56,7 @@ void propagate_updated_subtensor_through_loop(const LinearIR& linear_ir, // First step: set new dim value to the corresponding input_ports' dimensions if (most_outer_loop) { for (const auto& port : loop_info->get_input_ports()) { - if (port.get_type() != LoopPort::Type::NotProcessed) { + if (port.is_processed()) { const auto& expr = port.get_expr_port()->get_expr(); const auto& desc = port.get_expr_port()->get_descriptor_ptr(); auto subtensor = desc->get_subtensor(); @@ -77,7 +77,7 @@ void propagate_updated_subtensor_through_loop(const LinearIR& linear_ir, } auto update_only_dim_idx_with_subtensor_value = [&](const LoopPort& port) { - if (port.get_type() != LoopPort::Type::NotProcessed) { + if (port.is_processed()) { const auto desc = port.get_expr_port()->get_descriptor_ptr(); const auto expr = port.get_expr_port()->get_expr(); const auto parent_desc = expr->get_input_port_connector(port.get_expr_port()->get_index())->get_source().get_descriptor_ptr(); diff --git a/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp b/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp index f33cea152939a9..a8144bce9a1d80 100644 --- a/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp +++ b/src/common/snippets/src/lowered/pass/set_buffer_reg_group.cpp @@ -34,8 +34,8 @@ bool SetBufferRegGroup::can_be_in_one_reg_group(const UnifiedLoopInfo::LoopPortI const auto equal_invariant_shape_paths = MarkInvariantShapePath::getInvariantPortShapePath(*lhs_info.port.get_expr_port()) == MarkInvariantShapePath::getInvariantPortShapePath(*rhs_info.port.get_expr_port()); - const auto lhs_is_incremented = lhs_info.port.get_type() == LoopPort::Type::Incremented; - const auto rhs_is_incremented = rhs_info.port.get_type() == LoopPort::Type::Incremented; + const auto lhs_is_incremented = lhs_info.port.is_incremented(); + const auto rhs_is_incremented = rhs_info.port.is_incremented(); const auto equal_is_incremented = lhs_is_incremented == rhs_is_incremented; return equal_invariant_shape_paths && equal_is_incremented && (equal_element_type_sizes || !lhs_is_incremented || (lhs_info.desc.ptr_increment == 0 && lhs_info.desc.finalization_offset == 0)); diff --git a/src/common/snippets/src/lowered/pass/validate.cpp b/src/common/snippets/src/lowered/pass/validate.cpp index d5bee2a46d6192..e76f994d1284e1 100644 --- a/src/common/snippets/src/lowered/pass/validate.cpp +++ b/src/common/snippets/src/lowered/pass/validate.cpp @@ -117,8 +117,7 @@ void validate_loop_end(const ExpressionPtr& expr, const LinearIR& linear_ir) { const auto& final_offsets = loop_end->get_finalization_offsets(); auto validate_loop_ports = [&](const std::vector& loop_port_infos, size_t shift = 0) { for (size_t i = 0; i < loop_port_infos.size(); ++i) { - const auto is_port_incremented = loop_port_infos[i].port.get_type() == LoopPort::Type::Incremented; - OPENVINO_ASSERT(is_incremented[i + shift] == is_port_incremented && + OPENVINO_ASSERT(is_incremented[i + shift] == loop_port_infos[i].port.is_incremented() && ptr_increments[i + shift] == loop_port_infos[i].desc.ptr_increment && final_offsets[i + shift] == loop_port_infos[i].desc.finalization_offset, "Incompatible data ptr shifts in LoopEnd and the corresponding LoopInfo"); diff --git a/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp b/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp index fb75995961cf11..e127aaea0c11d3 100644 --- a/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp +++ b/src/common/snippets/src/lowered/pass/validate_unified_loops.cpp @@ -65,10 +65,10 @@ void ValidateUnifiedLoops::validate_loop_infos(const LoopManagerPtr& loop_manage OPENVINO_ASSERT(loop_info, "ValidateUnifiedLoops expects only UnifiedLoopInfo in LoopManager"); loop_info->iterate_through_ports(validate_loop_port); - // Validate that iteration dimnsion is broadcastable + // Validate that iteration dimension is broadcastable std::set unique_dimensions; loop_info->iterate_through_ports([&unique_dimensions](const LoopPort& loop_port) { - if (loop_port.get_type() != LoopPort::Type::NotProcessed) { + if (loop_port.is_processed()) { const auto is_input = loop_port.get_expr_port()->get_type() == ExpressionPort::Input; const auto planar_shape = is_input ? ov::snippets::utils::get_planar_vdims(*loop_port.get_expr_port()) : ov::snippets::utils::get_preordered_vdims(*loop_port.get_expr_port()); diff --git a/src/common/snippets/src/utils/loop_utils.cpp b/src/common/snippets/src/utils/loop_utils.cpp index 5efa22e4fc0752..4683006d49432b 100644 --- a/src/common/snippets/src/utils/loop_utils.cpp +++ b/src/common/snippets/src/utils/loop_utils.cpp @@ -13,7 +13,7 @@ namespace utils { using namespace ov::snippets::lowered; namespace { inline int64_t get_ptr_increment(const LoopPort& loop_port, size_t work_amount, size_t port_count) { - if (loop_port.get_type() != LoopPort::Type::Incremented) + if (!loop_port.is_incremented()) return 0; const auto& expr_port = loop_port.get_expr_port(); @@ -47,7 +47,7 @@ inline int64_t get_finalization_offset(size_t work_amount, int64_t ptr_increment inline void init_work_amount(const LoopInfoPtr& loop_info) { size_t work_amount = 1; loop_info->iterate_through_ports([&work_amount](const LoopPort& loop_port) { - if (loop_port.get_type() == LoopPort::Type::Incremented) { + if (loop_port.is_processed()) { const auto& desc = loop_port.get_expr_port()->get_descriptor_ptr(); const auto& shape = desc->get_shape(); const auto& layout = desc->get_layout(); diff --git a/src/common/snippets/tests/src/lir_comparator.cpp b/src/common/snippets/tests/src/lir_comparator.cpp index 5fc58b6af7e174..b547638eb798a3 100644 --- a/src/common/snippets/tests/src/lir_comparator.cpp +++ b/src/common/snippets/tests/src/lir_comparator.cpp @@ -182,7 +182,8 @@ LIRComparator::Result LIRComparator::compare_loop_ports(const std::vectorget_loop_manager()->mark_loop(begin, end, 512, vector_size, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), - LoopPort::create((*multiply.first)->get_input_port(1)), - LoopPort::create((*sub.first)->get_input_port(0))}, - std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); + std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), + LoopPort::create((*multiply.first)->get_input_port(1)), + LoopPort::create((*sub.first)->get_input_port(0))}, + std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); linear_ir->set_loop_depth(1); } { @@ -82,9 +82,9 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsWithParams) { auto begin = sub.first; auto end = result.first; linear_ir_ref->get_loop_manager()->mark_loop(begin, end, 512, vector_size, - std::vector{LoopPort::create((*sub.first)->get_input_port(0)), - LoopPort::create((*sub.first)->get_input_port(1))}, - std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); + std::vector{LoopPort::create((*sub.first)->get_input_port(0)), + LoopPort::create((*sub.first)->get_input_port(1))}, + std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); } } @@ -121,9 +121,9 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsWithScalar) { auto begin = scalar.first; auto end = result.first; linear_ir->get_loop_manager()->mark_loop(begin, end, 512, vector_size, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), - LoopPort::create((*sub.first)->get_input_port(0))}, - std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); + std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), + LoopPort::create((*sub.first)->get_input_port(0))}, + std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); linear_ir->set_loop_depth(1); } { @@ -139,9 +139,9 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsWithScalar) { auto begin = sub.first; auto end = result.first; linear_ir_ref->get_loop_manager()->mark_loop(begin, end, 512, vector_size, - std::vector{LoopPort::create((*sub.first)->get_input_port(0)), - LoopPort::create((*sub.first)->get_input_port(1))}, - std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); + std::vector{LoopPort::create((*sub.first)->get_input_port(0)), + LoopPort::create((*sub.first)->get_input_port(1))}, + std::vector{LoopPort::create((*sub.first)->get_output_port(0))}); } } @@ -184,19 +184,19 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsOutputLoopUpdateNotNeed auto begin = multiply.first; auto end = result1.first; linear_ir->get_loop_manager()->mark_loop(begin, end, 16, vector_size, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), - LoopPort::create((*multiply.first)->get_input_port(1)), - LoopPort::create((*add.first)->get_input_port(0)), - LoopPort::create((*sub.first)->get_input_port(0))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0)), - LoopPort::create((*sub.first)->get_output_port(0))}); + std::vector{LoopPort::create((*multiply.first)->get_input_port(0)), + LoopPort::create((*multiply.first)->get_input_port(1)), + LoopPort::create((*add.first)->get_input_port(0)), + LoopPort::create((*sub.first)->get_input_port(0))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0)), + LoopPort::create((*sub.first)->get_output_port(0))}); linear_ir->get_loop_manager()->mark_loop(begin, end, 3, 1, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 1), - LoopPort::create((*multiply.first)->get_input_port(1), 1), - LoopPort::create((*add.first)->get_input_port(0), 1), - LoopPort::create((*sub.first)->get_input_port(0), 1)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 1), - LoopPort::create((*sub.first)->get_output_port(0), 1)}); + std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 1), + LoopPort::create((*multiply.first)->get_input_port(1), 1), + LoopPort::create((*add.first)->get_input_port(0), 1), + LoopPort::create((*sub.first)->get_input_port(0), 1)}, + std::vector{LoopPort::create((*add.first)->get_output_port(0), 1), + LoopPort::create((*sub.first)->get_output_port(0), 1)}); linear_ir->set_loop_depth(2); } { @@ -214,21 +214,25 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsOutputLoopUpdateNotNeed auto result1 = linear_ir_ref->push_node(sub.second); auto begin_inner = add.first; auto end_inner = result1.first; - linear_ir_ref->get_loop_manager()->mark_loop(begin_inner, end_inner, 16, vector_size, - std::vector{LoopPort::create((*add.first)->get_input_port(0), 0), - LoopPort::create((*add.first)->get_input_port(1), 0), - LoopPort::create((*sub.first)->get_input_port(0), 0)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 0), - LoopPort::create((*sub.first)->get_output_port(0), 0)}); - auto begin_outer = multiply.first; - auto end_outer = result1.first; - linear_ir_ref->get_loop_manager()->mark_loop(begin_outer, end_outer, 3, 1, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 1), - LoopPort::create((*multiply.first)->get_input_port(1), 1), - LoopPort::create((*add.first)->get_input_port(0), 1), - LoopPort::create((*sub.first)->get_input_port(0), 1)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 1), - LoopPort::create((*sub.first)->get_output_port(0), 1)}); + { + const auto entry_ports = std::vector{LoopPort::create((*add.first)->get_input_port(0), 0), + LoopPort::create((*add.first)->get_input_port(1), 0), + LoopPort::create((*sub.first)->get_input_port(0), 0)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 0), + LoopPort::create((*sub.first)->get_output_port(0), 0)}; + linear_ir_ref->get_loop_manager()->mark_loop(begin_inner, end_inner, 16, vector_size, entry_ports, exit_ports); + } + { + auto begin_outer = multiply.first; + auto end_outer = result1.first; + const auto entry_ports = std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 1), + LoopPort::create((*multiply.first)->get_input_port(1), 1), + LoopPort::create((*add.first)->get_input_port(0), 1), + LoopPort::create((*sub.first)->get_input_port(0), 1)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 1), + LoopPort::create((*sub.first)->get_output_port(0), 1)}; + linear_ir_ref->get_loop_manager()->mark_loop(begin_outer, end_outer, 3, 1, entry_ports, exit_ports); + } } } @@ -259,14 +263,20 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsFromInnermostToLoopOuts auto add = linear_ir->push_node(param_0.second, broadcastmove.second); init_expr_descriptors(*add.first, {subtensor, subtensor, subtensor}, {layout, layout, layout}); auto result = linear_ir->push_node(add.second); - linear_ir->get_loop_manager()->mark_loop(broadcastmove.first, result.first, 512, vector_size, - std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0), 0), - LoopPort::create((*add.first)->get_input_port(0), 0)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 0)}); - linear_ir->get_loop_manager()->mark_loop(broadcastmove.first, result.first, 3, 1, - std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0), 1), - LoopPort::create((*add.first)->get_input_port(0), 1)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 1)}); + + { + const auto entry_ports = std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0), 0), + LoopPort::create((*add.first)->get_input_port(0), 0)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 0)}; + linear_ir->get_loop_manager()->mark_loop(broadcastmove.first, result.first, 512, vector_size, entry_ports, exit_ports); + } + { + const auto entry_ports = std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0), 1), + LoopPort::create((*add.first)->get_input_port(0), 1)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 1)}; + linear_ir->get_loop_manager()->mark_loop(broadcastmove.first, result.first, 3, 1, entry_ports, exit_ports); + } + linear_ir->set_loop_depth(2); } { @@ -277,14 +287,19 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsFromInnermostToLoopOuts auto add = linear_ir_ref->push_node(param_0.second, broadcastmove.second); init_expr_descriptors(*add.first, {subtensor, subtensor, subtensor}, {layout, layout, layout}); auto result = linear_ir_ref->push_node(add.second); - linear_ir_ref->get_loop_manager()->mark_loop(add.first, result.first, 512, vector_size, - std::vector{LoopPort::create((*add.first)->get_input_port(0), 0), - LoopPort::create((*add.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 0)}); - linear_ir_ref->get_loop_manager()->mark_loop(add.first, result.first, 3, 1, - std::vector{LoopPort::create((*add.first)->get_input_port(0), 1), - LoopPort::create((*add.first)->get_input_port(1), 1)}, - std::vector{LoopPort::create((*add.first)->get_output_port(0), 1)}); + + { + const auto entry_ports = std::vector{LoopPort::create((*add.first)->get_input_port(0), 0), + LoopPort::create((*add.first)->get_input_port(1), 0)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 0)}; + linear_ir_ref->get_loop_manager()->mark_loop(add.first, result.first, 512, vector_size, entry_ports, exit_ports); + } + { + const auto entry_ports = std::vector{LoopPort::create((*add.first)->get_input_port(0), 1), + LoopPort::create((*add.first)->get_input_port(1), 1)}; + const auto exit_ports = std::vector{LoopPort::create((*add.first)->get_output_port(0), 1)}; + linear_ir_ref->get_loop_manager()->mark_loop(add.first, result.first, 3, 1, entry_ports, exit_ports); + } } } @@ -312,12 +327,18 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsImpossible) { init_expr_descriptors(*load_reshape.first, {subtensor, subtensor}, {order, layout}); init_expr_descriptors(*store.first, {subtensor, subtensor}, {layout, layout}); auto result = linear_ir->push_node(store.second); - linear_ir->get_loop_manager()->mark_loop(load_reshape.first, result.first, 32, 1, - std::vector{LoopPort::create((*load_reshape.first)->get_input_port(0), 0)}, - std::vector{LoopPort::create((*store.first)->get_output_port(0), 0)}); - linear_ir->get_loop_manager()->mark_loop(load_reshape.first, result.first, 1, 1, - std::vector{LoopPort::create((*load_reshape.first)->get_input_port(0), 1)}, - std::vector{LoopPort::create((*store.first)->get_output_port(0), 1)}); + + { + const auto entry_ports = std::vector{LoopPort::create((*load_reshape.first)->get_input_port(0), 0)}; + const auto exit_ports = std::vector{LoopPort::create((*store.first)->get_output_port(0), 0)}; + linear_ir->get_loop_manager()->mark_loop(load_reshape.first, result.first, 32, 1, entry_ports, exit_ports); + } + { + const auto entry_ports = std::vector{LoopPort::create((*load_reshape.first)->get_input_port(0), 1)}; + const auto exit_ports = std::vector{LoopPort::create((*store.first)->get_output_port(0), 1)}; + linear_ir->get_loop_manager()->mark_loop(load_reshape.first, result.first, 1, 1, entry_ports, exit_ports); + } + linear_ir->set_loop_depth(2); } } @@ -352,17 +373,17 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsSplitLoops) { const auto result = linear_ir->push_node(add.second); const auto& loop_manager = linear_ir->get_loop_manager(); loop_manager->mark_loop(matmul.first, broadcastmove.first, 128, block_size, 1, - std::vector{LoopPort::create((*matmul.first)->get_input_port(0)), + std::vector{LoopPort::create((*matmul.first)->get_input_port(0)), LoopPort::create((*matmul.first)->get_input_port(1))}, - std::vector{LoopPort::create((*matmul.first)->get_output_port(0))}); + std::vector{LoopPort::create((*matmul.first)->get_output_port(0))}); loop_manager->mark_loop(broadcastmove.first, result.first, 64, vector_size, 0, - std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(0))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(0))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); loop_manager->mark_loop(broadcastmove.first, result.first, 128, 1, 1, - std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(0))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*broadcastmove.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(0))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); ov::snippets::lowered::pass::SplitLoops().run(*linear_ir, linear_ir->begin(), linear_ir->end()); } { @@ -377,17 +398,17 @@ TEST_F(ExtractLoopInvariantsTest, ExtractedLoopInvariantsSplitLoops) { auto result = linear_ir_ref->push_node(add.second); const auto& loop_manager = linear_ir_ref->get_loop_manager(); loop_manager->mark_loop(matmul.first, add.first, 128, block_size, 1, - std::vector{LoopPort::create((*matmul.first)->get_input_port(0)), + std::vector{LoopPort::create((*matmul.first)->get_input_port(0)), LoopPort::create((*matmul.first)->get_input_port(1))}, - std::vector{LoopPort::create((*matmul.first)->get_output_port(0))}); + std::vector{LoopPort::create((*matmul.first)->get_output_port(0))}); loop_manager->mark_loop(add.first, result.first, 64, vector_size, 0, - std::vector{LoopPort::create((*add.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(1))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*add.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(1))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); loop_manager->mark_loop(add.first, result.first, 128, 1, 1, - std::vector{LoopPort::create((*add.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(1))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*add.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(1))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); ov::snippets::lowered::pass::SplitLoops().run(*linear_ir_ref, linear_ir_ref->begin(), linear_ir_ref->end()); } } @@ -457,19 +478,20 @@ TEST_F(ExtractLoopInvariantsRemoveLoopsTest, ExtractedLoopInvariantsAllExprsInLo auto result = linear_ir->push_node(multiply.second); // 3 inner loop linear_ir->get_loop_manager()->mark_loop(max.first, hmax.first, 1, vector_size, - std::vector{LoopPort::create((*max.first)->get_input_port(0), 0), - LoopPort::create((*max.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*max.first)->get_output_port(0), 0)}); + std::vector{LoopPort::create((*max.first)->get_input_port(0), 0), + LoopPort::create((*max.first)->get_input_port(1), 0)}, + std::vector{LoopPort::create((*max.first)->get_output_port(0), 0)}); linear_ir->get_loop_manager()->mark_loop(sub.first, hsum.first, 1, vector_size, - std::vector{LoopPort::create((*sub.first)->get_input_port(0), 0), - LoopPort::create((*sub.first)->get_input_port(1), 0), - LoopPort::create((*add.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*exp.first)->get_output_port(0), 0), - LoopPort::create((*add.first)->get_output_port(0), 0)}); + std::vector{LoopPort::create((*sub.first)->get_input_port(0), 0), + LoopPort::create((*sub.first)->get_input_port(1), 0), + LoopPort::create((*add.first)->get_input_port(1), 0)}, + std::vector{LoopPort::create((*exp.first)->get_output_port(0), 0), + LoopPort::create((*add.first)->get_output_port(0), 0)}); linear_ir->get_loop_manager()->mark_loop(multiply.first, result.first, 1, vector_size, - std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 0), - LoopPort::create((*multiply.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*multiply.first)->get_output_port(0), 0)}); + std::vector{LoopPort::create((*multiply.first)->get_input_port(0), 0), + LoopPort::create((*multiply.first)->get_input_port(1), 0)}, + std::vector{LoopPort::create( + (*multiply.first)->get_output_port(0), 0)}); // outer loop info const auto loop_begin = std::make_shared(); auto loop_begin_expr = linear_ir->insert_node(loop_begin, std::vector{}, {}, false, max.first); @@ -477,10 +499,11 @@ TEST_F(ExtractLoopInvariantsRemoveLoopsTest, ExtractedLoopInvariantsAllExprsInLo std::vector loop_end_inputs{(*loop_begin_expr)->get_output_port_connector(0)}; auto loop_end_expr = linear_ir->insert_node(loop_end, loop_end_inputs, {}, false, result.first); linear_ir->get_loop_manager()->mark_loop(loop_begin_expr, result.first, 10, 1, - std::vector{LoopPort::create((*max.first)->get_input_port(0), 1), - LoopPort::create((*max.first)->get_input_port(1), 0), - LoopPort::create((*add.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*multiply.first)->get_output_port(0), 1)}); + std::vector{LoopPort::create((*max.first)->get_input_port(0), 1), + LoopPort::create((*max.first)->get_input_port(1), 0), + LoopPort::create((*add.first)->get_input_port(1), 0)}, + std::vector{LoopPort::create( + (*multiply.first)->get_output_port(0), 1)}); loop_end->set_id((*loop_end_expr)->get_loop_ids().back()); linear_ir->set_loop_depth(2); } @@ -510,10 +533,11 @@ TEST_F(ExtractLoopInvariantsRemoveLoopsTest, ExtractedLoopInvariantsAllExprsInLo std::vector loop_end_inputs{(*loop_begin_expr)->get_output_port_connector(0)}; auto loop_end_expr = linear_ir_ref->insert_node(loop_end, loop_end_inputs, {}, false, result.first); linear_ir_ref->get_loop_manager()->mark_loop(loop_begin_expr, result.first, 10, 1, - std::vector{LoopPort::create((*max.first)->get_input_port(0), 1), - LoopPort::create((*max.first)->get_input_port(1), 0), - LoopPort::create((*add.first)->get_input_port(1), 0)}, - std::vector{LoopPort::create((*multiply.first)->get_output_port(0), 1)}); + std::vector{LoopPort::create((*max.first)->get_input_port(0), 1), + LoopPort::create((*max.first)->get_input_port(1), 0), + LoopPort::create((*add.first)->get_input_port(1), 0)}, + std::vector{LoopPort::create( + (*multiply.first)->get_output_port(0), 1)}); loop_end->set_id((*loop_end_expr)->get_loop_ids().back()); } } diff --git a/src/common/snippets/tests/src/lowered/pass/loop.cpp b/src/common/snippets/tests/src/lowered/pass/loop.cpp index 76e747150f7ea1..651bc66d2949b7 100644 --- a/src/common/snippets/tests/src/lowered/pass/loop.cpp +++ b/src/common/snippets/tests/src/lowered/pass/loop.cpp @@ -42,17 +42,17 @@ static void init_linear_ir(const std::vector& in_shapes, LinearIR& li const auto loop_manager = linear_ir.get_loop_manager(); linear_ir.get_loop_manager()->mark_loop(matmul.first, add.first, in_shapes[0].front(), block_size, - std::vector{LoopPort::create((*matmul.first)->get_input_port(0), 1), + std::vector{LoopPort::create((*matmul.first)->get_input_port(0), 1), LoopPort::create((*matmul.first)->get_input_port(1))}, - std::vector{LoopPort::create((*matmul.first)->get_output_port(0), 1)}); + std::vector{LoopPort::create((*matmul.first)->get_output_port(0), 1)}); linear_ir.get_loop_manager()->mark_loop(add.first, result.first, in_shapes[2].back(), vector_size, 0, - std::vector{LoopPort::create((*add.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(1))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*add.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(1))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); linear_ir.get_loop_manager()->mark_loop(add.first, result.first, in_shapes[2].front(), 1, 1, - std::vector{LoopPort::create((*add.first)->get_input_port(0)), - LoopPort::create((*add.first)->get_input_port(1))}, - std::vector{LoopPort::create((*add.first)->get_output_port(0))}); + std::vector{LoopPort::create((*add.first)->get_input_port(0)), + LoopPort::create((*add.first)->get_input_port(1))}, + std::vector{LoopPort::create((*add.first)->get_output_port(0))}); } static void apply_transformations(LinearIR& linear_ir, const std::shared_ptr& config) { diff --git a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_base.cpp b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_base.cpp index cb6abe3ad8bfa0..8b3ed792fce535 100644 --- a/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_base.cpp +++ b/src/plugins/intel_cpu/src/emitters/snippets/x64/kernel_executors/brgemm_base.cpp @@ -182,10 +182,6 @@ void BrgemmBaseKernelExecutor::update_config(const ov::snippets::lowered::Expres return loop_manager->get_loop_info(loop_ids[loop_idx++]); }; - auto is_processed_port = [](const ov::snippets::lowered::LoopPort& p) { - return p.get_type() != ov::snippets::lowered::LoopPort::Type::NotProcessed; - }; - /* ------- Dimension M ----------*/ if (ov::snippets::utils::is_full_dim_value(M)) { M = *++in0_shape.rbegin(); @@ -197,7 +193,7 @@ void BrgemmBaseKernelExecutor::update_config(const ov::snippets::lowered::Expres // If BrgemmCopyB in the Loop by M -> first input port will be BrgemmCopyB with `incremented=false` // to avoid extra checks, we validate only first input port auto check_port = [&](const ov::snippets::lowered::LoopPort& p) { - return p.get_dim_idx() == 1 && is_processed_port(p); + return p.get_dim_idx() == 1 && p.is_processed(); }; OPENVINO_ASSERT( in_ports.size() > 1 && check_port(in_ports[0]) && out_ports.size() == 1 && check_port(out_ports[0]), @@ -216,9 +212,9 @@ void BrgemmBaseKernelExecutor::update_config(const ov::snippets::lowered::Expres const auto& out_ports = current_expanded_loop_info->get_output_ports(); // Quick validation check: Should we check that port is really Brgemm port? auto check_port = [&](const ov::snippets::lowered::LoopPort& p) { - return p.get_dim_idx() == 0 && is_processed_port(p); + return p.get_dim_idx() == 0 && p.is_processed(); }; - OPENVINO_ASSERT(in_ports.size() >= 2 && !is_processed_port(in_ports.front()) && + OPENVINO_ASSERT(in_ports.size() >= 2 && !in_ports.front().is_processed() && std::all_of(in_ports.cbegin() + 1, in_ports.cend(), check_port) && out_ports.size() == 1 && check_port(out_ports.back()), "Incorrect Loop by Brgemm dimension N"); @@ -241,9 +237,9 @@ void BrgemmBaseKernelExecutor::update_config(const ov::snippets::lowered::Expres const auto& out_ports = current_expanded_loop_info->get_output_ports(); // Quick validation check: Should we check that port is really Brgemm port? OPENVINO_ASSERT(in_ports.size() >= 2 && in_ports.front().get_dim_idx() == 0 && - is_processed_port(in_ports.front()) && in_ports.back().get_dim_idx() == 1 && - is_processed_port(in_ports.back()) && out_ports.size() == 1 && - !is_processed_port(out_ports.front()), + in_ports.front().is_processed() && in_ports.back().get_dim_idx() == 1 && + in_ports.back().is_processed() && out_ports.size() == 1 && + !out_ports.front().is_processed(), "Incorrect Loop by Brgemm dimension K"); K = current_expanded_loop_info->get_work_amount() > 0 ? current_expanded_loop_info->get_increment() : 0; input_pds[0]->set_subtensor_dim(0, K); diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp index f99ec2da6776e8..2be52cdd37e8b9 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/adjust_brgemm_copy_b_loop_ports.cpp @@ -31,7 +31,7 @@ bool pass::AdjustBrgemmCopyBLoopPorts::update_loop_info( * 2) Zero padding is applied if N4k < 256 or N2k < 64 */ if (brgemm_utils::with_repacking(brg->get_type()) && precision != element::f32 && - loop_port.get_type() == ov::snippets::lowered::LoopPort::Type::Incremented) { + loop_port.is_incremented()) { // K blocking loop: account for zero padding if (loop_port.get_dim_idx() == 1) { const auto ptr_incr = loop_desc.ptr_increment; diff --git a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp index d1e3a9799b1609..dcd97fdd74b638 100644 --- a/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp +++ b/src/plugins/intel_cpu/src/transformations/tpp/x64/pass/lowered/set_tpp_leading_dim.cpp @@ -40,7 +40,7 @@ bool has_directly_connected_buffer(const ExpressionPort& port, const snippets::l loop_info->get_input_ports() : loop_info->get_output_ports(); const auto& found = std::find_if(border_points.begin(), border_points.end(), pred); - if (found == border_points.end() || found->get_type() == LoopPort::Type::Incremented) + if (found == border_points.end() || found->is_incremented()) return false; } return true; diff --git a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp index 40d3894c872ff1..9f3b532bf0312a 100644 --- a/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp +++ b/src/plugins/intel_cpu/tests/unit/snippets_transformations/x64/lowered/brgemm_blocking.cpp @@ -56,8 +56,8 @@ void create_brgemm_loop_infos(const LinearIRPtr& linear_ir, if (k_block) { const auto loop_info = std::make_shared(k, k_blk, - std::vector{LoopPort::create(brgemm_expr->get_input_port(0), 0), - LoopPort::create(brgemm_expr->get_input_port(1), 1)}, + std::vector{LoopPort::create(brgemm_expr->get_input_port(0), 0), + LoopPort::create(brgemm_expr->get_input_port(1), 1)}, std::vector{LoopPort::create(brgemm_expr->get_output_port(0))}, get_k_loop_handlers(k, k_block, backend)); linear_ir->get_loop_manager()->add_loop_info(loop_info); @@ -66,18 +66,18 @@ void create_brgemm_loop_infos(const LinearIRPtr& linear_ir, linear_ir->get_loop_manager()->add_loop_info( std::make_shared(n, n_blk, std::vector{LoopPort::create(brgemm_expr->get_input_port(0)), - LoopPort::create(brgemm_expr->get_input_port(1))}, - std::vector{LoopPort::create(brgemm_expr->get_output_port(0))}, + LoopPort::create(brgemm_expr->get_input_port(1))}, + std::vector{LoopPort::create(brgemm_expr->get_output_port(0))}, BrgemmBlockingBase::get_default_blocking_loop_handlers(n, n_block))); } if (m_block) { - std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 1)}; + std::vector entries{LoopPort::create(brgemm_expr->get_input_port(0), 1)}; for (size_t i = 1; i < brgemm_expr->get_input_count(); ++i) entries.push_back(LoopPort::create(brgemm_expr->get_input_port(i))); linear_ir->get_loop_manager()->add_loop_info( std::make_shared(m, m_blk, entries, - std::vector{LoopPort::create(brgemm_expr->get_output_port(0), 1)}, + std::vector{LoopPort::create(brgemm_expr->get_output_port(0), 1)}, BrgemmBlockingBase::get_default_blocking_loop_handlers(m, m_block))); } } @@ -314,9 +314,9 @@ TEST_F(BrgemmCPUBlockingTest, AMX) { init_expr_descriptors(brgemm_expr, {{m_blk, k_blk}, {k_blk, n_blk}, get_default_subtensor(), {m_blk, n_blk}}); create_brgemm_loop_infos(linear_ir_ref, brgemm_expr, m, 0, k, k_blk, n, n_blk); - std::vector entries {LoopPort::create(brgemm_expr->get_input_port(0), 1), + std::vector entries {LoopPort::create(brgemm_expr->get_input_port(0), 1), LoopPort::create(brgemm_expr->get_input_port(1))}; - std::vector exits {LoopPort::create(brgemm_expr->get_output_port(0), 1)}; + std::vector exits {LoopPort::create(brgemm_expr->get_output_port(0), 1)}; auto handlers = BrgemmBlockingBase::get_default_blocking_loop_handlers(m, m_blk); linear_ir_ref->get_loop_manager()-> add_loop_info(std::make_shared(m, m_blk, entries, exits, handlers));