From 33aca7384250b46becbdd3dce66443b4e893a32a Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Thu, 7 Nov 2024 17:47:50 +0100 Subject: [PATCH 1/6] fix ts unsqueeze backward --- .../transpose_sinking/ts_unsqueeze.cpp | 8 ++++++ .../transpose_sinking/ts_common_test.cpp | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp index cdeb9226ed236c..5448fa1e9bb0a0 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -178,6 +178,14 @@ TSUnsqueezeBackward::TSUnsqueezeBackward() { return false; } + // if main_node does nothing, just remove it + if (main_node->get_input_partial_shape(0) == main_node->get_output_partial_shape(0)) { + auto parent_node = main_node->get_input_node_shared_ptr(0); + main_node->output(0).replace(parent_node->output(0)); + register_new_node(transpose); + return true; + } + auto transpose_order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); auto unsqueeze_axes = ov::as_type_ptr(main_node->get_input_node_shared_ptr(1)); if (!transpose_order || !unsqueeze_axes) diff --git a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp index d71c9006edd38a..9f8f543a0cc528 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp @@ -1677,6 +1677,31 @@ auto test_backward_unsqueeze_dyn_rank = []() { INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonUnsqueezeBackwardDynRank, TSTestFixture, test_backward_unsqueeze_dyn_rank()); + +TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameShape) { + auto create_transpose = [](const std::shared_ptr& parent) { + auto ts_order = std::make_shared(element::u64, Shape{3}, Shape{1, 0, 2}); + return std::make_shared(parent, ts_order); + }; + + const Shape input_shape = {4, 5, 6}; + { + auto X = std::make_shared(element::f32, input_shape); + auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); + auto reshape = std::make_shared(X, reshape_const, false); + auto transpose = create_transpose(reshape); + model = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); + } + + { + auto X = std::make_shared(element::f32, input_shape); + auto transpose = create_transpose(X); + model_ref = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); + } + + manager.register_pass(); +} + } // namespace common } // namespace testing } // namespace transpose_sinking From f79bf724cc410e5d5e2762cd9a91d136856c6b9b Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Tue, 12 Nov 2024 16:13:10 +0100 Subject: [PATCH 2/6] code review fixes --- .../transpose_sinking/ts_unsqueeze.cpp | 36 ++++++++++++++----- .../transpose_sinking/ts_common_test.cpp | 29 +++++++++++++-- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp index 5448fa1e9bb0a0..d0fd057c4a1b3f 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -158,6 +158,23 @@ TSUnsqueezeForward::TSUnsqueezeForward() { transpose_sinking(matcher_name, sinking_transformation); } +namespace { +bool AreInputOutputShapesEqual(const std::shared_ptr& reshape) { + const auto input_shape = reshape->get_input_partial_shape(0); + const auto output_shape = reshape->get_output_partial_shape(0); + + if (input_shape.is_dynamic() || output_shape.is_dynamic()) { + return false; + } + return input_shape == output_shape; +} + +bool HasSpecialOne(const std::shared_ptr& reshape_const) { + auto const_value = reshape_const->cast_vector(); + return std::find(const_value.begin(), const_value.end(), -1) != const_value.end(); +} +} // namespace + TSUnsqueezeBackward::TSUnsqueezeBackward() { MATCHER_SCOPE(TSUnsqueezeBackward); @@ -178,19 +195,22 @@ TSUnsqueezeBackward::TSUnsqueezeBackward() { return false; } - // if main_node does nothing, just remove it - if (main_node->get_input_partial_shape(0) == main_node->get_output_partial_shape(0)) { - auto parent_node = main_node->get_input_node_shared_ptr(0); - main_node->output(0).replace(parent_node->output(0)); - register_new_node(transpose); - return true; - } - auto transpose_order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); auto unsqueeze_axes = ov::as_type_ptr(main_node->get_input_node_shared_ptr(1)); if (!transpose_order || !unsqueeze_axes) return false; + // if main_node does nothing, just remove it + auto reshape = as_type_ptr(main_node); + if (reshape && AreInputOutputShapesEqual(reshape) && !HasSpecialOne(unsqueeze_axes)) { + for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_order, {0})) { + register_new_node(new_node); + } + main_node->validate_and_infer_types(); + RemoveTransposeConsumers(main_node); + return true; + } + std::vector non_negative_axes; if (as_type_ptr(main_node)) { auto success = shape_to_unsqueeze_axes(main_node, unsqueeze_axes, non_negative_axes); diff --git a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp index 9f8f543a0cc528..5cb41f9891facb 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp @@ -1683,12 +1683,15 @@ TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameS auto ts_order = std::make_shared(element::u64, Shape{3}, Shape{1, 0, 2}); return std::make_shared(parent, ts_order); }; + auto create_reshape = [](const std::shared_ptr& parent) { + auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); + return std::make_shared(parent, reshape_const, false); + }; const Shape input_shape = {4, 5, 6}; { auto X = std::make_shared(element::f32, input_shape); - auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); - auto reshape = std::make_shared(X, reshape_const, false); + auto reshape = create_reshape(X); auto transpose = create_transpose(reshape); model = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); } @@ -1696,12 +1699,32 @@ TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameS { auto X = std::make_shared(element::f32, input_shape); auto transpose = create_transpose(X); - model_ref = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); + auto reshape = create_reshape(transpose); + model_ref = std::make_shared(ov::OutputVector{reshape}, ov::ParameterVector{X}); } manager.register_pass(); } +TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameShapeSpecialOne) { + auto create_transpose = [](const std::shared_ptr& parent) { + auto ts_order = std::make_shared(element::u64, Shape{3}, Shape{1, 0, 2}); + return std::make_shared(parent, ts_order); + }; + + { + auto X = std::make_shared(element::f32, Shape{4, 5, 6}); + auto reshape_const = std::make_shared(element::i64, Shape{3}, std::vector{4, 5, -1}); + auto reshape = std::make_shared(X, reshape_const, false); + auto transpose = create_transpose(reshape); + model = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); + } + + model_ref = model->clone(); + + manager.register_pass(); +} + } // namespace common } // namespace testing } // namespace transpose_sinking From 1ea0128711e01b1538cd1346d44dad14e0fefe78 Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Tue, 12 Nov 2024 16:14:37 +0100 Subject: [PATCH 3/6] cleanup --- .../src/transformations/transpose_sinking/ts_unsqueeze.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp index d0fd057c4a1b3f..5bb56d0522fed0 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -200,7 +200,7 @@ TSUnsqueezeBackward::TSUnsqueezeBackward() { if (!transpose_order || !unsqueeze_axes) return false; - // if main_node does nothing, just remove it + // if main_node does nothing, just swap them auto reshape = as_type_ptr(main_node); if (reshape && AreInputOutputShapesEqual(reshape) && !HasSpecialOne(unsqueeze_axes)) { for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_order, {0})) { From a0c2cf35b0c2f94e337ff243f62ad203a6072469 Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Tue, 12 Nov 2024 19:17:20 +0100 Subject: [PATCH 4/6] fix --- .../transpose_sinking/ts_unsqueeze.cpp | 12 ++++++++++-- .../tests/transpose_sinking/ts_common_test.cpp | 13 +++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp index 5bb56d0522fed0..baf450d13bd88e 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -14,7 +14,6 @@ #include "openvino/op/transpose.hpp" #include "openvino/op/unsqueeze.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" -#include "transformations/rt_info/transpose_sinking_attr.hpp" #include "transformations/transpose_sinking/ts_utils.hpp" #include "transformations/utils/utils.hpp" @@ -200,12 +199,22 @@ TSUnsqueezeBackward::TSUnsqueezeBackward() { if (!transpose_order || !unsqueeze_axes) return false; + auto transpose_order_values = transpose_order->cast_vector(); + // if main_node does nothing, just swap them auto reshape = as_type_ptr(main_node); if (reshape && AreInputOutputShapesEqual(reshape) && !HasSpecialOne(unsqueeze_axes)) { + // insert Transpose before main_node on #0 input for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_order, {0})) { register_new_node(new_node); } + // transpose reshape const with Gather operation + auto axis = std::make_shared(element::i32, Shape{}, 0); + auto gather = ov::pass::transpose_sinking::utils::ChangeValuesOrder(reshape->input_value(1), + transpose_order_values, + axis); + main_node->input(1).replace_source_output(gather); + main_node->validate_and_infer_types(); RemoveTransposeConsumers(main_node); return true; @@ -233,7 +242,6 @@ TSUnsqueezeBackward::TSUnsqueezeBackward() { } } - auto transpose_order_values = transpose_order->cast_vector(); auto old_transpose_order_values = transpose_order_values; std::vector new_values; diff --git a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp index 5cb41f9891facb..3b5fe2f4e7db6b 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp @@ -1683,15 +1683,12 @@ TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameS auto ts_order = std::make_shared(element::u64, Shape{3}, Shape{1, 0, 2}); return std::make_shared(parent, ts_order); }; - auto create_reshape = [](const std::shared_ptr& parent) { - auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); - return std::make_shared(parent, reshape_const, false); - }; const Shape input_shape = {4, 5, 6}; { auto X = std::make_shared(element::f32, input_shape); - auto reshape = create_reshape(X); + auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); + auto reshape = std::make_shared(X, reshape_const, false); auto transpose = create_transpose(reshape); model = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); } @@ -1699,7 +1696,11 @@ TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameS { auto X = std::make_shared(element::f32, input_shape); auto transpose = create_transpose(X); - auto reshape = create_reshape(transpose); + auto reshape_const = std::make_shared(element::u64, Shape{3}, Shape{4, 5, 6}); + auto axis = std::make_shared(element::i32, Shape{}, 0); + auto indices = std::make_shared(element::i32, Shape{3}, Shape{1, 0, 2}); + auto gather = std::make_shared(reshape_const, indices, axis); + auto reshape = std::make_shared(transpose, gather, false); model_ref = std::make_shared(ov::OutputVector{reshape}, ov::ParameterVector{X}); } From 39264b2650f9e0804ebe4a2c5d808d0f30164efe Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Wed, 13 Nov 2024 16:48:33 +0100 Subject: [PATCH 5/6] add to forward transformation --- .../transpose_sinking/ts_unsqueeze.cpp | 56 +++++++++++++------ .../transpose_sinking/ts_common_test.cpp | 48 ++++++++++++++++ 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp index baf450d13bd88e..ce47caa10c4c0f 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -98,6 +98,22 @@ bool unsqueeze_axes_to_shape(const Output& input_node, } return true; } + +bool AreInputOutputShapesEqual(const std::shared_ptr& reshape) { + const auto input_shape = reshape->get_input_partial_shape(0); + const auto output_shape = reshape->get_output_partial_shape(0); + + if (input_shape.is_dynamic() || output_shape.is_dynamic()) { + return false; + } + return input_shape == output_shape; +} + +bool HasSpecialOne(const std::shared_ptr& reshape_const) { + auto const_value = reshape_const->cast_vector(); + return std::find(const_value.begin(), const_value.end(), -1) != const_value.end(); +} + } // namespace TSUnsqueezeForward::TSUnsqueezeForward() { @@ -111,6 +127,28 @@ TSUnsqueezeForward::TSUnsqueezeForward() { if (!unsqueeze_axes) { return false; } + auto ts_order_values = transpose_info.transpose_const->cast_vector(); + + // if main_node does nothing, just swap them + auto reshape = as_type_ptr(main_node); + if (reshape && AreInputOutputShapesEqual(reshape) && !HasSpecialOne(unsqueeze_axes)) { + TransposeInputsInfo transpose_input_info = {transpose_info.transpose, transpose_info.transpose_const, 0}; + // remove input Transpose + auto success = sink_forward::UpdateInputTransposes(main_node, transpose_input_info, {0}); + if (!success) { + return false; + } + + const auto reshape_order = ov::pass::transpose_sinking::utils::ReverseTransposeOrder(ts_order_values); + // transpose reshape const with Gather operation + auto axis = std::make_shared(element::i32, Shape{}, 0); + auto gather = + ov::pass::transpose_sinking::utils::ChangeValuesOrder(reshape->input_value(1), reshape_order, axis); + main_node->input(1).replace_source_output(gather); + + default_outputs_update(main_node, transpose_input_info); + return true; + } std::vector non_negative_axes; if (as_type_ptr(main_node)) { @@ -123,7 +161,6 @@ TSUnsqueezeForward::TSUnsqueezeForward() { non_negative_axes = ov::util::try_get_normalized_axis_vector(unsqueeze_axes->get_tensor_view(), rank, *main_node); } - auto ts_order_values = transpose_info.transpose_const->cast_vector(); ts_order_values = GetOrderBeforeReduction(non_negative_axes, ts_order_values); auto new_transpose_order = ov::op::v0::Constant::create(transpose_info.transpose_const->get_element_type(), @@ -157,23 +194,6 @@ TSUnsqueezeForward::TSUnsqueezeForward() { transpose_sinking(matcher_name, sinking_transformation); } -namespace { -bool AreInputOutputShapesEqual(const std::shared_ptr& reshape) { - const auto input_shape = reshape->get_input_partial_shape(0); - const auto output_shape = reshape->get_output_partial_shape(0); - - if (input_shape.is_dynamic() || output_shape.is_dynamic()) { - return false; - } - return input_shape == output_shape; -} - -bool HasSpecialOne(const std::shared_ptr& reshape_const) { - auto const_value = reshape_const->cast_vector(); - return std::find(const_value.begin(), const_value.end(), -1) != const_value.end(); -} -} // namespace - TSUnsqueezeBackward::TSUnsqueezeBackward() { MATCHER_SCOPE(TSUnsqueezeBackward); diff --git a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp index 3b5fe2f4e7db6b..fc5c315312cfaa 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp @@ -1726,6 +1726,54 @@ TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeBackwardSameS manager.register_pass(); } +TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeForwardSameShape) { + auto create_transpose = [](const std::shared_ptr& parent) { + auto ts_order = std::make_shared(element::u64, Shape{4}, Shape{1, 3, 0, 2}); + return std::make_shared(parent, ts_order); + }; + + const Shape input_shape = {4, 5, 6, 7}; + { + auto X = std::make_shared(element::f32, input_shape); + auto transpose = create_transpose(X); + auto reshape_const = std::make_shared(element::u64, Shape{4}, Shape{5, 7, 4, 6}); + auto reshape = std::make_shared(transpose, reshape_const, false); + model = std::make_shared(ov::OutputVector{reshape}, ov::ParameterVector{X}); + } + + { + auto X = std::make_shared(element::f32, input_shape); + auto reshape_const = std::make_shared(element::u64, Shape{4}, Shape{5, 7, 4, 6}); + auto axis = std::make_shared(element::i32, Shape{}, 0); + auto indices = std::make_shared(element::i32, Shape{4}, Shape{2, 0, 3, 1}); + auto gather = std::make_shared(reshape_const, indices, axis); + auto reshape = std::make_shared(X, gather, false); + auto transpose = create_transpose(reshape); + model_ref = std::make_shared(ov::OutputVector{transpose}, ov::ParameterVector{X}); + } + + manager.register_pass(); +} + +TEST_F(TransformationTestsF, TransposeSinkingCommonReshapeUnsqueezeForwardSameShapeSpecialOne) { + auto create_transpose = [](const std::shared_ptr& parent) { + auto ts_order = std::make_shared(element::u64, Shape{3}, Shape{1, 0, 2}); + return std::make_shared(parent, ts_order); + }; + + { + auto X = std::make_shared(element::f32, Shape{4, 5, 6}); + auto transpose = create_transpose(X); + auto reshape_const = std::make_shared(element::i64, Shape{3}, std::vector{4, 5, -1}); + auto reshape = std::make_shared(transpose, reshape_const, false); + model = std::make_shared(ov::OutputVector{reshape}, ov::ParameterVector{X}); + } + + model_ref = model->clone(); + + manager.register_pass(); +} + } // namespace common } // namespace testing } // namespace transpose_sinking From 2074a7d1b8666c16704581eaf209eb03519bc66f Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Thu, 14 Nov 2024 10:48:56 +0100 Subject: [PATCH 6/6] fix unit test --- .../tests/transpose_sinking/ts_general_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp index f00c69d2a8d734..7dc3a2b54c7bea 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp @@ -380,7 +380,7 @@ TEST_F(TransformationTestsF, TSGeneralTestMultipleTypes) { auto ng_order0 = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{0, 2, 3, 1}); auto transpose0 = std::make_shared(node0, ng_order0); - auto reshape_const = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{1, 40, 55, 96}); + auto reshape_const = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{2, 20, 55, 96}); auto reshape = std::make_shared(transpose0, reshape_const, false); auto ng_order1 = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{0, 3, 1, 2}); @@ -399,7 +399,7 @@ TEST_F(TransformationTestsF, TSGeneralTestMultipleTypes) { auto ng_order0 = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{0, 2, 3, 1}); auto transpose0 = std::make_shared(node0, ng_order0); - auto reshape_const = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{1, 40, 55, 96}); + auto reshape_const = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{2, 20, 55, 96}); auto reshape = std::make_shared(transpose0, reshape_const, false); auto node1 = MakeAllNodesSubgraph(reshape, 3, 3);