diff --git a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp index 52ba997e6ac2c5..dbb6608b50f0b5 100644 --- a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp +++ b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp @@ -15,6 +15,7 @@ #include "pyopenvino/core/common.hpp" #include "pyopenvino/core/infer_request.hpp" +#include "pyopenvino/utils/utils.hpp" namespace py = pybind11; @@ -64,7 +65,7 @@ class AsyncInferQueue { }); size_t idle_handle = m_idle_handles.front(); // wait for request to make sure it returned from callback - m_requests[idle_handle].m_request.wait(); + m_requests[idle_handle].m_request->wait(); if (m_errors.size() > 0) throw m_errors.front(); return idle_handle; @@ -75,7 +76,7 @@ class AsyncInferQueue { // release GIL to avoid deadlock on python callback py::gil_scoped_release release; for (auto&& request : m_requests) { - request.m_request.wait(); + request.m_request->wait(); } // acquire the mutex to access m_errors std::lock_guard lock(m_mutex); @@ -87,7 +88,7 @@ class AsyncInferQueue { for (size_t handle = 0; handle < m_requests.size(); handle++) { // auto end_time = m_requests[handle].m_end_time; // TODO: pass it bellow? like in InferRequestWrapper - m_requests[handle].m_request.set_callback([this, handle /* ... */](std::exception_ptr exception_ptr) { + m_requests[handle].m_request->set_callback([this, handle /* ... */](std::exception_ptr exception_ptr) { *m_requests[handle].m_end_time = Time::now(); { // acquire the mutex to access m_idle_handles @@ -110,14 +111,17 @@ class AsyncInferQueue { } void set_custom_callbacks(py::function f_callback) { + // need to acquire GIL before py::function deletion + auto callback_sp = Common::utils::wrap_pyfunction(std::move(f_callback)); + for (size_t handle = 0; handle < m_requests.size(); handle++) { - m_requests[handle].m_request.set_callback([this, f_callback, handle](std::exception_ptr exception_ptr) { + m_requests[handle].m_request->set_callback([this, callback_sp, handle](std::exception_ptr exception_ptr) { *m_requests[handle].m_end_time = Time::now(); if (exception_ptr == nullptr) { // Acquire GIL, execute Python function py::gil_scoped_acquire acquire; try { - f_callback(m_requests[handle], m_user_ids[handle]); + (*callback_sp)(m_requests[handle], m_user_ids[handle]); } catch (const py::error_already_set& py_error) { // This should behave the same as assert(!PyErr_Occurred()) // since constructor for pybind11's error_already_set is @@ -193,13 +197,13 @@ void regclass_AsyncInferQueue(py::module m) { // Set new inputs label/id from user self.m_user_ids[handle] = userdata; // Update inputs if there are any - self.m_requests[handle].m_request.set_input_tensor(inputs); + self.m_requests[handle].m_request->set_input_tensor(inputs); // Now GIL can be released - we are NOT working with Python objects in this block { py::gil_scoped_release release; *self.m_requests[handle].m_start_time = Time::now(); // Start InferRequest in asynchronus mode - self.m_requests[handle].m_request.start_async(); + self.m_requests[handle].m_request->start_async(); } }, py::arg("inputs"), @@ -239,13 +243,13 @@ void regclass_AsyncInferQueue(py::module m) { // Set new inputs label/id from user self.m_user_ids[handle] = userdata; // Update inputs if there are any - Common::set_request_tensors(self.m_requests[handle].m_request, inputs); + Common::set_request_tensors(*self.m_requests[handle].m_request, inputs); // Now GIL can be released - we are NOT working with Python objects in this block { py::gil_scoped_release release; *self.m_requests[handle].m_start_time = Time::now(); // Start InferRequest in asynchronus mode - self.m_requests[handle].m_request.start_async(); + self.m_requests[handle].m_request->start_async(); } }, py::arg("inputs"), diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp index 9f57b794e2bff6..179002127960cd 100644 --- a/src/bindings/python/src/pyopenvino/core/common.cpp +++ b/src/bindings/python/src/pyopenvino/core/common.cpp @@ -433,10 +433,14 @@ ov::op::v0::Constant create_shared(py::array& array) { // If ndim is equal to 0, creates scalar Constant. // If size is equal to 0, creates empty Constant. if (array_helpers::is_contiguous(array)) { - auto memory = std::make_shared>( + auto buffer = new ov::SharedBuffer( static_cast((array.ndim() == 0 || array.size() == 0) ? array.mutable_data() : array.mutable_data(0)), array.ndim() == 0 ? array.itemsize() : array.nbytes(), array); + std::shared_ptr> memory(buffer, [](ov::SharedBuffer* buffer) { + py::gil_scoped_acquire acquire; + delete buffer; + }); return ov::op::v0::Constant(type_helpers::get_ov_type(array), array_helpers::get_shape(array), memory); } // If passed array is not C-style, throw an error. @@ -614,7 +618,7 @@ uint32_t get_optimal_number_of_requests(const ov::CompiledModel& actual) { py::dict outputs_to_dict(InferRequestWrapper& request, bool share_outputs, bool decode_strings) { py::dict res; for (const auto& out : request.m_outputs) { - auto t = request.m_request.get_tensor(out); + auto t = request.m_request->get_tensor(out); if (t.get_element_type() == ov::element::string) { if (share_outputs) { PyErr_WarnEx(PyExc_RuntimeWarning, "Result of a string type will be copied to OVDict!", 1); diff --git a/src/bindings/python/src/pyopenvino/core/infer_request.cpp b/src/bindings/python/src/pyopenvino/core/infer_request.cpp index 93a52b1dad681f..9f572d273dc5f3 100644 --- a/src/bindings/python/src/pyopenvino/core/infer_request.cpp +++ b/src/bindings/python/src/pyopenvino/core/infer_request.cpp @@ -18,7 +18,7 @@ inline py::object run_sync_infer(InferRequestWrapper& self, bool share_outputs, { py::gil_scoped_release release; *self.m_start_time = Time::now(); - self.m_request.infer(); + self.m_request->infer(); *self.m_end_time = Time::now(); } return Common::outputs_to_dict(self, share_outputs, decode_strings); @@ -38,7 +38,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensors", [](InferRequestWrapper& self, const py::dict& inputs) { - Common::set_request_tensors(self.m_request, inputs); + Common::set_request_tensors(*self.m_request, inputs); }, py::arg("inputs"), R"( @@ -51,7 +51,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensors", [](InferRequestWrapper& self, const std::string& tensor_name, const std::vector& tensors) { - self.m_request.set_tensors(tensor_name, tensors); + self.m_request->set_tensors(tensor_name, tensors); }, py::arg("tensor_name"), py::arg("tensors"), @@ -73,7 +73,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensors", [](InferRequestWrapper& self, const ov::Output& port, const std::vector& tensors) { - self.m_request.set_tensors(port, tensors); + self.m_request->set_tensors(port, tensors); }, py::arg("port"), py::arg("tensors"), @@ -100,7 +100,7 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self, const py::dict& outputs) { auto outputs_map = Common::containers::cast_to_tensor_index_map(outputs); for (auto&& output : outputs_map) { - self.m_request.set_output_tensor(output.first, output.second); + self.m_request->set_output_tensor(output.first, output.second); } }, py::arg("outputs"), @@ -117,7 +117,7 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self, const py::dict& inputs) { auto inputs_map = Common::containers::cast_to_tensor_index_map(inputs); for (auto&& input : inputs_map) { - self.m_request.set_input_tensor(input.first, input.second); + self.m_request->set_input_tensor(input.first, input.second); } }, py::arg("inputs"), @@ -131,7 +131,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_input_tensors", [](InferRequestWrapper& self, const std::vector& tensors) { - self.m_request.set_input_tensors(tensors); + self.m_request->set_input_tensors(tensors); }, py::arg("tensors"), R"( @@ -148,7 +148,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_input_tensors", [](InferRequestWrapper& self, size_t idx, const std::vector& tensors) { - self.m_request.set_input_tensors(idx, tensors); + self.m_request->set_input_tensors(idx, tensors); }, py::arg("idx"), py::arg("tensors"), @@ -168,7 +168,7 @@ void regclass_InferRequest(py::module m) { cls.def( "infer", [](InferRequestWrapper& self, const ov::Tensor& inputs, bool share_outputs, bool decode_strings) { - self.m_request.set_input_tensor(inputs); + self.m_request->set_input_tensor(inputs); return run_sync_infer(self, share_outputs, decode_strings); }, py::arg("inputs"), @@ -197,7 +197,7 @@ void regclass_InferRequest(py::module m) { "infer", [](InferRequestWrapper& self, const py::dict& inputs, bool share_outputs, bool decode_strings) { // Update inputs if there are any - Common::set_request_tensors(self.m_request, inputs); + Common::set_request_tensors(*self.m_request, inputs); // Call Infer function return run_sync_infer(self, share_outputs, decode_strings); }, @@ -222,7 +222,7 @@ void regclass_InferRequest(py::module m) { "start_async", [](InferRequestWrapper& self, const ov::Tensor& inputs, py::object& userdata) { // Update inputs if there are any - self.m_request.set_input_tensor(inputs); + self.m_request->set_input_tensor(inputs); if (!userdata.is(py::none())) { if (self.m_user_callback_defined) { self.m_userdata = userdata; @@ -232,7 +232,7 @@ void regclass_InferRequest(py::module m) { } py::gil_scoped_release release; *self.m_start_time = Time::now(); - self.m_request.start_async(); + self.m_request->start_async(); }, py::arg("inputs"), py::arg("userdata"), @@ -261,7 +261,7 @@ void regclass_InferRequest(py::module m) { "start_async", [](InferRequestWrapper& self, const py::dict& inputs, py::object& userdata) { // Update inputs if there are any - Common::set_request_tensors(self.m_request, inputs); + Common::set_request_tensors(*self.m_request, inputs); if (!userdata.is(py::none())) { if (self.m_user_callback_defined) { self.m_userdata = userdata; @@ -271,7 +271,7 @@ void regclass_InferRequest(py::module m) { } py::gil_scoped_release release; *self.m_start_time = Time::now(); - self.m_request.start_async(); + self.m_request->start_async(); }, py::arg("inputs"), py::arg("userdata"), @@ -293,7 +293,7 @@ void regclass_InferRequest(py::module m) { cls.def( "cancel", [](InferRequestWrapper& self) { - self.m_request.cancel(); + self.m_request->cancel(); }, R"( Cancels inference request. @@ -303,7 +303,7 @@ void regclass_InferRequest(py::module m) { "wait", [](InferRequestWrapper& self) { py::gil_scoped_release release; - self.m_request.wait(); + self.m_request->wait(); }, R"( Waits for the result to become available. @@ -316,7 +316,7 @@ void regclass_InferRequest(py::module m) { "wait_for", [](InferRequestWrapper& self, const int timeout) { py::gil_scoped_release release; - return self.m_request.wait_for(std::chrono::milliseconds(timeout)); + return self.m_request->wait_for(std::chrono::milliseconds(timeout)); }, py::arg("timeout"), R"( @@ -337,7 +337,11 @@ void regclass_InferRequest(py::module m) { [](InferRequestWrapper& self, py::function callback, py::object& userdata) { self.m_userdata = userdata; self.m_user_callback_defined = true; - self.m_request.set_callback([&self, callback](std::exception_ptr exception_ptr) { + + // need to acquire GIL before py::function deletion + auto callback_sp = Common::utils::wrap_pyfunction(std::move(callback)); + + self.m_request->set_callback([&self, callback_sp](std::exception_ptr exception_ptr) { *self.m_end_time = Time::now(); try { if (exception_ptr) { @@ -348,7 +352,7 @@ void regclass_InferRequest(py::module m) { } // Acquire GIL, execute Python function py::gil_scoped_acquire acquire; - callback(self.m_userdata); + (*callback_sp)(self.m_userdata); }); }, py::arg("callback"), @@ -365,7 +369,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_tensor", [](InferRequestWrapper& self, const std::string& name) { - return self.m_request.get_tensor(name); + return self.m_request->get_tensor(name); }, py::arg("name"), R"( @@ -380,7 +384,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_tensor", [](InferRequestWrapper& self, const ov::Output& port) { - return self.m_request.get_tensor(port); + return self.m_request->get_tensor(port); }, py::arg("port"), R"( @@ -395,7 +399,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_tensor", [](InferRequestWrapper& self, const ov::Output& port) { - return self.m_request.get_tensor(port); + return self.m_request->get_tensor(port); }, py::arg("port"), R"( @@ -410,7 +414,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_input_tensor", [](InferRequestWrapper& self, size_t idx) { - return self.m_request.get_input_tensor(idx); + return self.m_request->get_input_tensor(idx); }, py::arg("index"), R"( @@ -427,7 +431,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_input_tensor", [](InferRequestWrapper& self) { - return self.m_request.get_input_tensor(); + return self.m_request->get_input_tensor(); }, R"( Gets input tensor of InferRequest. @@ -440,7 +444,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_output_tensor", [](InferRequestWrapper& self, size_t idx) { - return self.m_request.get_output_tensor(idx); + return self.m_request->get_output_tensor(idx); }, py::arg("index"), R"( @@ -456,7 +460,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_output_tensor", [](InferRequestWrapper& self) { - return self.m_request.get_output_tensor(); + return self.m_request->get_output_tensor(); }, R"( Gets output tensor of InferRequest. @@ -469,7 +473,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensor", [](InferRequestWrapper& self, const std::string& name, const ov::Tensor& tensor) { - self.m_request.set_tensor(name, tensor); + self.m_request->set_tensor(name, tensor); }, py::arg("name"), py::arg("tensor"), @@ -486,7 +490,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensor", [](InferRequestWrapper& self, const ov::Output& port, const ov::Tensor& tensor) { - self.m_request.set_tensor(port, tensor); + self.m_request->set_tensor(port, tensor); }, py::arg("port"), py::arg("tensor"), @@ -503,7 +507,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_tensor", [](InferRequestWrapper& self, const ov::Output& port, const ov::Tensor& tensor) { - self.m_request.set_tensor(port, tensor); + self.m_request->set_tensor(port, tensor); }, py::arg("port"), py::arg("tensor"), @@ -520,7 +524,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_input_tensor", [](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) { - self.m_request.set_input_tensor(idx, tensor); + self.m_request->set_input_tensor(idx, tensor); }, py::arg("index"), py::arg("tensor"), @@ -538,7 +542,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_input_tensor", [](InferRequestWrapper& self, const ov::Tensor& tensor) { - self.m_request.set_input_tensor(tensor); + self.m_request->set_input_tensor(tensor); }, py::arg("tensor"), R"( @@ -553,7 +557,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_output_tensor", [](InferRequestWrapper& self, size_t idx, const ov::Tensor& tensor) { - self.m_request.set_output_tensor(idx, tensor); + self.m_request->set_output_tensor(idx, tensor); }, py::arg("index"), py::arg("tensor"), @@ -570,7 +574,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_output_tensor", [](InferRequestWrapper& self, const ov::Tensor& tensor) { - self.m_request.set_output_tensor(tensor); + self.m_request->set_output_tensor(tensor); }, py::arg("tensor"), R"( @@ -585,7 +589,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_profiling_info", [](InferRequestWrapper& self) { - return self.m_request.get_profiling_info(); + return self.m_request->get_profiling_info(); }, py::call_guard(), R"( @@ -602,7 +606,7 @@ void regclass_InferRequest(py::module m) { cls.def( "query_state", [](InferRequestWrapper& self) { - return self.m_request.query_state(); + return self.m_request->query_state(); }, py::call_guard(), R"( @@ -617,7 +621,7 @@ void regclass_InferRequest(py::module m) { cls.def( "reset_state", [](InferRequestWrapper& self) { - return self.m_request.reset_state(); + return self.m_request->reset_state(); }, R"( Resets all internal variable states for relevant infer request to @@ -627,7 +631,7 @@ void regclass_InferRequest(py::module m) { cls.def( "get_compiled_model", [](InferRequestWrapper& self) { - return self.m_request.get_compiled_model(); + return self.m_request->get_compiled_model(); }, R"( Returns the compiled model. @@ -700,7 +704,7 @@ void regclass_InferRequest(py::module m) { cls.def_property_readonly( "profiling_info", [](InferRequestWrapper& self) { - return self.m_request.get_profiling_info(); + return self.m_request->get_profiling_info(); }, py::call_guard(), R"( diff --git a/src/bindings/python/src/pyopenvino/core/infer_request.hpp b/src/bindings/python/src/pyopenvino/core/infer_request.hpp index 69f0412a1745c9..719d0374af6ff3 100644 --- a/src/bindings/python/src/pyopenvino/core/infer_request.hpp +++ b/src/bindings/python/src/pyopenvino/core/infer_request.hpp @@ -32,7 +32,7 @@ class InferRequestWrapper { const std::vector>& outputs, bool set_default_callback = true, py::object userdata = py::none()) - : m_request{std::move(request)}, + : m_request{InferRequestWrapper::wrap_infer_request_to_sp(std::move(request))}, m_inputs{inputs}, m_outputs{outputs}, m_userdata{userdata} { @@ -44,7 +44,7 @@ class InferRequestWrapper { // Bump reference counter auto end_time = m_end_time; // Set standard callback which saves "end-time" for inference call - m_request.set_callback([end_time](std::exception_ptr exception_ptr) { + m_request->set_callback([end_time](std::exception_ptr exception_ptr) { *end_time = Time::now(); try { if (exception_ptr) { @@ -73,7 +73,7 @@ class InferRequestWrapper { } // Original ov::InferRequest class that is held by this wrapper - ov::InferRequest m_request; + std::shared_ptr m_request; // Inputs and Outputs inherrited from ov::CompiledModel std::vector> m_inputs; std::vector> m_outputs; @@ -91,11 +91,18 @@ class InferRequestWrapper { tensors.reserve(v.size()); for (auto&& node : v) { - tensors.push_back(m_request.get_tensor(node)); + tensors.push_back(m_request->get_tensor(node)); } return tensors; } + + static std::shared_ptr wrap_infer_request_to_sp(ov::InferRequest request) { + return std::shared_ptr(new ov::InferRequest(std::move(request)), [](ov::InferRequest* request) { + py::gil_scoped_release release; + delete request; + }); + } }; void regclass_InferRequest(py::module m); diff --git a/src/bindings/python/src/pyopenvino/frontend/extension.cpp b/src/bindings/python/src/pyopenvino/frontend/extension.cpp index a4f2e9cae1ca0c..4446ea2c9acc33 100644 --- a/src/bindings/python/src/pyopenvino/frontend/extension.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/extension.cpp @@ -30,19 +30,26 @@ void regclass_frontend_TelemetryExtension(py::module m) { py::function& send_event, py::function& send_error, py::function& send_stack_trace) { + auto send_event_sp = Common::utils::wrap_pyfunction(send_event); + auto send_error_sp = Common::utils::wrap_pyfunction(send_error); + auto send_stack_trace_sp = Common::utils::wrap_pyfunction(send_stack_trace); + return std::make_shared( event_category, - [send_event](const std::string& category, const std::string& action, const std::string& label, int value) { + [send_event_sp](const std::string& category, + const std::string& action, + const std::string& label, + int value) { py::gil_scoped_acquire acquire; - send_event(category, action, label, value); + (*send_event_sp)(category, action, label, value); }, - [send_error](const std::string& category, const std::string& error_message) { + [send_error_sp](const std::string& category, const std::string& error_message) { py::gil_scoped_acquire acquire; - send_error(category, error_message); + (*send_error_sp)(category, error_message); }, - [send_stack_trace](const std::string& category, const std::string& error_message) { + [send_stack_trace_sp](const std::string& category, const std::string& error_message) { py::gil_scoped_acquire acquire; - send_stack_trace(category, error_message); + (*send_stack_trace_sp)(category, error_message); }); })); diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index 27f015b14272c2..feeac2d7a02a73 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -419,5 +419,12 @@ ov::Any py_object_to_any(const py::object& py_obj) { } OPENVINO_ASSERT(false, "Unsupported attribute type."); } +std::shared_ptr wrap_pyfunction(py::function f_callback) { + auto callback_sp = std::shared_ptr(new py::function(std::move(f_callback)), [](py::function* c) { + py::gil_scoped_acquire acquire; + delete c; + }); + return callback_sp; +} }; // namespace utils }; // namespace Common diff --git a/src/bindings/python/src/pyopenvino/utils/utils.hpp b/src/bindings/python/src/pyopenvino/utils/utils.hpp index 1e0e7f23069d2e..e4048b3f52feb3 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.hpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.hpp @@ -58,5 +58,7 @@ namespace utils { ov::pass::Serialize::Version convert_to_version(const std::string& version); + std::shared_ptr wrap_pyfunction(py::function f_callback); + }; // namespace utils }; // namespace Common diff --git a/src/bindings/python/tests/test_graph/test_op.py b/src/bindings/python/tests/test_graph/test_op.py index 2bd609ef5278f1..5a8abdc55ea86c 100644 --- a/src/bindings/python/tests/test_graph/test_op.py +++ b/src/bindings/python/tests/test_graph/test_op.py @@ -107,9 +107,7 @@ def test_custom_add_model(): def test_custom_op(): model = create_snake_model() - # todo: CVS-141744 - # it hangs with AUTO plugin, but works well with CPU - compiled_model = compile_model(model, "CPU") + compiled_model = compile_model(model) assert isinstance(compiled_model, CompiledModel) request = compiled_model.create_infer_request()