From 4c3e100d644db9abfc9463e805a114009bee5448 Mon Sep 17 00:00:00 2001 From: Dmytro Podgornyi Date: Wed, 2 Oct 2024 11:39:02 +0300 Subject: [PATCH] issue: Start event handler thread during XLIO init Event handler manager is a global object with a single internal thread. Once XLIO is initialized, there is no option not to start the internal thread and it's used starting from the initialization method (to create a timer event for netlink). Therefore, there is no point in trying to start the thread with each post action to the event handler. This can save CPU in CPS scenario. Signed-off-by: Dmytro Podgornyi --- src/core/event/event_handler_manager.cpp | 65 +++++++++++------------- src/core/event/event_handler_manager.h | 2 +- src/core/main.cpp | 7 +++ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/core/event/event_handler_manager.cpp b/src/core/event/event_handler_manager.cpp index 2a6f4584c..6f754b170 100644 --- a/src/core/event/event_handler_manager.cpp +++ b/src/core/event/event_handler_manager.cpp @@ -344,56 +344,53 @@ int event_handler_manager::start_thread() { cpu_set_t cpu_set; pthread_attr_t tattr; + int ret; + bool affinity_requested = false; if (!m_b_continue_running) { + errno = ECANCELED; return -1; } - if (m_event_handler_tid != 0) { return 0; } - // m_reg_action_q.reserve(); //todo change to vector and reserve - - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_init(&tattr)) { - evh_logpanic("Failed to initialize thread attributes"); + ret = pthread_attr_init(&tattr); + if (ret != 0) { + return -1; } - BULLSEYE_EXCLUDE_BLOCK_END cpu_set = safe_mce_sys().internal_thread_affinity; if (strcmp(safe_mce_sys().internal_thread_affinity_str, "-1") && - !strcmp(safe_mce_sys().internal_thread_cpuset, - MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) { // no affinity - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set)) { - evh_logpanic("Failed to set CPU affinity"); + !strcmp(safe_mce_sys().internal_thread_cpuset, MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) { + ret = pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set); + if (ret != 0) { + evh_logwarn("Failed to set event handler thread affinity. (errno=%d)", errno); } - BULLSEYE_EXCLUDE_BLOCK_END + affinity_requested = (ret == 0); } else { evh_logdbg("Internal thread affinity not set."); } - int ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); - if (ret) { - // maybe it's the cset issue? try without affinity - evh_logwarn("Failed to start event handler thread with thread affinity - trying without. " - "[errno=%d %s]", - ret, strerror(ret)); - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_init(&tattr)) { - evh_logpanic("Failed to initialize thread attributes"); - } - if (pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this)) { - evh_logpanic("Failed to start event handler thread"); - } - BULLSEYE_EXCLUDE_BLOCK_END + ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); + if (ret != 0 && affinity_requested) { + // Try without affinity in case this is a cset issue. + evh_logwarn("Failed to start event handler thread with a thread affinity. Trying default " + "thread affinity. (errno=%d)", + errno); + pthread_attr_destroy(&tattr); + ret = pthread_attr_init(&tattr) + ?: pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); } - + // Destroy will either succeed or return EINVAL if the init fails in the above block. pthread_attr_destroy(&tattr); - evh_logdbg("Started event handler thread"); - return 0; + if (ret == 0) { + evh_logdbg("Started event handler thread."); + } else { + evh_logerr("Failed to start event handler thread. (errno=%d)", errno); + } + return ret; } void event_handler_manager::stop_thread() @@ -483,15 +480,11 @@ void event_handler_manager::post_new_reg_action(reg_action_t ®_action) return; } - start_thread(); - evh_logfunc("add event action %s (%d)", reg_action_str(reg_action.type), reg_action.type); - bool is_empty = false; + bool is_empty; m_reg_action_q_lock.lock(); - if (m_p_reg_action_q_to_push_to->empty()) { - is_empty = true; - } + is_empty = m_p_reg_action_q_to_push_to->empty(); m_p_reg_action_q_to_push_to->push_back(reg_action); m_reg_action_q_lock.unlock(); if (is_empty) { diff --git a/src/core/event/event_handler_manager.h b/src/core/event/event_handler_manager.h index 666c7e34e..5e2c0761a 100644 --- a/src/core/event/event_handler_manager.h +++ b/src/core/event/event_handler_manager.h @@ -208,6 +208,7 @@ class event_handler_manager : public wakeup_pipe { void register_command_event(int fd, command *cmd); void *thread_loop(); + int start_thread(); void stop_thread(); bool is_running() { return m_b_continue_running; }; @@ -251,7 +252,6 @@ class event_handler_manager : public wakeup_pipe { void handle_registration_action(reg_action_t ®_action); void process_ibverbs_event(event_handler_map_t::iterator &i); void process_rdma_cm_event(event_handler_map_t::iterator &i); - int start_thread(); void event_channel_post_process_for_rdma_events(void *p_event); void *event_channel_pre_process_for_rdma_events(void *p_event_channel_handle, void **p_event); diff --git a/src/core/main.cpp b/src/core/main.cpp index 8b41c4314..8b33557ef 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -983,6 +983,7 @@ static void do_global_ctors_helper() { static lock_spin_recursive g_globals_lock; std::lock_guard lock(g_globals_lock); + int rc; if (g_init_global_ctors_done) { return; @@ -1018,6 +1019,12 @@ static void do_global_ctors_helper() // Create all global management objects NEW_CTOR(g_p_event_handler_manager, event_handler_manager()); + rc = g_p_event_handler_manager->start_thread(); + if (rc != 0) { + BULLSEYE_EXCLUDE_BLOCK_START + throw_xlio_exception("Failed to start event handler thread.\n"); + BULLSEYE_EXCLUDE_BLOCK_END + } xlio_shmem_stats_open(&g_p_vlogger_level, &g_p_vlogger_details); *g_p_vlogger_level = g_vlogger_level;