[BugFix] Shutdown update manager & tracer during BE and UT exit (Star…

…Rocks#30829) Signed-off-by: Binglin Chang <[email protected]>
JingDas · Sep 13, 2023 · 4b9768e · 4b9768e
1 parent ac2b169
commit 4b9768e
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 10 deletions.
diff --git a/be/src/common/tracer.cpp b/be/src/common/tracer.cpp
@@ -36,6 +36,10 @@ Tracer& Tracer::Instance() {
     return global_tracer;
 }
 
+void Tracer::release_instance() {
+    Instance().shutdown();
+}
+
 void Tracer::init(const std::string& service_name) {
     if (!config::jaeger_endpoint.empty()) {
         opentelemetry::exporter::jaeger::JaegerExporterOptions opts;
@@ -64,7 +68,14 @@ void Tracer::init(const std::string& service_name) {
 }
 
 void Tracer::shutdown() {
-    _tracer->CloseWithMicroseconds(1);
+    if (_tracer) {
+        _tracer->CloseWithMicroseconds(1);
+        _tracer = nullptr;
+    }
+}
+
+void shutdown_tracer() {
+    Tracer::release_instance();
 }
 
 bool Tracer::is_enabled() const {

diff --git a/be/src/common/tracer.h b/be/src/common/tracer.h
@@ -52,6 +52,8 @@ class Tracer {
     // Get the global tracer instance.
     static Tracer& Instance();
 
+    static void release_instance();
+
     // Return true if trace is enabled.
     bool is_enabled() const;
 
@@ -92,4 +94,6 @@ class Tracer {
     opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> _tracer;
 };
 
+void shutdown_tracer();
+
 } // namespace starrocks
diff --git a/be/src/service/service_be/starrocks_be.cpp b/be/src/service/service_be/starrocks_be.cpp
@@ -105,6 +105,8 @@ StorageEngine* init_storage_engine(GlobalEnv* global_env, std::vector<StorePath>
     return engine;
 }
 
+extern void shutdown_tracer();
+
 void start_be(const std::vector<StorePath>& paths, bool as_cn) {
     std::string process_name = as_cn ? "CN" : "BE";
 
@@ -274,6 +276,8 @@ void start_be(const std::vector<StorePath>& paths, bool as_cn) {
     global_env->stop();
     LOG(INFO) << process_name << " exit step " << exit_step++ << ": global env stop successfully";
 
+    shutdown_tracer();
+
     LOG(INFO) << process_name << " exited successfully";
 }
 } // namespace starrocks
diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp
@@ -643,6 +643,10 @@ void StorageEngine::stop() {
     if (_compaction_checker_thread.joinable()) {
         _compaction_checker_thread.join();
     }
+
+    if (_update_manager) {
+        _update_manager->stop();
+    }
 }
 
 void StorageEngine::clear_transaction_task(const TTransactionId transaction_id) {

diff --git a/be/src/storage/update_manager.cpp b/be/src/storage/update_manager.cpp
@@ -73,15 +73,6 @@ UpdateManager::UpdateManager(MemTracker* mem_tracker)
 }
 
 UpdateManager::~UpdateManager() {
-    if (_apply_thread_pool != nullptr) {
-        // DynamicCache may be still used by apply thread.
-        // Before deconstrut the DynamicCache, apply thread
-        // should be shutdown.
-        _apply_thread_pool->shutdown();
-    }
-    if (_get_pindex_thread_pool) {
-        _get_pindex_thread_pool->shutdown();
-    }
     clear_cache();
     if (_compaction_state_mem_tracker) {
         _compaction_state_mem_tracker.reset();
@@ -119,6 +110,15 @@ Status UpdateManager::init() {
     return Status::OK();
 }
 
+void UpdateManager::stop() {
+    if (_get_pindex_thread_pool) {
+        _get_pindex_thread_pool->shutdown();
+    }
+    if (_apply_thread_pool) {
+        _apply_thread_pool->shutdown();
+    }
+}
+
 int64_t UpdateManager::get_index_cache_expire_ms(const Tablet& tablet) const {
     const int32_t tablet_index_cache_expire_sec = tablet.tablet_meta()->get_primary_index_cache_expire_sec();
     if (tablet_index_cache_expire_sec > 0) {

diff --git a/be/src/storage/update_manager.h b/be/src/storage/update_manager.h
@@ -70,6 +70,8 @@ class UpdateManager {
 
     Status init();
 
+    void stop();
+
     void set_cache_expire_ms(int64_t expire_ms) { _cache_expire_ms = expire_ms; }
 
     int64_t get_cache_expire_ms() const { return _cache_expire_ms; }

diff --git a/be/src/testutil/init_test_env.h b/be/src/testutil/init_test_env.h
@@ -38,6 +38,8 @@
 
 namespace starrocks {
 
+extern void shutdown_tracer();
+
 int init_test_env(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
     if (getenv("STARROCKS_HOME") == nullptr) {
@@ -115,6 +117,8 @@ int init_test_env(int argc, char** argv) {
     exec_env->destroy();
     global_env->stop();
 
+    shutdown_tracer();
+
     shutdown_logging();
 
     return r;