From 4b9768ea1646c541fc0dd73cdbd786943c12b20f Mon Sep 17 00:00:00 2001
From: Binglin Chang <decstery@gmail.com>
Date: Wed, 13 Sep 2023 14:57:14 +0800
Subject: [PATCH] [BugFix] Shutdown update manager & tracer during BE and UT
 exit (#30829)

Signed-off-by: Binglin Chang <decstery@gmail.com>
---
 be/src/common/tracer.cpp                   | 13 ++++++++++++-
 be/src/common/tracer.h                     |  4 ++++
 be/src/service/service_be/starrocks_be.cpp |  4 ++++
 be/src/storage/storage_engine.cpp          |  4 ++++
 be/src/storage/update_manager.cpp          | 18 +++++++++---------
 be/src/storage/update_manager.h            |  2 ++
 be/src/testutil/init_test_env.h            |  4 ++++
 7 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/be/src/common/tracer.cpp b/be/src/common/tracer.cpp
index 8efcaa93910ff3..138222bff897a7 100644
--- a/be/src/common/tracer.cpp
+++ b/be/src/common/tracer.cpp
@@ -36,6 +36,10 @@ Tracer& Tracer::Instance() {
     return global_tracer;
 }
 
+void Tracer::release_instance() {
+    Instance().shutdown();
+}
+
 void Tracer::init(const std::string& service_name) {
     if (!config::jaeger_endpoint.empty()) {
         opentelemetry::exporter::jaeger::JaegerExporterOptions opts;
@@ -64,7 +68,14 @@ void Tracer::init(const std::string& service_name) {
 }
 
 void Tracer::shutdown() {
-    _tracer->CloseWithMicroseconds(1);
+    if (_tracer) {
+        _tracer->CloseWithMicroseconds(1);
+        _tracer = nullptr;
+    }
+}
+
+void shutdown_tracer() {
+    Tracer::release_instance();
 }
 
 bool Tracer::is_enabled() const {
diff --git a/be/src/common/tracer.h b/be/src/common/tracer.h
index 4f10278f9b0836..5587e79df518cf 100644
--- a/be/src/common/tracer.h
+++ b/be/src/common/tracer.h
@@ -52,6 +52,8 @@ class Tracer {
     // Get the global tracer instance.
     static Tracer& Instance();
 
+    static void release_instance();
+
     // Return true if trace is enabled.
     bool is_enabled() const;
 
@@ -92,4 +94,6 @@ class Tracer {
     opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> _tracer;
 };
 
+void shutdown_tracer();
+
 } // namespace starrocks
diff --git a/be/src/service/service_be/starrocks_be.cpp b/be/src/service/service_be/starrocks_be.cpp
index 9cf1fa38778a11..0cc63e1b76b4c8 100644
--- a/be/src/service/service_be/starrocks_be.cpp
+++ b/be/src/service/service_be/starrocks_be.cpp
@@ -105,6 +105,8 @@ StorageEngine* init_storage_engine(GlobalEnv* global_env, std::vector<StorePath>
     return engine;
 }
 
+extern void shutdown_tracer();
+
 void start_be(const std::vector<StorePath>& paths, bool as_cn) {
     std::string process_name = as_cn ? "CN" : "BE";
 
@@ -274,6 +276,8 @@ void start_be(const std::vector<StorePath>& paths, bool as_cn) {
     global_env->stop();
     LOG(INFO) << process_name << " exit step " << exit_step++ << ": global env stop successfully";
 
+    shutdown_tracer();
+
     LOG(INFO) << process_name << " exited successfully";
 }
 } // namespace starrocks
diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp
index b2e3a1c4573943..2f8113e9d89695 100644
--- a/be/src/storage/storage_engine.cpp
+++ b/be/src/storage/storage_engine.cpp
@@ -643,6 +643,10 @@ void StorageEngine::stop() {
     if (_compaction_checker_thread.joinable()) {
         _compaction_checker_thread.join();
     }
+
+    if (_update_manager) {
+        _update_manager->stop();
+    }
 }
 
 void StorageEngine::clear_transaction_task(const TTransactionId transaction_id) {
diff --git a/be/src/storage/update_manager.cpp b/be/src/storage/update_manager.cpp
index 003a55f5e6d6f7..f946b05bd2734f 100644
--- a/be/src/storage/update_manager.cpp
+++ b/be/src/storage/update_manager.cpp
@@ -73,15 +73,6 @@ UpdateManager::UpdateManager(MemTracker* mem_tracker)
 }
 
 UpdateManager::~UpdateManager() {
-    if (_apply_thread_pool != nullptr) {
-        // DynamicCache may be still used by apply thread.
-        // Before deconstrut the DynamicCache, apply thread
-        // should be shutdown.
-        _apply_thread_pool->shutdown();
-    }
-    if (_get_pindex_thread_pool) {
-        _get_pindex_thread_pool->shutdown();
-    }
     clear_cache();
     if (_compaction_state_mem_tracker) {
         _compaction_state_mem_tracker.reset();
@@ -119,6 +110,15 @@ Status UpdateManager::init() {
     return Status::OK();
 }
 
+void UpdateManager::stop() {
+    if (_get_pindex_thread_pool) {
+        _get_pindex_thread_pool->shutdown();
+    }
+    if (_apply_thread_pool) {
+        _apply_thread_pool->shutdown();
+    }
+}
+
 int64_t UpdateManager::get_index_cache_expire_ms(const Tablet& tablet) const {
     const int32_t tablet_index_cache_expire_sec = tablet.tablet_meta()->get_primary_index_cache_expire_sec();
     if (tablet_index_cache_expire_sec > 0) {
diff --git a/be/src/storage/update_manager.h b/be/src/storage/update_manager.h
index 104008b809528c..78f6cd5c3d08e9 100644
--- a/be/src/storage/update_manager.h
+++ b/be/src/storage/update_manager.h
@@ -70,6 +70,8 @@ class UpdateManager {
 
     Status init();
 
+    void stop();
+
     void set_cache_expire_ms(int64_t expire_ms) { _cache_expire_ms = expire_ms; }
 
     int64_t get_cache_expire_ms() const { return _cache_expire_ms; }
diff --git a/be/src/testutil/init_test_env.h b/be/src/testutil/init_test_env.h
index 41c636467152f5..e517f4a03d3460 100644
--- a/be/src/testutil/init_test_env.h
+++ b/be/src/testutil/init_test_env.h
@@ -38,6 +38,8 @@
 
 namespace starrocks {
 
+extern void shutdown_tracer();
+
 int init_test_env(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
     if (getenv("STARROCKS_HOME") == nullptr) {
@@ -115,6 +117,8 @@ int init_test_env(int argc, char** argv) {
     exec_env->destroy();
     global_env->stop();
 
+    shutdown_tracer();
+
     shutdown_logging();
 
     return r;