Save rocksdb statistics to file every minute (ton-blockchain#932)

* Save rocksdb statistics to file every minute

* Add flag to disable collecting rocksdb statistics

tddb/td/db/RocksDb.cpp

@@ -59,9 +59,8 @@ RocksDb RocksDb::clone() const {
   return RocksDb{db_, statistics_};
 }
 
-Result<RocksDb> RocksDb::open(std::string path) {
+Result<RocksDb> RocksDb::open(std::string path, std::shared_ptr<rocksdb::Statistics> statistics) {
   rocksdb::OptimisticTransactionDB *db;
-  auto statistics = rocksdb::CreateDBStatistics();
   {
     rocksdb::Options options;
 
@@ -94,6 +93,18 @@ Result<RocksDb> RocksDb::open(std::string path) {
   return RocksDb(std::shared_ptr<rocksdb::OptimisticTransactionDB>(db), std::move(statistics));
 }
 
+std::shared_ptr<rocksdb::Statistics> RocksDb::create_statistics() {
+  return rocksdb::CreateDBStatistics();
+}
+
+std::string RocksDb::statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics) {
+  return statistics->ToString();
+}
+
+void RocksDb::reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics) {
+  statistics->Reset();
+}
+
 std::unique_ptr<KeyValueReader> RocksDb::snapshot() {
   auto res = std::make_unique<RocksDb>(clone());
   res->begin_snapshot().ensure();

tddb/td/db/RocksDb.h

@@ -38,7 +38,7 @@ class RocksDb : public KeyValue {
  public:
   static Status destroy(Slice path);
   RocksDb clone() const;
-  static Result<RocksDb> open(std::string path);
+  static Result<RocksDb> open(std::string path, std::shared_ptr<rocksdb::Statistics> statistics = nullptr);
 
   Result<GetStatus> get(Slice key, std::string &value) override;
   Status set(Slice key, Slice value) override;
@@ -60,6 +60,10 @@ class RocksDb : public KeyValue {
   std::unique_ptr<KeyValueReader> snapshot() override;
   std::string stats() const override;
 
+  static std::shared_ptr<rocksdb::Statistics> create_statistics();
+  static std::string statistics_to_string(const std::shared_ptr<rocksdb::Statistics> statistics);
+  static void reset_statistics(const std::shared_ptr<rocksdb::Statistics> statistics);
+
   RocksDb(RocksDb &&);
   RocksDb &operator=(RocksDb &&);
   ~RocksDb();

validator-engine/validator-engine.cpp

@@ -1367,6 +1367,7 @@ td::Status ValidatorEngine::load_global_config() {
   validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_);
   validator_options_.write().set_max_open_archive_files(max_open_archive_files_);
   validator_options_.write().set_archive_preload_period(archive_preload_period_);
+  validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_);
 
   std::vector<ton::BlockIdExt> h;
   for (auto &x : conf.validator_->hardforks_) {
@@ -3816,6 +3817,9 @@ int main(int argc, char *argv[]) {
   p.add_option('\0', "enable-precompiled-smc",
                "enable exectuion of precompiled contracts (experimental, disabled by default)",
                []() { block::precompiled::set_precompiled_execution_enabled(true); });
+  p.add_option('\0', "disable-rocksdb-stats", "disable gathering rocksdb statistics (enabled by default)", [&]() {
+    acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_disable_rocksdb_stats, true); });
+  });
   auto S = p.run(argc, argv);
   if (S.is_error()) {
     LOG(ERROR) << "failed to parse options: " << S.move_as_error();

validator-engine/validator-engine.hpp

@@ -206,6 +206,7 @@ class ValidatorEngine : public td::actor::Actor {
   td::uint32 celldb_compress_depth_ = 0;
   size_t max_open_archive_files_ = 0;
   double archive_preload_period_ = 0.0;
+  bool disable_rocksdb_stats_ = false;
   bool read_config_ = false;
   bool started_keyring_ = false;
   bool started_ = false;
@@ -272,6 +273,9 @@ class ValidatorEngine : public td::actor::Actor {
   void set_archive_preload_period(double value) {
     archive_preload_period_ = value;
   }
+  void set_disable_rocksdb_stats(bool value) {
+    disable_rocksdb_stats_ = value;
+  }
   void start_up() override;
   ValidatorEngine() {
   }

validator/db/archive-manager.cpp

@@ -601,7 +601,7 @@ void ArchiveManager::load_package(PackageId id) {
   }
 
   desc.file =
-      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
+      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
 
   m.emplace(id, std::move(desc));
   update_permanent_slices();
@@ -636,7 +636,7 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull
   td::mkdir(db_root_ + id.path()).ensure();
   std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
   new_desc.file =
-      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get());
+      td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_);
   const FileDescription &desc = f.emplace(id, std::move(new_desc));
   if (!id.temp) {
     update_desc(f, desc, shard, seqno, ts, lt);
@@ -829,7 +829,10 @@ void ArchiveManager::start_up() {
   if (opts_->get_max_open_archive_files() > 0) {
     archive_lru_ = td::actor::create_actor<ArchiveLru>("archive_lru", opts_->get_max_open_archive_files());
   }
-  index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok());
+  if (!opts_->get_disable_rocksdb_stats()) {
+    statistics_ = td::RocksDb::create_statistics();
+  }
+  index_ = std::make_shared<td::RocksDb>(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_).move_as_ok());
   std::string value;
   auto v = index_->get(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(), value);
   v.ensure();
@@ -903,6 +906,17 @@ void ArchiveManager::start_up() {
       break;
     }
   }
+
+  if (!opts_->get_disable_rocksdb_stats()) {
+    alarm_timestamp() = td::Timestamp::in(60.0);
+  }
+}
+
+void ArchiveManager::alarm() {
+  alarm_timestamp() = td::Timestamp::in(60.0);
+  auto stats = td::RocksDb::statistics_to_string(statistics_);
+  td::atomic_write_file(db_root_ + "/db_stats.txt", stats);
+  td::RocksDb::reset_statistics(statistics_);
 }
 
 void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) {

validator/db/archive-manager.hpp

@@ -70,6 +70,7 @@ class ArchiveManager : public td::actor::Actor {
                          td::Promise<td::BufferSlice> promise);
 
   void start_up() override;
+  void alarm() override;
 
   void commit_transaction();
   void set_async_mode(bool mode, td::Promise<td::Unit> promise);
@@ -173,6 +174,8 @@ class ArchiveManager : public td::actor::Actor {
   bool huge_transaction_started_ = false;
   td::uint32 huge_transaction_size_ = 0;
 
+  std::shared_ptr<rocksdb::Statistics> statistics_;
+
   FileMap &get_file_map(const PackageId &p) {
     return p.key ? key_files_ : p.temp ? temp_files_ : files_;
   }

validator/db/archive-slice.cpp

@@ -465,7 +465,7 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::
 void ArchiveSlice::before_query() {
   if (status_ == st_closed) {
     LOG(DEBUG) << "Opening archive slice " << db_path_;
-    kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_).move_as_ok());
+    kv_ = std::make_unique<td::RocksDb>(td::RocksDb::open(db_path_, statistics_).move_as_ok());
     std::string value;
     auto R2 = kv_->get("status", value);
     R2.ensure();
@@ -604,14 +604,15 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise<td::Unit> promise) {
 }
 
 ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
-                           td::actor::ActorId<ArchiveLru> archive_lru)
+                           td::actor::ActorId<ArchiveLru> archive_lru, std::shared_ptr<rocksdb::Statistics> statistics)
     : archive_id_(archive_id)
     , key_blocks_only_(key_blocks_only)
     , temp_(temp)
     , finalized_(finalized)
     , p_id_(archive_id_, key_blocks_only_, temp_)
     , db_root_(std::move(db_root))
-    , archive_lru_(std::move(archive_lru)) {
+    , archive_lru_(std::move(archive_lru))
+    , statistics_(statistics) {
   db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index";
 }
 

validator/db/archive-slice.hpp

@@ -23,6 +23,10 @@
 #include "fileref.hpp"
 #include <map>
 
+namespace rocksdb {
+class Statistics;
+}
+
 namespace ton {
 
 namespace validator {
@@ -81,7 +85,7 @@ class ArchiveLru;
 class ArchiveSlice : public td::actor::Actor {
  public:
   ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root,
-               td::actor::ActorId<ArchiveLru> archive_lru);
+               td::actor::ActorId<ArchiveLru> archive_lru, std::shared_ptr<rocksdb::Statistics> statistics = nullptr);
 
   void get_archive_id(BlockSeqno masterchain_seqno, td::Promise<td::uint64> promise);
 
@@ -151,6 +155,7 @@ class ArchiveSlice : public td::actor::Actor {
 
   std::string db_root_;
   td::actor::ActorId<ArchiveLru> archive_lru_;
+  std::shared_ptr<rocksdb::Statistics> statistics_;
   std::unique_ptr<td::KeyValue> kv_;
 
   struct PackageInfo {

validator/db/celldb.cpp

@@ -20,6 +20,7 @@
 #include "rootdb.hpp"
 
 #include "td/db/RocksDb.h"
+#include "td/utils/filesystem.h"
 
 #include "ton/ton-tl.hpp"
 #include "ton/ton-io.hpp"
@@ -83,7 +84,12 @@ void CellDbIn::start_up() {
   };
 
   CellDbBase::start_up();
-  cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_).move_as_ok());
+  if (!opts_->get_disable_rocksdb_stats()) {
+    statistics_ = td::RocksDb::create_statistics();
+    statistics_flush_at_ = td::Timestamp::in(60.0);
+  }
+  cell_db_ = std::make_shared<td::RocksDb>(td::RocksDb::open(path_, statistics_).move_as_ok());
+
 
   boc_ = vm::DynamicBagOfCellsDb::create();
   boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth());
@@ -156,6 +162,13 @@ void CellDbIn::get_cell_db_reader(td::Promise<std::shared_ptr<vm::CellDbReader>>
 }
 
 void CellDbIn::alarm() {
+  if (statistics_flush_at_ && statistics_flush_at_.is_in_past()) {
+    statistics_flush_at_ = td::Timestamp::in(60.0);
+    auto stats = td::RocksDb::statistics_to_string(statistics_);
+    td::atomic_write_file(path_ + "/db_stats.txt", stats);
+    td::RocksDb::reset_statistics(statistics_);
+  }
+
   if (migrate_after_ && migrate_after_.is_in_past()) {
     migrate_cells();
   }

validator/db/celldb.hpp

@@ -27,6 +27,10 @@
 #include "auto/tl/ton_api.h"
 #include "validator.h"
 
+namespace rocksdb {
+class Statistics;
+}
+
 namespace ton {
 
 namespace validator {
@@ -103,6 +107,8 @@ class CellDbIn : public CellDbBase {
 
   std::unique_ptr<vm::DynamicBagOfCellsDb> boc_;
   std::shared_ptr<vm::KeyValue> cell_db_;
+  std::shared_ptr<rocksdb::Statistics> statistics_;
+  td::Timestamp statistics_flush_at_ = td::Timestamp::never();
 
   std::function<void(const vm::CellLoader::LoadResult&)> on_load_callback_;
   std::set<td::Bits256> cells_to_migrate_;

validator/validator-options.hpp

@@ -123,6 +123,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   double get_archive_preload_period() const override {
     return archive_preload_period_;
   }
+  bool get_disable_rocksdb_stats() const override {
+    return disable_rocksdb_stats_;
+  }
 
   void set_zero_block_id(BlockIdExt block_id) override {
     zero_block_id_ = block_id;
@@ -185,6 +188,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   void set_archive_preload_period(double value) override {
     archive_preload_period_ = value;
   }
+  void set_disable_rocksdb_stats(bool value) override {
+    disable_rocksdb_stats_ = value;
+  }
 
   ValidatorManagerOptionsImpl *make_copy() const override {
     return new ValidatorManagerOptionsImpl(*this);
@@ -230,6 +236,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions {
   td::uint32 celldb_compress_depth_{0};
   size_t max_open_archive_files_ = 0;
   double archive_preload_period_ = 0.0;
+  bool disable_rocksdb_stats_;
 };
 
 }  // namespace validator

validator/validator.h

@@ -84,6 +84,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual td::uint32 get_celldb_compress_depth() const = 0;
   virtual size_t get_max_open_archive_files() const = 0;
   virtual double get_archive_preload_period() const = 0;
+  virtual bool get_disable_rocksdb_stats() const = 0;
 
   virtual void set_zero_block_id(BlockIdExt block_id) = 0;
   virtual void set_init_block_id(BlockIdExt block_id) = 0;
@@ -106,6 +107,7 @@ struct ValidatorManagerOptions : public td::CntObject {
   virtual void set_celldb_compress_depth(td::uint32 value) = 0;
   virtual void set_max_open_archive_files(size_t value) = 0;
   virtual void set_archive_preload_period(double value) = 0;
+  virtual void set_disable_rocksdb_stats(bool value) = 0;
 
   static td::Ref<ValidatorManagerOptions> create(
       BlockIdExt zero_block_id, BlockIdExt init_block_id,