Move storage to libs

vmware · Mar 21, 2023 · b6760c5 · b6760c5
1 parent a1b4091
commit b6760c5
Show file tree

Hide file tree

Showing 124 changed files with 264 additions and 213 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -193,7 +193,6 @@ add_subdirectory(kvbc)
 add_subdirectory(performance)
 add_subdirectory(bftengine)
 add_subdirectory(tools)
-add_subdirectory(storage)
 add_subdirectory(scripts)
 add_subdirectory(diagnostics)
 add_subdirectory(reconfiguration)

diff --git a/bftengine/include/bftengine/DbCheckpointManager.hpp b/bftengine/include/bftengine/DbCheckpointManager.hpp
@@ -31,7 +31,7 @@
 #include "DbCheckpointMetadata.hpp"
 #include "util/Metrics.hpp"
 #include "InternalBFTClient.hpp"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 #include "util/filesystem.hpp"
 #include "kv_types.hpp"
 namespace _fs = fs;

diff --git a/bftengine/include/bftengine/DbMetadataStorage.hpp b/bftengine/include/bftengine/DbMetadataStorage.hpp
@@ -19,8 +19,8 @@
 
 #include "log/logger.hpp"
 #include "bftengine/MetadataStorage.hpp"
-#include "storage/db_interface.h"
-#include "storage/key_manipulator_interface.h"
+#include "storage/utils/db_interface.hpp"
+#include "storage/utils/key_manipulator_interface.hpp"
 #include "util/sliver.hpp"
 
 namespace concord {

diff --git a/bftengine/src/bcstatetransfer/BCStateTran.cpp b/bftengine/src/bcstatetransfer/BCStateTran.cpp
@@ -27,9 +27,9 @@
 #include "util/json_output.hpp"
 #include "ReservedPagesClient.hpp"
 #include "DBDataStore.hpp"
-#include "storage/db_interface.h"
-#include "storage/key_manipulator_interface.h"
-#include "memorydb/client.h"
+#include "storage/utils/db_interface.hpp"
+#include "storage/utils/key_manipulator_interface.hpp"
+#include "storage/memorydb/client.hpp"
 #include "client/reconfiguration/client_reconfiguration_engine.hpp"
 #include "client/reconfiguration/poll_based_state_client.hpp"
 #include "RVBManager.hpp"

diff --git a/bftengine/src/bcstatetransfer/DBDataStore.cpp b/bftengine/src/bcstatetransfer/DBDataStore.cpp
@@ -1,6 +1,6 @@
 #include <algorithm>
 #include "DBDataStore.hpp"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 #include "util/serializable.hpp"
 
 using concord::serialize::Serializable;

diff --git a/bftengine/src/bcstatetransfer/DBDataStore.hpp b/bftengine/src/bcstatetransfer/DBDataStore.hpp
@@ -20,7 +20,7 @@
 #include "log/logger.hpp"
 #include "util/string.hpp"
 #include "InMemoryDataStore.hpp"
-#include "storage/key_manipulator_interface.h"
+#include "storage/utils/key_manipulator_interface.hpp"
 
 namespace bftEngine {
 namespace bcst {

diff --git a/bftengine/src/bcstatetransfer/DataStore.hpp b/bftengine/src/bcstatetransfer/DataStore.hpp
@@ -19,7 +19,7 @@
 
 #include "log/logger.hpp"
 #include "util/assertUtils.hpp"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 #include "crypto/digest.hpp"
 
 using std::set;

diff --git a/bftengine/src/simplestatetransfer/SimpleStateTran.cpp b/bftengine/src/simplestatetransfer/SimpleStateTran.cpp
@@ -21,9 +21,9 @@
 #include "util/assertUtils.hpp"
 #include "SimpleStateTransfer.hpp"
 #include "SimpleBCStateTransfer.hpp"
-#include "memorydb/client.h"
-#include "memorydb/key_comparator.h"
-#include "storage/direct_kv_key_manipulator.h"
+#include "storage/memorydb/client.hpp"
+#include "storage/memorydb/key_comparator.hpp"
+#include "storage/utils/direct_kv_key_manipulator.hpp"
 #include "util/Timers.hpp"
 
 namespace bftEngine {

diff --git a/bftengine/tests/bcstatetransfer/bcstatetransfer_tests.cpp b/bftengine/tests/bcstatetransfer/bcstatetransfer_tests.cpp
@@ -33,8 +33,8 @@
 #include "test_replica.hpp"
 #include "DBDataStore.hpp"
 #include "direct_kv_db_adapter.h"
-#include "memorydb/client.h"
-#include "storage/direct_kv_key_manipulator.h"
+#include "storage/memorydb/client.hpp"
+#include "storage/utils/direct_kv_key_manipulator.hpp"
 #include "ReservedPagesMock.hpp"
 #include "EpochManager.hpp"
 #include "Messages.hpp"
@@ -45,8 +45,8 @@
 #include "messages/StateTransferMsg.hpp"
 
 #ifdef USE_ROCKSDB
-#include "rocksdb/client.h"
-#include "rocksdb/key_comparator.h"
+#include "storage/rocksdb/client.hpp"
+#include "storage/rocksdb/key_comparator.hpp"
 using concord::storage::rocksdb::Client;
 using concord::storage::rocksdb::KeyComparator;
 #endif

diff --git a/bftengine/tests/metadataStorage/metadataStorage_test.cpp b/bftengine/tests/metadataStorage/metadataStorage_test.cpp
@@ -7,11 +7,11 @@
 
 #include "log/logger.hpp"
 #include "gtest/gtest.h"
-#include "rocksdb/key_comparator.h"
-#include "rocksdb/client.h"
+#include "storage/rocksdb/key_comparator.hpp"
+#include "storage/rocksdb/client.hpp"
 #include "DbMetadataStorage.hpp"
 #include "direct_kv_db_adapter.h"
-#include "storage/direct_kv_key_manipulator.h"
+#include "storage/utils/direct_kv_key_manipulator.hpp"
 
 #include <util/filesystem.hpp>
 

diff --git a/examples/replica/main.cpp b/examples/replica/main.cpp
@@ -26,8 +26,8 @@
 #include "util/Metrics.hpp"
 
 #ifdef USE_ROCKSDB
-#include "rocksdb/client.h"
-#include "rocksdb/key_comparator.h"
+#include "storage/rocksdb/client.hpp"
+#include "storage/rocksdb/key_comparator.hpp"
 #endif
 
 #include <csignal>

diff --git a/examples/replica/src/SetupReplica.cpp b/examples/replica/src/SetupReplica.cpp
@@ -17,7 +17,7 @@
 
 #include "SetupReplica.hpp"
 #include "common.hpp"
-#include "memorydb/client.h"
+#include "storage/memorydb/client.hpp"
 #include "secrets/secrets_manager_plain.h"
 #include "secrets/secrets_manager_enc.h"
 #include "direct_kv_storage_factory.h"

diff --git a/kvbc/benchmark/kvbcbench/main.cpp b/kvbc/benchmark/kvbcbench/main.cpp
@@ -34,7 +34,7 @@
 #include "categorized_kvbc_msgs.cmf.hpp"
 #include "kvbc_adapter/replica_adapter.hpp"
 #include "performance_handler.h"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "diagnostics.h"
 #include "diagnostics_server.h"
 #include "input.h"

diff --git a/kvbc/benchmark/sparse_merkle_benchmark.cpp b/kvbc/benchmark/sparse_merkle_benchmark.cpp
@@ -17,7 +17,7 @@
 
 #include "util/endianness.hpp"
 #include "util/Handoff.hpp"
-#include "memorydb/client.h"
+#include "storage/memorydb/client.hpp"
 #include "merkle_tree_db_adapter.h"
 #include "merkle_tree_key_manipulator.h"
 #include "merkle_tree_serialization.h"

diff --git a/kvbc/benchmark/state_snapshot_benchmarks/hash_state_benchmark.cpp b/kvbc/benchmark/state_snapshot_benchmarks/hash_state_benchmark.cpp
@@ -17,7 +17,7 @@
 #include "categorization/details.h"
 #include "util/hex_tools.hpp"
 #include "kv_types.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "util/thread_pool.hpp"
 
 #include "multi_get_batch.hpp"

diff --git a/kvbc/benchmark/state_snapshot_benchmarks/multi_get_batch.hpp b/kvbc/benchmark/state_snapshot_benchmarks/multi_get_batch.hpp
@@ -15,7 +15,7 @@
 
 #include "util/assertUtils.hpp"
 #include "categorized_kvbc_msgs.cmf.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 #include <cstddef>
 #include <cstdint>

diff --git a/kvbc/include/KVBCInterfaces.h b/kvbc/include/KVBCInterfaces.h
@@ -17,7 +17,7 @@
 #include "util/sliver.hpp"
 #include "communication/ICommunication.hpp"
 #include "util/Metrics.hpp"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 #include "db_interfaces.h"
 #include "bftengine/Replica.hpp"
 #include "kv_types.hpp"

diff --git a/kvbc/include/Replica.h b/kvbc/include/Replica.h
@@ -23,7 +23,7 @@
 #include "replica_state_sync_imp.hpp"
 #include "db_adapter_interface.h"
 #include "db_interfaces.h"
-#include "memorydb/client.h"
+#include "storage/memorydb/client.hpp"
 #include "bftengine/DbMetadataStorage.hpp"
 #include "storage_factory_interface.h"
 #include "ControlStateManager.hpp"

diff --git a/kvbc/include/categorization/block_merkle_category.h b/kvbc/include/categorization/block_merkle_category.h
@@ -16,7 +16,7 @@
 #ifdef USE_ROCKSDB
 
 #include "log/logger.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "sparse_merkle/tree.h"
 
 #include "base_types.h"

diff --git a/kvbc/include/categorization/details.h b/kvbc/include/categorization/details.h
@@ -15,7 +15,7 @@
 
 #include "base_types.h"
 #include "categorized_kvbc_msgs.cmf.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 #include <algorithm>
 #include <cstddef>

diff --git a/kvbc/include/categorization/immutable_kv_category.h b/kvbc/include/categorization/immutable_kv_category.h
@@ -14,7 +14,7 @@
 #pragma once
 
 #include "kv_types.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 #include "base_types.h"
 #include "categorized_kvbc_msgs.cmf.hpp"

diff --git a/kvbc/include/categorization/kv_blockchain.h b/kvbc/include/categorization/kv_blockchain.h
@@ -15,7 +15,7 @@
 
 #include "updates.h"
 #include "blockchain_misc.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "blocks.h"
 #include "blockchain.h"
 #include "immutable_kv_category.h"

diff --git a/kvbc/include/categorization/versioned_kv_category.h b/kvbc/include/categorization/versioned_kv_category.h
@@ -14,7 +14,7 @@
 #pragma once
 
 #include "kv_types.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 #include "base_types.h"
 #include "categorized_kvbc_msgs.cmf.hpp"

diff --git a/kvbc/include/direct_kv_db_adapter.h b/kvbc/include/direct_kv_db_adapter.h
@@ -7,8 +7,8 @@
 
 #include "db_adapter_interface.h"
 #include "kv_types.hpp"
-#include "storage/db_interface.h"
-#include "storage/direct_kv_key_manipulator.h"
+#include "storage/utils/db_interface.hpp"
+#include "storage/utils/direct_kv_key_manipulator.hpp"
 #include "PerformanceManager.hpp"
 
 #include <memory>

diff --git a/kvbc/include/direct_kv_storage_factory.h b/kvbc/include/direct_kv_storage_factory.h
@@ -16,7 +16,7 @@
 #include "PerformanceManager.hpp"
 
 #ifdef USE_S3_OBJECT_STORE
-#include "s3/client.hpp"
+#include "storage/s3/client.hpp"
 #endif
 
 #include <string>

diff --git a/kvbc/include/kvbc_adapter/common/state_snapshot_adapter.hpp b/kvbc/include/kvbc_adapter/common/state_snapshot_adapter.hpp
@@ -20,7 +20,7 @@
 #include "categorized_kvbc_msgs.cmf.hpp"
 #include "state_snapshot_interface.hpp"
 #include "categorization/kv_blockchain.h"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 namespace concord::kvbc::adapter::common::statesnapshot {
 class KVBCStateSnapshot : public concord::kvbc::IKVBCStateSnapshot {

diff --git a/kvbc/include/kvbc_adapter/replica_adapter.hpp b/kvbc/include/kvbc_adapter/replica_adapter.hpp
@@ -21,7 +21,7 @@
 #include "util/assertUtils.hpp"
 
 #include "kv_types.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "blockchain_misc.hpp"
 #include "ReplicaConfig.hpp"
 #include "db_interfaces.h"
@@ -30,7 +30,7 @@
 #include "replica_adapter_auxilliary_types.hpp"
 #include "categorization/kv_blockchain.h"
 #include "v4blockchain/v4_blockchain.h"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 
 namespace concord::kvbc::adapter {
 class ReplicaBlockchain : public IBlocksDeleter,

diff --git a/kvbc/include/merkle_tree_db_adapter.h b/kvbc/include/merkle_tree_db_adapter.h
@@ -19,7 +19,7 @@
 #include "merkle_tree_block.h"
 #include "util/sliver.hpp"
 #include "sparse_merkle/tree.h"
-#include "storage/db_interface.h"
+#include "storage/utils/db_interface.hpp"
 #include "util/Statistics.hpp"
 #include "PerformanceManager.hpp"
 

diff --git a/kvbc/include/merkle_tree_key_manipulator.h b/kvbc/include/merkle_tree_key_manipulator.h
@@ -18,7 +18,7 @@
 #include "util/sliver.hpp"
 #include "sparse_merkle/base_types.h"
 #include "sparse_merkle/keys.h"
-#include "storage/db_types.h"
+#include "storage/utils/db_types.hpp"
 
 namespace concord::kvbc::v2MerkleTree::detail {
 

diff --git a/kvbc/include/merkle_tree_serialization.h b/kvbc/include/merkle_tree_serialization.h
@@ -21,7 +21,7 @@
 #include "sparse_merkle/base_types.h"
 #include "sparse_merkle/internal_node.h"
 #include "sparse_merkle/keys.h"
-#include "storage/db_types.h"
+#include "storage/utils/db_types.hpp"
 #include "util/string.hpp"
 
 #include <algorithm>

diff --git a/kvbc/include/migrations/block_merkle_latest_ver_cf_migration.h b/kvbc/include/migrations/block_merkle_latest_ver_cf_migration.h
@@ -12,7 +12,7 @@
 
 #pragma once
 
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 
 #include <rocksdb/utilities/checkpoint.h>
 

diff --git a/kvbc/include/replica_state_sync.h b/kvbc/include/replica_state_sync.h
@@ -15,7 +15,7 @@
 
 #pragma once
 
-#include "storage/db_types.h"
+#include "storage/utils/db_types.hpp"
 #include "db_adapter_interface.h"
 #include "kvbc_adapter/replica_adapter.hpp"
 #include "PersistentStorage.hpp"

diff --git a/kvbc/include/storage_factory_interface.h b/kvbc/include/storage_factory_interface.h
@@ -13,8 +13,8 @@
 #pragma once
 
 #include "db_adapter_interface.h"
-#include "storage/db_interface.h"
-#include "storage/key_manipulator_interface.h"
+#include "storage/utils/db_interface.hpp"
+#include "storage/utils/key_manipulator_interface.hpp"
 
 #include <memory>
 

diff --git a/kvbc/include/v4blockchain/detail/blockchain.h b/kvbc/include/v4blockchain/detail/blockchain.h
@@ -19,7 +19,7 @@
 #include <atomic>
 #include <limits>
 #include <mutex>
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include <memory>
 #include "kv_types.hpp"
 #include "v4blockchain/detail/blocks.h"

diff --git a/kvbc/include/v4blockchain/detail/categories.h b/kvbc/include/v4blockchain/detail/categories.h
@@ -15,7 +15,7 @@
 #include <string>
 #include <memory>
 #include <unordered_map>
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include <optional>
 #include "categorization/base_types.h"
 

diff --git a/kvbc/include/v4blockchain/detail/detail.h b/kvbc/include/v4blockchain/detail/detail.h
@@ -14,7 +14,7 @@
 #pragma once
 #include <chrono>
 #include "kvbc_key_types.hpp"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "blockchain_misc.hpp"
 
 namespace concord::kvbc::v4blockchain::detail {

diff --git a/kvbc/include/v4blockchain/detail/latest_keys.h b/kvbc/include/v4blockchain/detail/latest_keys.h
@@ -13,7 +13,7 @@
 
 #pragma once
 
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include <memory>
 #include <unordered_map>
 #include "categorization/updates.h"

diff --git a/kvbc/include/v4blockchain/detail/st_chain.h b/kvbc/include/v4blockchain/detail/st_chain.h
@@ -14,7 +14,7 @@
 #pragma once
 
 #include <memory>
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "kv_types.hpp"
 #include "v4blockchain/detail/blocks.h"
 #include "v4blockchain/detail/blockchain.h"

diff --git a/kvbc/include/v4blockchain/v4_blockchain.h b/kvbc/include/v4blockchain/v4_blockchain.h
@@ -14,7 +14,7 @@
 #pragma once
 
 #include "categorization/updates.h"
-#include "rocksdb/native_client.h"
+#include "storage/rocksdb/native_client.hpp"
 #include "v4blockchain/detail/st_chain.h"
 #include "v4blockchain/detail/latest_keys.h"
 #include "v4blockchain/detail/blockchain.h"