From f9570b2b27b769c5454830b6353bef5878c6568c Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Fri, 6 Sep 2024 10:26:10 +0200 Subject: [PATCH] implement missing stl_allocator stuff and fix warnings --- extension/httpfs/crypto.cpp | 12 +++-- .../internal/jemalloc_internal_defs.h | 2 + extension/jemalloc/jemalloc_extension.cpp | 6 --- .../types/column/column_data_collection.cpp | 7 +-- .../types/column/column_data_consumer.cpp | 2 +- src/common/vector_operations/vector_copy.cpp | 2 +- src/core_functions/lambda_functions.cpp | 2 +- src/execution/index/art/iterator.cpp | 4 +- src/execution/window_segment_tree.cpp | 2 +- src/function/cast/union_casts.cpp | 6 +-- src/include/duckdb/common/stl_allocator.hpp | 50 +++++++++---------- src/include/duckdb/common/winapi.hpp | 2 +- .../storage/compression/alp/algorithm/alp.hpp | 12 ++--- .../compression/alprd/algorithm/alprd.hpp | 2 +- src/planner/operator/logical_join.cpp | 2 +- third_party/concurrentqueue/concurrentqueue.h | 6 ++- 16 files changed, 60 insertions(+), 59 deletions(-) diff --git a/extension/httpfs/crypto.cpp b/extension/httpfs/crypto.cpp index 377c2bbae05..13c094af1fb 100644 --- a/extension/httpfs/crypto.cpp +++ b/extension/httpfs/crypto.cpp @@ -1,7 +1,9 @@ #include "crypto.hpp" + +#include "duckdb/common/common.hpp" #include "mbedtls_wrapper.hpp" + #include -#include "duckdb/common/common.hpp" #include #define CPPHTTPLIB_OPENSSL_SUPPORT @@ -111,7 +113,7 @@ size_t AESGCMStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, i auto text_len = out_len; switch (mode) { - case ENCRYPT: + case ENCRYPT: { if (1 != EVP_EncryptFinal_ex(gcm_context, data_ptr_cast(out) + out_len, reinterpret_cast(&out_len))) { throw InternalException("EncryptFinal failed"); } @@ -121,7 +123,8 @@ size_t AESGCMStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, i throw InternalException("Calculating the tag failed"); } return text_len; - case DECRYPT: + } + case DECRYPT: { // Set expected tag value if (!EVP_CIPHER_CTX_ctrl(gcm_context, EVP_CTRL_GCM_SET_TAG, tag_len, tag)) { throw InternalException("Finalizing tag failed"); @@ -136,6 +139,9 @@ size_t AESGCMStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, i } throw InvalidInputException("Computed AES tag differs from read AES tag, are you using the right key?"); } + default: + throw InternalException("Unknown mode in AESGCMStateSSL::Finalize"); + } } } // namespace duckdb diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h index fff513e65a6..e6b9b40a987 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h @@ -525,7 +525,9 @@ /* * If defined, all the features necessary for background threads are present. */ +#ifndef __APPLE__ #define JEMALLOC_BACKGROUND_THREAD +#endif /* * If defined, jemalloc symbols are not exported (doesn't work when diff --git a/extension/jemalloc/jemalloc_extension.cpp b/extension/jemalloc/jemalloc_extension.cpp index 10a49cf99a1..a8d546619c6 100644 --- a/extension/jemalloc/jemalloc_extension.cpp +++ b/extension/jemalloc/jemalloc_extension.cpp @@ -87,15 +87,9 @@ void JemallocExtension::ThreadIdle() { } void JemallocExtension::FlushAll() { - // Flush thread-local cache - SetJemallocCTL("thread.tcache.flush"); - // Flush all arenas const auto purge_arena = PurgeArenaString(MALLCTL_ARENAS_ALL); SetJemallocCTL(purge_arena.c_str()); - - // Reset the peak after resetting - SetJemallocCTL("thread.peak.reset"); } void JemallocExtension::SetBackgroundThreads(bool enable) { diff --git a/src/common/types/column/column_data_collection.cpp b/src/common/types/column/column_data_collection.cpp index c3740444aa2..482b9b5e406 100644 --- a/src/common/types/column/column_data_collection.cpp +++ b/src/common/types/column/column_data_collection.cpp @@ -1,14 +1,14 @@ #include "duckdb/common/types/column/column_data_collection.hpp" #include "duckdb/common/printer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" +#include "duckdb/common/serializer/serializer.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/column/column_data_collection_segment.hpp" #include "duckdb/common/types/value_map.hpp" #include "duckdb/common/uhugeint.hpp" #include "duckdb/common/vector_operations/vector_operations.hpp" #include "duckdb/storage/buffer_manager.hpp" -#include "duckdb/common/serializer/serializer.hpp" -#include "duckdb/common/serializer/deserializer.hpp" namespace duckdb { @@ -548,7 +548,8 @@ void ColumnDataCopy(ColumnDataMetaData &meta_data, const UnifiedVector } if (heap_size != 0) { - current_segment.swizzle_data.emplace_back(child_index, current_segment.count, append_count); + current_segment.swizzle_data.emplace_back(child_index, current_segment.count, + UnsafeNumericCast(append_count)); } current_segment.count += append_count; diff --git a/src/common/types/column/column_data_consumer.cpp b/src/common/types/column/column_data_consumer.cpp index fa20f1d5b01..a1f00b2156a 100644 --- a/src/common/types/column/column_data_consumer.cpp +++ b/src/common/types/column/column_data_consumer.cpp @@ -28,7 +28,7 @@ void ColumnDataConsumer::InitializeScan() { chunk_references.reserve(chunk_count); for (auto &segment : collection.GetSegments()) { for (idx_t chunk_index = 0; chunk_index < segment->chunk_data.size(); chunk_index++) { - chunk_references.emplace_back(segment.get(), chunk_index); + chunk_references.emplace_back(segment.get(), NumericCast(chunk_index)); } } std::sort(chunk_references.begin(), chunk_references.end()); diff --git a/src/common/vector_operations/vector_copy.cpp b/src/common/vector_operations/vector_copy.cpp index 0880f23be63..fb45e74a8a0 100644 --- a/src/common/vector_operations/vector_copy.cpp +++ b/src/common/vector_operations/vector_copy.cpp @@ -227,7 +227,7 @@ void VectorOperations::Copy(const Vector &source_p, Vector &target, const Select auto source_idx = sel->get_index(source_offset + i); auto &source_entry = sdata[source_idx]; for (idx_t j = 0; j < source_entry.length; ++j) { - child_rows.emplace_back(source_entry.offset + j); + child_rows.emplace_back(UnsafeNumericCast(source_entry.offset + j)); } } } diff --git a/src/core_functions/lambda_functions.cpp b/src/core_functions/lambda_functions.cpp index 9a3a0310210..c46005a898b 100644 --- a/src/core_functions/lambda_functions.cpp +++ b/src/core_functions/lambda_functions.cpp @@ -93,7 +93,7 @@ struct ListFilterFunctor { } //! Pushes an empty list to the entry_lengths vector static void PushEmptyList(vector &entry_lengths) { - entry_lengths.emplace_back(0); + entry_lengths.emplace_back(0ULL); } //! Pushes the length of the original list to the entry_lengths vector static void SetResultEntry(list_entry_t *, idx_t &, const list_entry_t &entry, const idx_t, diff --git a/src/execution/index/art/iterator.cpp b/src/execution/index/art/iterator.cpp index ca6f5c71305..0fd5989ad69 100644 --- a/src/execution/index/art/iterator.cpp +++ b/src/execution/index/art/iterator.cpp @@ -119,7 +119,7 @@ void Iterator::FindMinimum(const Node &node) { D_ASSERT(nested_depth < Prefix::ROW_ID_SIZE); } } - nodes.emplace(node, 0); + nodes.emplace(node, static_cast(0)); return FindMinimum(*prefix.ptr); } @@ -188,7 +188,7 @@ bool Iterator::LowerBound(const Node &node, const ARTKey &key, const bool equal, for (idx_t i = 0; i < prefix.data[Prefix::Count(art)]; i++) { current_key.Push(prefix.data[i]); } - nodes.emplace(node, 0); + nodes.emplace(node, static_cast(0)); // We compare the prefix bytes with the key bytes. for (idx_t i = 0; i < prefix.data[Prefix::Count(art)]; i++) { diff --git a/src/execution/window_segment_tree.cpp b/src/execution/window_segment_tree.cpp index be71e85c78b..04795a8a43f 100644 --- a/src/execution/window_segment_tree.cpp +++ b/src/execution/window_segment_tree.cpp @@ -218,7 +218,7 @@ WindowConstantAggregatorGlobalState::WindowConstantAggregatorGlobalState(const W // Locate the partition boundaries if (partition_mask.AllValid()) { - partition_offsets.emplace_back(0); + partition_offsets.emplace_back(0ULL); } else { idx_t entry_idx; idx_t shift; diff --git a/src/function/cast/union_casts.cpp b/src/function/cast/union_casts.cpp index 72b171745a3..c5bf4295e91 100644 --- a/src/function/cast/union_casts.cpp +++ b/src/function/cast/union_casts.cpp @@ -1,8 +1,8 @@ #include "duckdb/common/exception/conversion_exception.hpp" #include "duckdb/common/numeric_utils.hpp" +#include "duckdb/function/cast/bound_cast_data.hpp" #include "duckdb/function/cast/cast_function_set.hpp" #include "duckdb/function/cast/default_casts.hpp" -#include "duckdb/function/cast/bound_cast_data.hpp" #include // for std::sort @@ -24,8 +24,8 @@ unique_ptr BindToUnionCast(BindCastInput &input, const LogicalTyp auto member_cast_cost = input.function_set.ImplicitCastCost(source, member_type); if (member_cast_cost != -1) { auto member_cast_info = input.GetCastFunction(source, member_type); - candidates.emplace_back(member_idx, member_name, member_type, member_cast_cost, - std::move(member_cast_info)); + candidates.emplace_back(UnsafeNumericCast(member_idx), member_name, member_type, + member_cast_cost, std::move(member_cast_info)); } }; diff --git a/src/include/duckdb/common/stl_allocator.hpp b/src/include/duckdb/common/stl_allocator.hpp index 1bee21a405a..a11a290a0fd 100644 --- a/src/include/duckdb/common/stl_allocator.hpp +++ b/src/include/duckdb/common/stl_allocator.hpp @@ -8,12 +8,14 @@ #pragma once +#include "duckdb/common/winapi.hpp" + #include namespace duckdb { -void *stl_malloc(size_t size); // NOLINT: not using camelcase on purpose here -void stl_free(void *ptr); // NOLINT: not using camelcase on purpose here +DUCKDB_API void *stl_malloc(size_t size); // NOLINT: not using camelcase on purpose here +DUCKDB_API void stl_free(void *ptr); // NOLINT: not using camelcase on purpose here template T *stl_new_array_uninitialized(size_t size) { @@ -77,14 +79,15 @@ class stl_allocator { // NOLINT: not using camelcase on purpose here using const_reference = typename original::const_reference; using size_type = typename original::size_type; using difference_type = typename original::difference_type; - using propagate_on_container_move_assignment = typename original::propagate_on_container_move_assignment; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; template struct rebind { typedef stl_allocator other; }; using is_always_equal = typename original::is_always_equal; -public: stl_allocator() noexcept { } stl_allocator(const stl_allocator &) noexcept { @@ -95,28 +98,31 @@ class stl_allocator { // NOLINT: not using camelcase on purpose here ~stl_allocator() { } -public: - pointer allocate(size_type count, const void * = 0) { // NOLINT: matching name of std - return static_cast(stl_malloc(count * sizeof(value_type))); + pointer allocate(size_type n, const void * = 0) { // NOLINT: matching name of std + return static_cast(stl_malloc(n * sizeof(value_type))); } void deallocate(T *p, size_type) { // NOLINT: matching name of std stl_free(p); } -}; -template -bool operator==(const stl_allocator &, const stl_allocator &) noexcept { - return true; -} + size_type max_size() const noexcept { // NOLINT: matching name of std + return PTRDIFF_MAX / sizeof(value_type); + } -template -bool operator==(stl_allocator &, const stl_allocator &) noexcept { - return true; -} + template + void construct(U *p, Args &&...args) { // NOLINT: matching name of std + new (p) U(std::forward(args)...); + } + + template + void destroy(U *p) { // NOLINT: matching name of std + p->~U(); + } +}; template -bool operator==(const stl_allocator &, stl_allocator &) noexcept { +bool operator==(const stl_allocator &, const stl_allocator &) noexcept { return true; } @@ -125,14 +131,4 @@ bool operator!=(const stl_allocator &, const stl_allocator &) noexcept { return false; } -template -bool operator!=(stl_allocator &, const stl_allocator &) noexcept { - return false; -} - -template -bool operator!=(const stl_allocator &, stl_allocator &) noexcept { - return false; -} - } // namespace duckdb diff --git a/src/include/duckdb/common/winapi.hpp b/src/include/duckdb/common/winapi.hpp index 1b77335ca9a..e6a31b25d86 100644 --- a/src/include/duckdb/common/winapi.hpp +++ b/src/include/duckdb/common/winapi.hpp @@ -1,7 +1,7 @@ //===----------------------------------------------------------------------===// // DuckDB // -// duckdb/main/winapi.hpp +// duckdb/common/winapi.hpp // // //===----------------------------------------------------------------------===// diff --git a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp index f99ed09b093..717c51398fe 100644 --- a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp +++ b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp @@ -10,10 +10,10 @@ #include "duckdb/common/bitpacking.hpp" #include "duckdb/common/common.hpp" -#include "duckdb/common/unordered_map.hpp" -#include "duckdb/common/pair.hpp" #include "duckdb/common/limits.hpp" +#include "duckdb/common/pair.hpp" #include "duckdb/common/types/hash.hpp" +#include "duckdb/common/unordered_map.hpp" #include "duckdb/storage/compression/alp/alp_constants.hpp" #include "duckdb/storage/compression/alp/alp_utils.hpp" @@ -233,10 +233,10 @@ struct AlpCompression { // Note that this vector is always small (< 10 combinations) vector best_k_combinations; for (auto const &combination : best_k_combinations_hash) { - best_k_combinations.emplace_back( - combination.first, // Encoding Indices - combination.second, // N of times it appeared (hash value) - 0 // Compression size is irrelevant at this phase since we compare combinations from different vectors + best_k_combinations.emplace_back(combination.first, // Encoding Indices + combination.second, // N of times it appeared (hash value) + 0ULL // Compression size is irrelevant at this phase since we compare + // combinations from different vectors ); } sort(best_k_combinations.begin(), best_k_combinations.end(), CompareALPCombinations); diff --git a/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp b/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp index 1dac66f4ab9..454b3b4464f 100644 --- a/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp +++ b/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp @@ -80,7 +80,7 @@ struct AlpRDCompression { template static double BuildLeftPartsDictionary(const vector &values, uint8_t right_bit_width, State &state) { - unordered_map left_parts_hash; + unordered_map left_parts_hash; vector left_parts_sorted_repetitions; // Building a hash for all the left parts and how many times they appear diff --git a/src/planner/operator/logical_join.cpp b/src/planner/operator/logical_join.cpp index eacf7b8e75f..a5c4e640c31 100644 --- a/src/planner/operator/logical_join.cpp +++ b/src/planner/operator/logical_join.cpp @@ -18,7 +18,7 @@ vector LogicalJoin::GetColumnBindings() { if (join_type == JoinType::MARK) { // for MARK join we project the left hand side plus the MARK column - left_bindings.emplace_back(mark_index, 0); + left_bindings.emplace_back(mark_index, 0ULL); return left_bindings; } // for other join types we project both the LHS and the RHS diff --git a/third_party/concurrentqueue/concurrentqueue.h b/third_party/concurrentqueue/concurrentqueue.h index 0f5ad0a4d62..f1f8c33fe68 100644 --- a/third_party/concurrentqueue/concurrentqueue.h +++ b/third_party/concurrentqueue/concurrentqueue.h @@ -54,6 +54,8 @@ #include #include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading +#include "duckdb/common/stl_allocator.hpp" + // Platform-specific definitions of a numeric thread ID type and an invalid value namespace duckdb_moodycamel { namespace details { template struct thread_id_converter { @@ -335,8 +337,8 @@ struct ConcurrentQueueDefaultTraits static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); } static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); } #else - static inline void* malloc(size_t size) { return std::malloc(size); } - static inline void free(void* ptr) { return std::free(ptr); } + static inline void* malloc(size_t size) { return duckdb::stl_malloc(size); } + static inline void free(void* ptr) { return duckdb::stl_free(ptr); } #endif #else // Debug versions when running under the Relacy race detector (ignore