Skip to content

Commit

Permalink
Bump duckdb to e70015aeac5595391b87faf250b9e980cfda3516
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlopi
carlopi committed Jan 27, 2025
1 parent 31f5169 commit bcaec19
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 107 deletions.
167 changes: 76 additions & 91 deletions patches/duckdb/extension_install_rework.patch
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp
index 2a6fffa994..f4ef687ddd 100644
index ed956daa64..d4774faac5 100644
--- a/src/include/duckdb/main/database.hpp
+++ b/src/include/duckdb/main/database.hpp
@@ -96,6 +96,10 @@ private:
shared_ptr<DatabaseCacheEntry> db_cache_entry;
@@ -97,6 +97,10 @@ private:
shared_ptr<LogManager> log_manager;

duckdb_ext_api_v1 (*create_api_v1)();
+public:
Expand Down Expand Up @@ -31,10 +31,10 @@ index 6ccd1a1156..8040f537b6 100644
//! Debugging repositories (target local, relative paths that are produced by DuckDB's build system)
static constexpr const char *BUILD_DEBUG_REPOSITORY_PATH = "./build/debug/repository";
diff --git a/src/main/database.cpp b/src/main/database.cpp
index 0a82e4598e..7ef5b96f7d 100644
index 084dab6f30..6e21356bd9 100644
--- a/src/main/database.cpp
+++ b/src/main/database.cpp
@@ -336,6 +336,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t
@@ -344,6 +344,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t
DuckDB::~DuckDB() {
}

Expand Down Expand Up @@ -64,18 +64,18 @@ index 0a82e4598e..7ef5b96f7d 100644
return *config.secret_manager;
}
diff --git a/src/main/extension/extension_helper.cpp b/src/main/extension/extension_helper.cpp
index 494832417e..17a39d04b4 100644
index c7b613226a..00885ee6a4 100644
--- a/src/main/extension/extension_helper.cpp
+++ b/src/main/extension/extension_helper.cpp
@@ -328,7 +328,6 @@ vector<ExtensionUpdateResult> ExtensionHelper::UpdateExtensions(ClientContext &c
@@ -326,7 +326,6 @@ vector<ExtensionUpdateResult> ExtensionHelper::UpdateExtensions(ClientContext &c
vector<ExtensionUpdateResult> result;
DatabaseInstance &db = DatabaseInstance::GetDatabase(context);

-#ifndef WASM_LOADABLE_EXTENSIONS
case_insensitive_set_t seen_extensions;

// scan the install directory for installed extensions
@@ -345,7 +344,6 @@ vector<ExtensionUpdateResult> ExtensionHelper::UpdateExtensions(ClientContext &c
@@ -343,7 +342,6 @@ vector<ExtensionUpdateResult> ExtensionHelper::UpdateExtensions(ClientContext &c

result.push_back(UpdateExtensionInternal(context, db, fs, fs.JoinPath(ext_directory, path), extension_name));
});
Expand All @@ -84,10 +84,10 @@ index 494832417e..17a39d04b4 100644
return result;
}
diff --git a/src/main/extension/extension_install.cpp b/src/main/extension/extension_install.cpp
index b0ca9fb775..67dfcdfb26 100644
index e8ab595ab0..fb3e6371a3 100644
--- a/src/main/extension/extension_install.cpp
+++ b/src/main/extension/extension_install.cpp
@@ -144,6 +144,9 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
@@ -155,6 +155,9 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DatabaseInstance &db, FileSystem &fs,
const string &extension,
ExtensionInstallOptions &options) {
Expand All @@ -97,7 +97,7 @@ index b0ca9fb775..67dfcdfb26 100644
#ifdef WASM_LOADABLE_EXTENSIONS
// Install is currently a no-op
return nullptr;
@@ -154,6 +157,9 @@ unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DatabaseInsta
@@ -165,6 +168,9 @@ unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DatabaseInsta

unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(ClientContext &context, const string &extension,
ExtensionInstallOptions &options) {
Expand All @@ -107,7 +107,7 @@ index b0ca9fb775..67dfcdfb26 100644
#ifdef WASM_LOADABLE_EXTENSIONS
// Install is currently a no-op
return nullptr;
@@ -198,7 +204,7 @@ string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const DatabaseInstance
@@ -209,7 +215,7 @@ string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const DatabaseInstance
versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension";
}
#ifdef WASM_LOADABLE_EXTENSIONS
Expand All @@ -117,10 +117,10 @@ index b0ca9fb775..67dfcdfb26 100644
#else
string default_endpoint = ExtensionRepository::DEFAULT_REPOSITORY_URL;
diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp
index b0282a7103..b43d9c6684 100644
index 84b28fef09..ea2ebba1cd 100644
--- a/src/main/extension/extension_load.cpp
+++ b/src/main/extension/extension_load.cpp
@@ -302,7 +302,20 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
@@ -328,7 +328,20 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
direct_load = false;
string extension_name = ApplyExtensionAlias(extension);
#ifdef WASM_LOADABLE_EXTENSIONS
Expand All @@ -142,7 +142,7 @@ index b0282a7103..b43d9c6684 100644
string url = ExtensionFinalizeUrlTemplate(url_template, extension_name);

char *str = (char *)EM_ASM_PTR(
@@ -343,73 +356,223 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
@@ -369,35 +382,31 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
direct_load = true;
filename = fs.ExpandPath(filename);
}
Expand All @@ -155,8 +155,6 @@ index b0282a7103..b43d9c6684 100644
- error = StringUtil::Format("Extension \"%s\" not found.\n%s", filename, message);
- return false;
- }
-
- auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
+ /* if (!fs.FileExists(filename)) {
+ string message;
+ bool exact_match = ExtensionHelper::CreateSuggestions(extension, message);
Expand All @@ -167,27 +165,43 @@ index b0282a7103..b43d9c6684 100644
+ return false;
+ }
+ */
+

- auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
+ /* auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
+

- // Parse the extension metadata from the extension binary
- auto parsed_metadata = ParseExtensionMetaData(*handle);
+ // Parse the extension metadata from the extension binary
+ auto parsed_metadata = ParseExtensionMetaData(*handle);
+

- auto metadata_mismatch_error = parsed_metadata.GetInvalidMetadataError();
+ auto metadata_mismatch_error = parsed_metadata.GetInvalidMetadataError();
+

- if (!metadata_mismatch_error.empty()) {
- metadata_mismatch_error = StringUtil::Format("Failed to load '%s', %s", extension, metadata_mismatch_error);
- }
+ if (!metadata_mismatch_error.empty()) {
+ metadata_mismatch_error = StringUtil::Format("Failed to load '%s', %s", extension, metadata_mismatch_error);
+ }
+

- if (!db.config.options.allow_unsigned_extensions) {
- bool signature_valid;
- if (parsed_metadata.AppearsValid()) {
- signature_valid =
- CheckExtensionSignature(*handle, parsed_metadata, db.config.options.allow_community_extensions);
- } else {
- signature_valid = false;
- }
+ if (!db.config.options.allow_unsigned_extensions) {
+ bool signature_valid =
+ CheckExtensionSignature(*handle, parsed_metadata, db.config.options.allow_community_extensions);
+
+ if (!signature_valid) {
+ throw IOException(db.config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) +
+ metadata_mismatch_error);
+ }
+

if (!metadata_mismatch_error.empty()) {
throw InvalidInputException(metadata_mismatch_error);
@@ -414,26 +423,192 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
}
}

+ if (!metadata_mismatch_error.empty()) {
+ // Signed extensions perform the full check
+ throw InvalidInputException(metadata_mismatch_error);
Expand All @@ -200,19 +214,24 @@ index b0282a7103..b43d9c6684 100644
+ }
+ }
+ */
+ auto filebase = fs.ExtractBaseName(filename);
auto filebase = fs.ExtractBaseName(filename);

- // Parse the extension metadata from the extension binary
- auto parsed_metadata = ParseExtensionMetaData(*handle);
+#ifdef WASM_LOADABLE_EXTENSIONS
#ifdef WASM_LOADABLE_EXTENSIONS
- EM_ASM(
+ auto basename = fs.ExtractBaseName(filename);
+ char *exe = NULL;
+ exe = (char *)EM_ASM_PTR(
+ {
+ // Next few lines should argubly in separate JavaScript-land function call
+ // TODO: move them out / have them configurable

- auto metadata_mismatch_error = parsed_metadata.GetInvalidMetadataError();
{
// Next few lines should argubly in separate JavaScript-land function call
// TODO: move them out / have them configurable
- const xhr = new XMLHttpRequest();
- xhr.open("GET", UTF8ToString($0), false);
- xhr.responseType = "arraybuffer";
- xhr.send(null);
- var uInt8Array = xhr.response;
- WebAssembly.validate(uInt8Array);
- console.log('Loading extension ', UTF8ToString($1));
+
+ var url = (UTF8ToString($0));
+
+ if (typeof XMLHttpRequest === "undefined") {
Expand Down Expand Up @@ -303,25 +322,19 @@ index b0282a7103..b43d9c6684 100644
+ // console.log(new Uint8Array(Module.HEAPU8, fileOnWasmHeap, len+4));
+ // console.log('Loading extension ', UTF8ToString($1));

- if (!metadata_mismatch_error.empty()) {
- metadata_mismatch_error = StringUtil::Format("Failed to load '%s', %s", extension, metadata_mismatch_error);
+ // Here we add the uInt8Array to Emscripten's filesystem, for it to be found by dlopen
+ FS.writeFile(UTF8ToString($1), new Uint8Array(uInt8Array));
// Here we add the uInt8Array to Emscripten's filesystem, for it to be found by dlopen
FS.writeFile(UTF8ToString($1), new Uint8Array(uInt8Array));
+ return fileOnWasmHeap;
+ },
},
- filename.c_str(), filebase.c_str());
- auto dopen_from = filebase;
+ filename.c_str(), basename.c_str());
+ if (!exe) {
+ throw IOException("Extension %s is not available", filename);
}
+ }
+
+ auto dopen_from = basename;
if (!db.config.options.allow_unsigned_extensions) {
- bool signature_valid;
- if (parsed_metadata.AppearsValid()) {
- signature_valid =
- CheckExtensionSignature(*handle, parsed_metadata, db.config.options.allow_community_extensions);
- } else {
- signature_valid = false;
+ if (!db.config.options.allow_unsigned_extensions) {
+ // signature is the last 256 bytes of the file
+
+ string signature;
Expand All @@ -346,7 +359,7 @@ index b0282a7103..b43d9c6684 100644
+
+ for (idx_t i = 0; i < numChunks; i++) {
+ splits[i] = maxLenChunks * i;
}
+ }
+ splits.back() = signature_offset;
+
+ for (idx_t i = 0; i < numChunks; i++) {
Expand All @@ -360,17 +373,11 @@ index b0282a7103..b43d9c6684 100644
+
+ string hash_concatenation;
+ hash_concatenation.reserve(32 * numChunks); // 256 bits -> 32 bytes per chunk

- if (!signature_valid) {
- throw IOException(db.config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) +
- metadata_mismatch_error);
+
+ for (auto &hash_chunk : hash_chunks) {
+ hash_concatenation += hash_chunk;
}

- if (!metadata_mismatch_error.empty()) {
- // Signed extensions perform the full check
- throw InvalidInputException(metadata_mismatch_error);
+ }
+
+ string two_level_hash;
+ ComputeSHA256String(hash_concatenation, &two_level_hash);
+
Expand All @@ -383,45 +390,23 @@ index b0282a7103..b43d9c6684 100644
+ any_valid = true;
+ break;
+ }
}
- } else if (!db.config.options.allow_extensions_metadata_mismatch) {
- if (!metadata_mismatch_error.empty()) {
- // Unsigned extensions AND configuration allowing n, loading allowed, mainly for
- // debugging purposes
- throw InvalidInputException(metadata_mismatch_error);
+ }
+ if (!any_valid) {
+ throw IOException(db.config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename));
}
}
-
- auto filebase = fs.ExtractBaseName(filename);
-
-#ifdef WASM_LOADABLE_EXTENSIONS
- EM_ASM(
- {
- // Next few lines should argubly in separate JavaScript-land function call
- // TODO: move them out / have them configurable
- const xhr = new XMLHttpRequest();
- xhr.open("GET", UTF8ToString($0), false);
- xhr.responseType = "arraybuffer";
- xhr.send(null);
- var uInt8Array = xhr.response;
- WebAssembly.validate(uInt8Array);
- console.log('Loading extension ', UTF8ToString($1));
-
- // Here we add the uInt8Array to Emscripten's filesystem, for it to be found by dlopen
- FS.writeFile(UTF8ToString($1), new Uint8Array(uInt8Array));
- },
- filename.c_str(), filebase.c_str());
- auto dopen_from = filebase;
+ }
+ }
+ if (exe) {
+ free(exe);
+ }
#else
auto dopen_from = filename;
#endif
@@ -427,25 +590,27 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
@@ -449,28 +624,30 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
result.filebase = lowercase_extension_name;
result.filename = filename;
result.lib_hdl = lib_hdl;
- result.abi_type = parsed_metadata.abi_type;
+ result.abi_type = ExtensionABIType::CPP;

if (!direct_load) {
- auto info_file_name = filename + ".info";
Expand Down
13 changes: 0 additions & 13 deletions patches/duckdb/hardcode_abi_type.patch
View file

This file was deleted.

4 changes: 2 additions & 2 deletions patches/duckdb/smaller_casts.patch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ index 7e22cb932d..2b5b604982 100644
mask.SetInvalid(row_idx);
}
diff --git a/src/function/cast/string_cast.cpp b/src/function/cast/string_cast.cpp
index 9f8b5ee298..5fd0d370fe 100644
index f3a19c4273..187db70d93 100644
--- a/src/function/cast/string_cast.cpp
+++ b/src/function/cast/string_cast.cpp
@@ -22,8 +22,10 @@ bool StringEnumCastLoop(const string_t *source_data, ValidityMask &source_mask,
Expand All @@ -31,7 +31,7 @@ index 9f8b5ee298..5fd0d370fe 100644
result_data[i] = UnsafeNumericCast<T>(pos);
}
diff --git a/src/include/duckdb/common/operator/cast_operators.hpp b/src/include/duckdb/common/operator/cast_operators.hpp
index 3e4cb35e92..2b157157e2 100644
index d86b29d9c9..8a334fc0cd 100644
--- a/src/include/duckdb/common/operator/cast_operators.hpp
+++ b/src/include/duckdb/common/operator/cast_operators.hpp
@@ -51,19 +51,31 @@ struct TryCastErrorMessageCommaSeparated {
Expand Down
2 changes: 1 addition & 1 deletion submodules/duckdb
Submodule duckdb updated 95 files
+1 −1 .github/config/out_of_tree_extensions.cmake
+2 −2 .github/workflows/InternalIssuesCreateMirror.yml
+0 −2 .github/workflows/LinuxRelease.yml
+1 −66 .github/workflows/NightlyTests.yml
+1 −1 .github/workflows/Python.yml
+1 −0 .github/workflows/Regression.yml
+2 −0 data/csv/filename_filter/a.csv
+2 −0 data/csv/filename_filter/b.csv
+2 −0 data/csv/filename_filter/c.csv
+4 −0 data/csv/filename_filter/d.csv
+1 −1 extension/parquet/column_writer.cpp
+2 −2 src/common/arrow/arrow_converter.cpp
+32 −19 src/common/arrow/arrow_type_extension.cpp
+5 −1 src/common/compressed_file_system.cpp
+1 −1 src/common/error_data.cpp
+1 −1 src/common/multi_file_reader.cpp
+1 −1 src/common/types.cpp
+27 −34 src/execution/index/fixed_size_allocator.cpp
+2 −1 src/execution/index/fixed_size_buffer.cpp
+27 −25 src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp
+3 −7 src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp
+8 −8 src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp
+26 −9 src/execution/operator/csv_scanner/util/csv_error.cpp
+12 −9 src/execution/operator/csv_scanner/util/csv_reader_options.cpp
+5 −1 src/execution/sample/reservoir_sample.cpp
+6 −1 src/function/table/arrow/arrow_duck_schema.cpp
+10 −10 src/function/table/arrow_conversion.cpp
+1 −1 src/function/table/copy_csv.cpp
+3 −2 src/function/table/read_csv.cpp
+4 −3 src/function/window/window_token_tree.cpp
+2 −2 src/include/duckdb/common/arrow/arrow_type_extension.hpp
+4 −4 src/include/duckdb/execution/index/fixed_size_allocator.hpp
+37 −32 src/include/duckdb/execution/index/fixed_size_buffer.hpp
+5 −5 src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp
+1 −1 src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp
+0 −2 src/include/duckdb/logging/logging.hpp
+1 −0 src/include/duckdb/main/config.hpp
+2 −2 src/include/duckdb/storage/serialization/nodes.json
+4 −3 src/include/duckdb/storage/string_uncompressed.hpp
+3 −1 src/main/client_context.cpp
+13 −5 src/main/extension/CMakeLists.txt
+26 −2 src/parser/transform/expression/transform_function.cpp
+2 −2 src/storage/compression/fsst.cpp
+11 −6 src/storage/compression/string_uncompressed.cpp
+1 −1 src/storage/magic_bytes.cpp
+3 −3 src/storage/serialization/serialize_nodes.cpp
+7 −0 test/api/test_pending_query.cpp
+6 −0 test/sql/attach/attach_new_compression.test
+3 −0 test/sql/attach/attach_storage_version.test
+1 −1 test/sql/copy/csv/14512.test
+84 −84 test/sql/copy/csv/afl/test_fuzz_3977.test
+3 −3 test/sql/copy/csv/auto/test_csv_auto.test
+2 −2 test/sql/copy/csv/bug_10283.test_slow
+1 −1 test/sql/copy/csv/csv_duck_fuzz.test
+20 −0 test/sql/copy/csv/duck_fuzz/test_internal_4048.test
+7 −7 test/sql/copy/csv/multidelimiter/test_abac.test
+1 −1 test/sql/copy/csv/pollock/test_field_delimiter.test
+1 −1 test/sql/copy/csv/pollock/test_quotation_char.test
+4 −4 test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test
+1 −1 test/sql/copy/csv/rejects/csv_rejects_read.test
+4 −4 test/sql/copy/csv/rejects/csv_unquoted_rejects.test
+1 −1 test/sql/copy/csv/rejects/test_mixed.test
+11 −11 test/sql/copy/csv/rejects/test_multiple_errors_same_line.test
+15 −15 test/sql/copy/csv/relaxed_quotes.test
+3 −3 test/sql/copy/csv/test_big_line.test_slow
+3 −3 test/sql/copy/csv/test_bug_9952.test_slow
+1 −1 test/sql/copy/csv/test_comment_midline.test
+1 −1 test/sql/copy/csv/test_copy.test
+1 −1 test/sql/copy/csv/test_csv_column_count_mismatch.test_slow
+1 −1 test/sql/copy/csv/test_extra_delimiters_rfc.test
+37 −0 test/sql/copy/csv/test_filename_filter.test
+11 −1 test/sql/copy/csv/test_hits_problematic.test
+1 −1 test/sql/copy/csv/test_ignore_errors.test
+1 −1 test/sql/copy/csv/test_ignore_mid_null_line.test
+1 −1 test/sql/copy/csv/test_mixed_line_endings.test
+3 −3 test/sql/copy/csv/test_mixed_lines.test_slow
+5 −5 test/sql/copy/csv/test_mixed_new_line.test
+8 −8 test/sql/copy/csv/test_null_padding_projection.test
+9 −3 test/sql/copy/csv/test_thijs_unquoted_file.test
+1 −1 test/sql/copy/csv/unquoted_escape/32k_rows.test_slow
+2 −2 test/sql/copy/csv/unquoted_escape/basic.test
+2 −2 test/sql/copy/csv/unquoted_escape/human_eval.test
+1 −1 test/sql/copy/csv/unquoted_escape/identical.test
+1 −1 test/sql/copy/csv/unquoted_escape/mixed.test
+3 −0 test/sql/index/art/types/test_art_union.test
+5 −0 test/sql/settings/errors_as_json.test
+6 −0 test/sql/storage/compression/fsst/fsst_disable_compression.test
+3 −0 test/sql/storage/storage_versions.test
+24 −0 test/sql/window/test_cume_dist_orderby.test
+63 −0 test/sql/window/test_invalid_window.test
+7 −10 tools/pythonpkg/src/arrow/arrow_array_stream.cpp
+0 −1 tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp
+26 −27 tools/pythonpkg/src/pyconnection.cpp
+24 −0 tools/pythonpkg/tests/fast/api/test_read_csv.py
+8 −2 tools/pythonpkg/tests/fast/arrow/test_polars.py

0 comments on commit bcaec19

Please sign in to comment.