diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8c5fc5e3a98e42..c7c41345761643 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,59 +1,35 @@ ### What problem does this PR solve? - - - + Issue Number: close #xxx - Related PR: #xxx Problem Summary: -### Check List (For Committer) +### Release note -- Test +None +### Check List (For Author) + +- Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - - [ ] No colde files have been changed. + - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - - [ ] No. - [ ] Yes. - Does this need documentation? - - [ ] No. - [ ] Yes. -- Release note - - - - None - ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index d9efe6dbedde24..8a034001378f6f 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1436,15 +1436,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr DCHECK_EQ(existed_fs->type(), io::FileSystemType::S3) << param.id << ' ' << param.name; auto client = static_cast(existed_fs.get())->client_holder(); auto new_s3_conf = S3Conf::get_s3_conf(param.s3_storage_param); - S3ClientConf conf { - .endpoint {}, - .region {}, - .ak = std::move(new_s3_conf.client_conf.ak), - .sk = std::move(new_s3_conf.client_conf.sk), - .token = std::move(new_s3_conf.client_conf.token), - .bucket {}, - .provider = new_s3_conf.client_conf.provider, - }; + S3ClientConf conf = std::move(new_s3_conf.client_conf); st = client->reset(conf); fs = std::move(existed_fs); } @@ -1452,7 +1444,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr if (!st.ok()) { LOG(WARNING) << "update s3 resource failed: " << st; } else { - LOG_INFO("successfully update hdfs resource") + LOG_INFO("successfully update s3 resource") .tag("resource_id", param.id) .tag("resource_name", param.name); put_storage_resource(param.id, {std::move(fs)}, param.version); diff --git a/be/src/apache-orc b/be/src/apache-orc index 903ea6ccdc463b..db01184f765c03 160000 --- a/be/src/apache-orc +++ b/be/src/apache-orc @@ -1 +1 @@ -Subproject commit 903ea6ccdc463b8a17af2604975107ba7d895380 +Subproject commit db01184f765c03496e4107bd3ac37c077ac4bc5f diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 8eb92577693487..6b74e70ee1b4b8 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -363,12 +363,12 @@ Status CloudCumulativeCompaction::modify_rowsets() { if (config::enable_delete_bitmap_merge_on_compaction && _tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { - process_old_version_delete_bitmap(); + RETURN_IF_ERROR(process_old_version_delete_bitmap()); } return Status::OK(); } -void CloudCumulativeCompaction::process_old_version_delete_bitmap() { +Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { // agg previously rowset old version delete bitmap std::vector pre_rowsets {}; std::vector pre_rowset_ids {}; @@ -407,40 +407,29 @@ void CloudCumulativeCompaction::process_old_version_delete_bitmap() { } if (!new_delete_bitmap->empty()) { // store agg delete bitmap - Status update_st; DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed", { - update_st = Status::InternalError( + return Status::InternalError( "test fail to update delete bitmap for tablet_id {}", cloud_tablet()->tablet_id()); }); - if (update_st.ok()) { - update_st = _engine.meta_mgr().update_delete_bitmap_without_lock( - *cloud_tablet(), new_delete_bitmap.get()); - } - if (!update_st.ok()) { - std::stringstream ss; - ss << "failed to update delete bitmap for tablet=" << cloud_tablet()->tablet_id() - << " st=" << update_st.to_string(); - std::string msg = ss.str(); - LOG(WARNING) << msg; - } else { - Version version(_input_rowsets.front()->start_version(), - _input_rowsets.back()->end_version()); - for (auto it = new_delete_bitmap->delete_bitmap.begin(); - it != new_delete_bitmap->delete_bitmap.end(); it++) { - _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); - } - _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), - to_remove_vec); - DBUG_EXECUTE_IF( - "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { - static_cast(_tablet.get()) - ->delete_expired_stale_rowsets(); - }); + RETURN_IF_ERROR(_engine.meta_mgr().cloud_update_delete_bitmap_without_lock( + *cloud_tablet(), new_delete_bitmap.get())); + + Version version(_input_rowsets.front()->start_version(), + _input_rowsets.back()->end_version()); + for (auto it = new_delete_bitmap->delete_bitmap.begin(); + it != new_delete_bitmap->delete_bitmap.end(); it++) { + _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); } + _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), + to_remove_vec); + DBUG_EXECUTE_IF( + "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", + { static_cast(_tablet.get())->delete_expired_stale_rowsets(); }); } } + return Status::OK(); } void CloudCumulativeCompaction::garbage_collection() { diff --git a/be/src/cloud/cloud_cumulative_compaction.h b/be/src/cloud/cloud_cumulative_compaction.h index 62c7cb44ea5bf5..1159dcb59ceef1 100644 --- a/be/src/cloud/cloud_cumulative_compaction.h +++ b/be/src/cloud/cloud_cumulative_compaction.h @@ -47,7 +47,7 @@ class CloudCumulativeCompaction : public CloudCompactionMixin { void update_cumulative_point(); - void process_old_version_delete_bitmap(); + Status process_old_version_delete_bitmap(); ReaderType compaction_type() const override { return ReaderType::READER_CUMULATIVE_COMPACTION; } diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/cloud/cloud_delete_bitmap_action.cpp index 672574a5aa8901..60db5896dfab8a 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ b/be/src/cloud/cloud_delete_bitmap_action.cpp @@ -95,8 +95,6 @@ Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); auto size = tablet->tablet_meta()->delete_bitmap().get_size(); - LOG(INFO) << "show_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count - << ",cardinality=" << cardinality << ",size=" << size; rapidjson::Document root; root.SetObject(); diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 74d14911f62b98..7e52d7e5949384 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -704,11 +704,19 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ const auto& segment_ids = res.segment_ids(); const auto& vers = res.versions(); const auto& delete_bitmaps = res.segment_delete_bitmaps(); + if (rowset_ids.size() != segment_ids.size() || rowset_ids.size() != vers.size() || + rowset_ids.size() != delete_bitmaps.size()) { + return Status::Error( + "get delete bitmap data wrong," + "rowset_ids.size={},segment_ids.size={},vers.size={},delete_bitmaps.size={}", + rowset_ids.size(), segment_ids.size(), vers.size(), delete_bitmaps.size()); + } for (size_t i = 0; i < rowset_ids.size(); i++) { RowsetId rst_id; rst_id.init(rowset_ids[i]); - delete_bitmap->merge({rst_id, segment_ids[i], vers[i]}, - roaring::Roaring::read(delete_bitmaps[i].data())); + delete_bitmap->merge( + {rst_id, segment_ids[i], vers[i]}, + roaring::Roaring::readSafe(delete_bitmaps[i].data(), delete_bitmaps[i].length())); } int64_t latency = cntl.latency_us(); if (latency > 100 * 1000) { // 100ms @@ -1061,9 +1069,9 @@ Status CloudMetaMgr::update_delete_bitmap(const CloudTablet& tablet, int64_t loc return st; } -Status CloudMetaMgr::update_delete_bitmap_without_lock(const CloudTablet& tablet, - DeleteBitmap* delete_bitmap) { - LOG(INFO) << "update_delete_bitmap_without_lock , tablet_id: " << tablet.tablet_id() +Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(const CloudTablet& tablet, + DeleteBitmap* delete_bitmap) { + LOG(INFO) << "cloud_update_delete_bitmap_without_lock , tablet_id: " << tablet.tablet_id() << ",delete_bitmap size:" << delete_bitmap->delete_bitmap.size(); UpdateDeleteBitmapRequest req; UpdateDeleteBitmapResponse res; @@ -1215,12 +1223,8 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) { } if (rs_meta.tablet_schema()->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - auto indices = rs_meta.tablet_schema()->indexes(); + const auto& indices = rs_meta.tablet_schema()->inverted_indexes(); for (auto& index : indices) { - // only get file_size for inverted index - if (index.index_type() != IndexType::INVERTED) { - continue; - } for (int seg_id = 0; seg_id < rs_meta.num_segments(); ++seg_id) { std::string segment_path = StorageResource().remote_segment_path( rs_meta.tablet_id(), rs_meta.rowset_id().to_string(), seg_id); @@ -1229,7 +1233,7 @@ int64_t CloudMetaMgr::get_inverted_index_file_szie(const RowsetMeta& rs_meta) { std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(segment_path), - index.index_id(), index.get_index_suffix()); + index->index_id(), index->get_index_suffix()); auto st = fs->file_size(inverted_index_file_path, &file_size); if (!st.ok()) { file_size = 0; diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index a6d7ccd201f608..2b103e20f12a8f 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -95,8 +95,8 @@ class CloudMetaMgr { Status update_delete_bitmap(const CloudTablet& tablet, int64_t lock_id, int64_t initiator, DeleteBitmap* delete_bitmap); - Status update_delete_bitmap_without_lock(const CloudTablet& tablet, - DeleteBitmap* delete_bitmap); + Status cloud_update_delete_bitmap_without_lock(const CloudTablet& tablet, + DeleteBitmap* delete_bitmap); Status get_delete_bitmap_update_lock(const CloudTablet& tablet, int64_t lock_id, int64_t initiator); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index b467703637c961..601e9486edf031 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -288,15 +288,13 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ auto schema_ptr = rowset_meta->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); if (idx_version == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : schema_ptr->indexes()) { - if (index.index_type() == IndexType::INVERTED) { - auto idx_path = storage_resource.value()->remote_idx_v1_path( - *rowset_meta, seg_id, index.index_id(), - index.get_index_suffix()); - download_idx_file(idx_path); - } + for (const auto& index : schema_ptr->inverted_indexes()) { + auto idx_path = storage_resource.value()->remote_idx_v1_path( + *rowset_meta, seg_id, index->index_id(), + index->get_index_suffix()); + download_idx_file(idx_path); } - } else if (idx_version == InvertedIndexStorageFormatPB::V2) { + } else { if (schema_ptr->has_inverted_index()) { auto idx_path = storage_resource.value()->remote_idx_v2_path( *rowset_meta, seg_id); diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 07beeaeb078a46..06d6df11dc4cc3 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -63,14 +63,14 @@ void CloudWarmUpManager::handle_jobs() { #ifndef BE_TEST constexpr int WAIT_TIME_SECONDS = 600; while (true) { - JobMeta cur_job; + std::shared_ptr cur_job = nullptr; { std::unique_lock lock(_mtx); _cond.wait(lock, [this]() { return _closed || !_pending_job_metas.empty(); }); if (_closed) break; - cur_job = std::move(_pending_job_metas.front()); + cur_job = _pending_job_metas.front(); } - for (int64_t tablet_id : cur_job.tablet_ids) { + for (int64_t tablet_id : cur_job->tablet_ids) { if (_cur_job_id == 0) { // The job is canceled break; } @@ -147,15 +147,13 @@ void CloudWarmUpManager::handle_jobs() { auto schema_ptr = rs->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); if (idx_version == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : schema_ptr->indexes()) { - if (index.index_type() == IndexType::INVERTED) { - wait->add_count(); - auto idx_path = storage_resource.value()->remote_idx_v1_path( - *rs, seg_id, index.index_id(), index.get_index_suffix()); - download_idx_file(idx_path); - } + for (const auto& index : schema_ptr->inverted_indexes()) { + wait->add_count(); + auto idx_path = storage_resource.value()->remote_idx_v1_path( + *rs, seg_id, index->index_id(), index->get_index_suffix()); + download_idx_file(idx_path); } - } else if (idx_version == InvertedIndexStorageFormatPB::V2) { + } else { if (schema_ptr->has_inverted_index()) { wait->add_count(); auto idx_path = @@ -173,7 +171,7 @@ void CloudWarmUpManager::handle_jobs() { } { std::unique_lock lock(_mtx); - _finish_job.push_back(std::move(cur_job)); + _finish_job.push_back(cur_job); _pending_job_metas.pop_front(); } } @@ -230,8 +228,9 @@ Status CloudWarmUpManager::check_and_set_batch_id(int64_t job_id, int64_t batch_ void CloudWarmUpManager::add_job(const std::vector& job_metas) { { std::lock_guard lock(_mtx); - std::for_each(job_metas.begin(), job_metas.end(), - [this](const TJobMeta& meta) { _pending_job_metas.emplace_back(meta); }); + std::for_each(job_metas.begin(), job_metas.end(), [this](const TJobMeta& meta) { + _pending_job_metas.emplace_back(std::make_shared(meta)); + }); } _cond.notify_all(); } diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index fd034b2c5bc38c..219dedc58065a6 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -74,8 +74,8 @@ class CloudWarmUpManager { std::condition_variable _cond; int64_t _cur_job_id {0}; int64_t _cur_batch_id {-1}; - std::deque _pending_job_metas; - std::vector _finish_job; + std::deque> _pending_job_metas; + std::vector> _finish_job; std::thread _download_thread; bool _closed {false}; // the attribute for compile in ut diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 1f780824e32c3d..b6b8d3934eecc6 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -503,6 +503,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in) if (in.has_schema_version()) { out->set_schema_version(in.schema_version()); } + out->set_storage_page_size(in.storage_page_size()); } void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { @@ -569,6 +570,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { if (in.has_schema_version()) { out->set_schema_version(in.schema_version()); } + out->set_storage_page_size(in.storage_page_size()); } TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) { @@ -644,6 +646,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in) if (in.has_schema_version()) { out->set_schema_version(in.schema_version()); } + out->set_storage_page_size(in.storage_page_size()); } void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { @@ -710,6 +713,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { if (in.has_schema_version()) { out->set_schema_version(in.schema_version()); } + out->set_storage_page_size(in.storage_page_size()); } } // namespace doris::cloud diff --git a/be/src/clucene b/be/src/clucene index 5a458e6112b7e5..7cf6cf410d41d9 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit 5a458e6112b7e5010262594147adf22830b096e6 +Subproject commit 7cf6cf410d41d95456edba263cc55b7b6f5ab027 diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 16d5f2daba61e7..3aa0e944a822c5 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -26,6 +26,9 @@ #include #include +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_tablet_mgr.h" #include "cloud/config.h" #include "common/status.h" #include "olap/olap_common.h" @@ -35,6 +38,7 @@ #include "olap/tablet.h" #include "olap/tablet_manager.h" #include "runtime/define_primitive_type.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/runtime_profile.h" #include "vec/common/string_ref.h" @@ -78,7 +82,19 @@ Status SchemaRowsetsScanner::start(RuntimeState* state) { Status SchemaRowsetsScanner::_get_all_rowsets() { if (config::is_cloud_mode()) { - return Status::NotSupported("SchemaRowsetsScanner::_get_all_rowsets is not implemented"); + // only query cloud tablets in lru cache instead of all tablets + std::vector> tablets = + ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_mgr().get_weak_tablets(); + for (const std::weak_ptr& tablet : tablets) { + if (!tablet.expired()) { + auto t = tablet.lock(); + std::shared_lock rowset_ldlock(t->get_header_lock()); + for (const auto& it : t->rowset_map()) { + rowsets_.emplace_back(it.second); + } + } + } + return Status::OK(); } std::vector tablets = ExecEnv::GetInstance()->storage_engine().to_local().tablet_manager()->get_all_tablet(); diff --git a/be/src/http/action/file_cache_action.cpp b/be/src/http/action/file_cache_action.cpp index f31c040c5cf672..740bac46edf2a7 100644 --- a/be/src/http/action/file_cache_action.cpp +++ b/be/src/http/action/file_cache_action.cpp @@ -17,10 +17,15 @@ #include "file_cache_action.h" +#include + +#include #include #include #include #include +#include +#include #include "common/status.h" #include "http/http_channel.h" @@ -30,6 +35,7 @@ #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" #include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" #include "olap/olap_define.h" #include "olap/tablet_meta.h" #include "util/easy_json.h" @@ -43,6 +49,7 @@ constexpr static std::string_view PATH = "path"; constexpr static std::string_view CLEAR = "clear"; constexpr static std::string_view RESET = "reset"; constexpr static std::string_view HASH = "hash"; +constexpr static std::string_view LIST_CACHE = "list_cache"; constexpr static std::string_view CAPACITY = "capacity"; constexpr static std::string_view RELEASE = "release"; constexpr static std::string_view BASE_PATH = "base_path"; @@ -66,7 +73,14 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri *json_metrics = json.ToString(); } else if (operation == CLEAR) { const std::string& sync = req->param(SYNC.data()); - auto ret = io::FileCacheFactory::instance()->clear_file_caches(to_lower(sync) == "true"); + const std::string& segment_path = req->param(VALUE.data()); + if (segment_path.empty()) { + io::FileCacheFactory::instance()->clear_file_caches(to_lower(sync) == "true"); + } else { + io::UInt128Wrapper hash = io::BlockFileCache::hash(segment_path); + io::BlockFileCache* cache = io::FileCacheFactory::instance()->get_by_path(hash); + cache->remove_if_cached(hash); + } } else if (operation == RESET) { std::string capacity = req->param(CAPACITY.data()); int64_t new_capacity = 0; @@ -96,6 +110,23 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri json[HASH.data()] = ret.to_string(); *json_metrics = json.ToString(); } + } else if (operation == LIST_CACHE) { + const std::string& segment_path = req->param(VALUE.data()); + if (segment_path.empty()) { + st = Status::InvalidArgument("missing parameter: {} is required", VALUE.data()); + } else { + io::UInt128Wrapper cache_hash = io::BlockFileCache::hash(segment_path); + std::vector cache_files = + io::FileCacheFactory::instance()->get_cache_file_by_path(cache_hash); + if (cache_files.empty()) { + *json_metrics = "[]"; + } else { + EasyJson json; + std::for_each(cache_files.begin(), cache_files.end(), + [&json](auto& x) { json.PushBack(x); }); + *json_metrics = json.ToString(); + } + } } else { st = Status::InternalError("invalid operation: {}", operation); } diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 8370962ddd5fe1..2d0d25735fe2fd 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -21,6 +21,9 @@ #include "io/cache/block_file_cache_factory.h" #include + +#include +#include #if defined(__APPLE__) #include #else @@ -118,6 +121,20 @@ Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, return Status::OK(); } +std::vector FileCacheFactory::get_cache_file_by_path(const UInt128Wrapper& hash) { + io::BlockFileCache* cache = io::FileCacheFactory::instance()->get_by_path(hash); + auto blocks = cache->get_blocks_by_key(hash); + std::vector ret; + if (blocks.empty()) { + return ret; + } else { + for (auto& [_, fb] : blocks) { + ret.emplace_back(fb->get_cache_file()); + } + } + return ret; +} + BlockFileCache* FileCacheFactory::get_by_path(const UInt128Wrapper& key) { // dont need lock mutex because _caches is immutable after create_file_cache return _caches[KeyHash()(key) % _caches.size()].get(); diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index 12714fd2087982..b00bd7bdfcb315 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -62,6 +62,8 @@ class FileCacheFactory { [[nodiscard]] size_t get_cache_instance_size() const { return _caches.size(); } + std::vector get_cache_file_by_path(const UInt128Wrapper& hash); + BlockFileCache* get_by_path(const UInt128Wrapper& hash); BlockFileCache* get_by_path(const std::string& cache_base_path); std::vector get_query_context_holders( diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp index 4576b9dbba892f..44cad5520ead06 100644 --- a/be/src/io/cache/file_block.cpp +++ b/be/src/io/cache/file_block.cpp @@ -272,6 +272,10 @@ std::string FileBlock::state_to_string(FileBlock::State state) { } } +std::string FileBlock::get_cache_file() const { + return _mgr->_storage->get_local_file(this->_key); +} + FileBlocksHolder::~FileBlocksHolder() { for (auto file_block_it = file_blocks.begin(); file_block_it != file_blocks.end();) { auto current_file_block_it = file_block_it; diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h index 6e49a597b7b95c..3a4490d67a3f9d 100644 --- a/be/src/io/cache/file_block.h +++ b/be/src/io/cache/file_block.h @@ -123,6 +123,8 @@ class FileBlock { uint64_t expiration_time() const { return _key.meta.expiration_time; } + std::string get_cache_file() const; + State state_unlock(std::lock_guard&) const; FileBlock& operator=(const FileBlock&) = delete; diff --git a/be/src/io/cache/file_cache_storage.h b/be/src/io/cache/file_cache_storage.h index 642c4711cf6c62..024e701c6fa08b 100644 --- a/be/src/io/cache/file_cache_storage.h +++ b/be/src/io/cache/file_cache_storage.h @@ -65,6 +65,8 @@ class FileCacheStorage { // force clear all current data in the cache virtual Status clear(std::string& msg) = 0; virtual FileCacheStorageType get_type() = 0; + // get local cached file + virtual std::string get_local_file(const FileCacheKey& key) = 0; }; } // namespace doris::io diff --git a/be/src/io/cache/fs_file_cache_storage.cpp b/be/src/io/cache/fs_file_cache_storage.cpp index bacd0820c66099..d99869c1c8fb89 100644 --- a/be/src/io/cache/fs_file_cache_storage.cpp +++ b/be/src/io/cache/fs_file_cache_storage.cpp @@ -660,6 +660,11 @@ Status FSFileCacheStorage::clear(std::string& msg) { return Status::OK(); } +std::string FSFileCacheStorage::get_local_file(const FileCacheKey& key) { + return get_path_in_local_cache(get_path_in_local_cache(key.hash, key.meta.expiration_time), + key.offset, key.meta.type, false); +} + FSFileCacheStorage::~FSFileCacheStorage() { if (_cache_background_load_thread.joinable()) { _cache_background_load_thread.join(); diff --git a/be/src/io/cache/fs_file_cache_storage.h b/be/src/io/cache/fs_file_cache_storage.h index 23e98f422ac884..fb3490bcfe0ca3 100644 --- a/be/src/io/cache/fs_file_cache_storage.h +++ b/be/src/io/cache/fs_file_cache_storage.h @@ -70,6 +70,7 @@ class FSFileCacheStorage : public FileCacheStorage { void load_blocks_directly_unlocked(BlockFileCache* _mgr, const FileCacheKey& key, std::lock_guard& cache_lock) override; Status clear(std::string& msg) override; + std::string get_local_file(const FileCacheKey& key) override; [[nodiscard]] static std::string get_path_in_local_cache(const std::string& dir, size_t offset, FileCacheType type, diff --git a/be/src/io/cache/mem_file_cache_storage.cpp b/be/src/io/cache/mem_file_cache_storage.cpp index bffa75ae305b59..7e76dd5f88c565 100644 --- a/be/src/io/cache/mem_file_cache_storage.cpp +++ b/be/src/io/cache/mem_file_cache_storage.cpp @@ -128,4 +128,8 @@ Status MemFileCacheStorage::clear(std::string& msg) { return Status::OK(); } +std::string MemFileCacheStorage::get_local_file(const FileCacheKey& key) { + return ""; +} + } // namespace doris::io diff --git a/be/src/io/cache/mem_file_cache_storage.h b/be/src/io/cache/mem_file_cache_storage.h index 20fdd8ce9f6520..82064c6e9edc78 100644 --- a/be/src/io/cache/mem_file_cache_storage.h +++ b/be/src/io/cache/mem_file_cache_storage.h @@ -44,6 +44,7 @@ class MemFileCacheStorage : public FileCacheStorage { void load_blocks_directly_unlocked(BlockFileCache* _mgr, const FileCacheKey& key, std::lock_guard& cache_lock) override; Status clear(std::string& msg) override; + std::string get_local_file(const FileCacheKey& key) override; FileCacheStorageType get_type() override { return MEMORY; } diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 20d5684734e2d4..7fd85caa43b6c0 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -869,5 +869,107 @@ Result DelegateReader::create_file_reader( return reader; }); } + +Status LinearProbeRangeFinder::get_range_for(int64_t desired_offset, + io::PrefetchRange& result_range) { + while (index < _ranges.size()) { + io::PrefetchRange& range = _ranges[index]; + if (range.end_offset > desired_offset) { + if (range.start_offset > desired_offset) [[unlikely]] { + return Status::InvalidArgument("Invalid desiredOffset"); + } + result_range = range; + return Status::OK(); + } + ++index; + } + return Status::InvalidArgument("Invalid desiredOffset"); +} + +RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileReaderSPtr inner_reader, + std::shared_ptr range_finder) + : _profile(profile), + _inner_reader(std::move(inner_reader)), + _range_finder(std::move(range_finder)) { + _size = _inner_reader->size(); + uint64_t max_cache_size = + std::max((uint64_t)4096, (uint64_t)_range_finder->get_max_range_size()); + _cache = OwnedSlice(max_cache_size); + + if (_profile != nullptr) { + const char* random_profile = "RangeCacheFileReader"; + ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _request_io = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, + random_profile, 1); + _request_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RequestTime", random_profile, 1); + _read_to_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadToCacheTime", random_profile, 1); + _cache_refresh_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "CacheRefreshCount", + TUnit::UNIT, random_profile, 1); + _read_to_cache_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadToCacheBytes", + TUnit::BYTES, random_profile, 1); + } +} + +Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) { + auto request_size = result.size; + + _cache_statistics.request_io++; + _cache_statistics.request_bytes += request_size; + SCOPED_RAW_TIMER(&_cache_statistics.request_time); + + PrefetchRange range; + if (_range_finder->get_range_for(offset, range)) [[likely]] { + if (_current_start_offset != range.start_offset) { // need read new range to cache. + auto range_size = range.end_offset - range.start_offset; + + _cache_statistics.cache_refresh_count++; + _cache_statistics.read_to_cache_bytes += range_size; + SCOPED_RAW_TIMER(&_cache_statistics.read_to_cache_time); + + Slice cache_slice = {_cache.data(), range_size}; + RETURN_IF_ERROR( + _inner_reader->read_at(range.start_offset, cache_slice, bytes_read, io_ctx)); + + if (*bytes_read != range_size) [[unlikely]] { + return Status::InternalError( + "RangeCacheFileReader use inner reader read bytes {} not eq expect size {}", + *bytes_read, range_size); + } + + _current_start_offset = range.start_offset; + } + + int64_t buffer_offset = offset - _current_start_offset; + memcpy(result.data, _cache.data() + buffer_offset, request_size); + *bytes_read = request_size; + + return Status::OK(); + } else { + return Status::InternalError("RangeCacheFileReader read not in Ranges. Offset = {}", + offset); + // RETURN_IF_ERROR(_inner_reader->read_at(offset, result , bytes_read, io_ctx)); + // return Status::OK(); + // think return error is ok,otherwise it will cover up the error. + } +} + +void RangeCacheFileReader::_collect_profile_before_close() { + if (_profile != nullptr) { + COUNTER_UPDATE(_request_io, _cache_statistics.request_io); + COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); + COUNTER_UPDATE(_request_time, _cache_statistics.request_time); + COUNTER_UPDATE(_read_to_cache_time, _cache_statistics.read_to_cache_time); + COUNTER_UPDATE(_cache_refresh_count, _cache_statistics.cache_refresh_count); + COUNTER_UPDATE(_read_to_cache_bytes, _cache_statistics.read_to_cache_bytes); + if (_inner_reader != nullptr) { + _inner_reader->collect_profile_before_close(); + } + } +} + } // namespace io } // namespace doris diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index 907ea11b216ac4..67e07665fbfd9f 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -53,6 +53,147 @@ struct PrefetchRange { : start_offset(start_offset), end_offset(end_offset) {} PrefetchRange() : start_offset(0), end_offset(0) {} + + bool operator==(const PrefetchRange& other) const { + return (start_offset == other.start_offset) && (end_offset == other.end_offset); + } + + bool operator!=(const PrefetchRange& other) const { return !(*this == other); } + + PrefetchRange span(const PrefetchRange& other) const { + return {std::min(start_offset, other.end_offset), std::max(start_offset, other.end_offset)}; + } + PrefetchRange seq_span(const PrefetchRange& other) const { + return {start_offset, other.end_offset}; + } + + //Ranges needs to be sorted. + static std::vector merge_adjacent_seq_ranges( + const std::vector& seq_ranges, int64_t max_merge_distance_bytes, + int64_t once_max_read_bytes) { + if (seq_ranges.empty()) { + return {}; + } + // Merge overlapping ranges + std::vector result; + PrefetchRange last = seq_ranges.front(); + for (size_t i = 1; i < seq_ranges.size(); ++i) { + PrefetchRange current = seq_ranges[i]; + PrefetchRange merged = last.seq_span(current); + if (merged.end_offset <= once_max_read_bytes + merged.start_offset && + last.end_offset + max_merge_distance_bytes >= current.start_offset) { + last = merged; + } else { + result.push_back(last); + last = current; + } + } + result.push_back(last); + return result; + } +}; + +class RangeFinder { +public: + virtual ~RangeFinder() = default; + virtual Status get_range_for(int64_t desired_offset, io::PrefetchRange& result_range) = 0; + virtual size_t get_max_range_size() const = 0; +}; + +class LinearProbeRangeFinder : public RangeFinder { +public: + LinearProbeRangeFinder(std::vector&& ranges) : _ranges(std::move(ranges)) {} + + Status get_range_for(int64_t desired_offset, io::PrefetchRange& result_range) override; + + size_t get_max_range_size() const override { + size_t max_range_size = 0; + for (const auto& range : _ranges) { + max_range_size = std::max(max_range_size, range.end_offset - range.start_offset); + } + return max_range_size; + } + + ~LinearProbeRangeFinder() override = default; + +private: + std::vector _ranges; + size_t index {0}; +}; + +/** + * The reader provides a solution to read one range at a time. You can customize RangeFinder to meet your scenario. + * For me, since there will be tiny stripes when reading orc files, in order to reduce the requests to hdfs, + * I first merge the access to the orc files to be read (of course there is a problem of read amplification, + * but in my scenario, compared with reading hdfs multiple times, it is faster to read more data on hdfs at one time), + * and then because the actual reading of orc files is in order from front to back, I provide LinearProbeRangeFinder. + */ +class RangeCacheFileReader : public io::FileReader { + struct RangeCacheReaderStatistics { + int64_t request_io = 0; + int64_t request_bytes = 0; + int64_t request_time = 0; + int64_t read_to_cache_time = 0; + int64_t cache_refresh_count = 0; + int64_t read_to_cache_bytes = 0; + }; + +public: + RangeCacheFileReader(RuntimeProfile* profile, io::FileReaderSPtr inner_reader, + std::shared_ptr range_finder); + + ~RangeCacheFileReader() override = default; + + Status close() override { + if (!_closed) { + _closed = true; + } + return Status::OK(); + } + + const io::Path& path() const override { return _inner_reader->path(); } + + size_t size() const override { return _size; } + + bool closed() const override { return _closed; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + + void _collect_profile_before_close() override; + +private: + RuntimeProfile* _profile = nullptr; + io::FileReaderSPtr _inner_reader; + std::shared_ptr _range_finder; + + OwnedSlice _cache; + int64_t _current_start_offset = -1; + + size_t _size; + bool _closed = false; + + RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _request_bytes = nullptr; + RuntimeProfile::Counter* _request_time = nullptr; + RuntimeProfile::Counter* _read_to_cache_time = nullptr; + RuntimeProfile::Counter* _cache_refresh_count = nullptr; + RuntimeProfile::Counter* _read_to_cache_bytes = nullptr; + RangeCacheReaderStatistics _cache_statistics; + /** + * `RangeCacheFileReader`: + * 1. `CacheRefreshCount`: how many IOs are merged + * 2. `ReadToCacheBytes`: how much data is actually read after merging + * 3. `ReadToCacheTime`: how long it takes to read data after merging + * 4. `RequestBytes`: how many bytes does the apache-orc library actually need to read the orc file + * 5. `RequestIO`: how many times the apache-orc library calls this read interface + * 6. `RequestTime`: how long it takes the apache-orc library to call this read interface + * + * It should be noted that `RangeCacheFileReader` is a wrapper of the reader that actually reads data,such as + * the hdfs reader, so strictly speaking, `CacheRefreshCount` is not equal to how many IOs are initiated to hdfs, + * because each time the hdfs reader is requested, the hdfs reader may not be able to read all the data at once. + */ }; /** diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 3a5fffb2549938..d841c79ed66069 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -86,7 +86,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { S3ClientConf reset_conf; { std::shared_lock lock(_mtx); - if (conf.ak == _conf.ak && conf.sk == _conf.sk && conf.token == _conf.token) { + if (conf.get_hash() == _conf.get_hash()) { return Status::OK(); // Same conf } @@ -95,6 +95,10 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.sk = conf.sk; reset_conf.token = conf.token; reset_conf.bucket = conf.bucket; + reset_conf.connect_timeout_ms = conf.connect_timeout_ms; + reset_conf.max_connections = conf.max_connections; + reset_conf.request_timeout_ms = conf.request_timeout_ms; + reset_conf.use_virtual_addressing = conf.use_virtual_addressing; // Should check endpoint here? } diff --git a/be/src/io/hdfs_util.cpp b/be/src/io/hdfs_util.cpp index 62546c9bbd4ffb..92d8933d8b5c92 100644 --- a/be/src/io/hdfs_util.cpp +++ b/be/src/io/hdfs_util.cpp @@ -67,7 +67,8 @@ Status create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name if (t.joinable()) t.join(); bthread::butex_destroy(btx); }); - timespec tmout {.tv_sec = std::chrono::system_clock::now().time_since_epoch().count() + 60}; + timespec tmout {.tv_sec = std::chrono::system_clock::now().time_since_epoch().count() + 60, + .tv_nsec = 0}; if (int ret = bthread::butex_wait(btx, 1, &tmout); ret != 0) { std::string msg = "failed to wait _create_hdfs_fs fs_name=" + fs_name; LOG(WARNING) << msg << " error=" << std::strerror(errno); diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 1e819de7c554d6..e5ec38738155e5 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -468,13 +468,9 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest DCHECK_EQ(segments_key_bounds.size(), num_segments); std::vector picked_segments; for (int i = num_segments - 1; i >= 0; i--) { - // If mow table has cluster keys, the key bounds is short keys, not primary keys - // use PrimaryKeyIndexMetaPB in primary key index? - if (schema->cluster_key_idxes().empty()) { - if (key_without_seq.compare(segments_key_bounds[i].max_key()) > 0 || - key_without_seq.compare(segments_key_bounds[i].min_key()) < 0) { - continue; - } + if (key_without_seq.compare(segments_key_bounds[i].max_key()) > 0 || + key_without_seq.compare(segments_key_bounds[i].min_key()) < 0) { + continue; } picked_segments.emplace_back(i); } diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index ab40e4abde6d19..85e0a94c87413d 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -644,7 +644,7 @@ Status Compaction::do_inverted_index_compaction() { Status status = Status::OK(); for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); - const auto* index_meta = _cur_tablet_schema->get_inverted_index(col); + const auto* index_meta = _cur_tablet_schema->inverted_index(col); std::vector dest_index_dirs(dest_segment_num); try { @@ -682,15 +682,11 @@ Status Compaction::do_inverted_index_compaction() { } void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { - for (const auto& index : _cur_tablet_schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - - auto col_unique_ids = index.col_unique_ids(); + for (const auto& index : _cur_tablet_schema->inverted_indexes()) { + auto col_unique_ids = index->col_unique_ids(); // check if column unique ids is empty to avoid crash if (col_unique_ids.empty()) { - LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" << index.index_id() + LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] index[" << index->index_id() << "] has no column unique id, will skip index compaction." << " tablet_schema=" << _cur_tablet_schema->dump_full_schema(); continue; @@ -705,10 +701,9 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { bool is_continue = false; std::optional> first_properties; for (const auto& rowset : _input_rowsets) { - const auto* tablet_index = - rowset->tablet_schema()->get_inverted_index(col_unique_id, ""); + const auto* tablet_index = rowset->tablet_schema()->inverted_index(col_unique_id); // no inverted index or index id is different from current index id - if (tablet_index == nullptr || tablet_index->index_id() != index.index_id()) { + if (tablet_index == nullptr || tablet_index->index_id() != index->index_id()) { is_continue = true; break; } @@ -741,7 +736,7 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { return false; } - const auto* index_meta = rowset->tablet_schema()->get_inverted_index(col_unique_id, ""); + const auto* index_meta = rowset->tablet_schema()->inverted_index(col_unique_id); if (index_meta == nullptr) { LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" << col_unique_id << "] index meta is null, will skip index compaction"; @@ -826,6 +821,7 @@ Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) ctx.tablet_schema = _cur_tablet_schema; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; + ctx.storage_page_size = _tablet->tablet_meta()->storage_page_size(); _output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(ctx, _is_vertical)); _pending_rs_guard = _engine.add_pending_rowset(ctx); return Status::OK(); @@ -1128,6 +1124,7 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx.tablet_schema = _cur_tablet_schema; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; + ctx.storage_page_size = _tablet->tablet_meta()->storage_page_size(); auto compaction_policy = _tablet->tablet_meta()->compaction_policy(); if (_tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2) { diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index e0e3a5281bcb95..88277775f96101 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -254,7 +254,7 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const PNodeInfo& node_info) auto tablet_schema = cur_rowset->rowset_meta()->tablet_schema(); if (!tablet_schema->skip_write_index_on_load()) { for (auto& column : tablet_schema->columns()) { - const TabletIndex* index_meta = tablet_schema->get_inverted_index(*column); + const TabletIndex* index_meta = tablet_schema->inverted_index(*column); if (index_meta) { indices_ids.emplace_back(index_meta->index_id(), index_meta->get_index_suffix()); } diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp index a6fb0154489042..b770d3c07906b3 100644 --- a/be/src/olap/delta_writer_v2.cpp +++ b/be/src/olap/delta_writer_v2.cpp @@ -125,6 +125,9 @@ Status DeltaWriterV2::init() { context.partial_update_info = _partial_update_info; context.memtable_on_sink_support_index_v2 = true; + auto tablet = DORIS_TRY(ExecEnv::GetInstance()->storage_engine().get_tablet(_req.tablet_id)); + context.storage_page_size = tablet->tablet_meta()->storage_page_size(); + _rowset_writer = std::make_shared(_streams); RETURN_IF_ERROR(_rowset_writer->init(context)); std::shared_ptr wg_sptr = nullptr; diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 4b51dcc3530476..bbb2ca72b4ae7f 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -81,12 +81,7 @@ Status BetaRowset::get_inverted_index_size(size_t* index_size) { } if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - auto indices = _schema->indexes(); - for (auto& index : indices) { - // only get file_size for inverted index - if (index.index_type() != IndexType::INVERTED) { - continue; - } + for (const auto& index : _schema->inverted_indexes()) { for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { auto seg_path = DORIS_TRY(segment_path(seg_id)); int64_t file_size = 0; @@ -94,7 +89,7 @@ Status BetaRowset::get_inverted_index_size(size_t* index_size) { std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), - index.index_id(), index.get_index_suffix()); + index->index_id(), index->get_index_suffix()); RETURN_IF_ERROR(fs->file_size(inverted_index_file_path, &file_size)); *index_size += file_size; } @@ -122,7 +117,7 @@ void BetaRowset::clear_inverted_index_cache() { auto index_path_prefix = InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path); for (const auto& column : tablet_schema()->columns()) { - const TabletIndex* index_meta = tablet_schema()->get_inverted_index(*column); + const TabletIndex* index_meta = tablet_schema()->inverted_index(*column); if (index_meta) { auto inverted_index_file_cache_key = InvertedIndexDescriptor::get_index_file_cache_key( @@ -227,7 +222,7 @@ Status BetaRowset::remove() { if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { for (auto& column : _schema->columns()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); + const TabletIndex* index_meta = _schema->inverted_index(*column); if (index_meta) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( @@ -311,22 +306,19 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, return status; }); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : _schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - auto index_id = index.index_id(); + for (const auto& index : _schema->inverted_indexes()) { + auto index_id = index->index_id(); if (without_index_uids != nullptr && without_index_uids->count(index_id)) { continue; } std::string inverted_index_src_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(src_path), - index_id, index.get_index_suffix()); + index_id, index->get_index_suffix()); std::string inverted_index_dst_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(dst_path), - index_id, index.get_index_suffix()); + index_id, index->get_index_suffix()); bool index_file_exists = true; RETURN_IF_ERROR(local_fs->exists(inverted_index_src_file_path, &index_file_exists)); if (index_file_exists) { @@ -405,7 +397,7 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { for (auto& column : _schema->columns()) { // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); + const TabletIndex* index_meta = _schema->inverted_index(*column); if (index_meta) { std::string inverted_index_src_file_path = InvertedIndexDescriptor::get_index_file_path_v1( @@ -464,7 +456,7 @@ Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { for (auto& column : _schema->columns()) { // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); + const TabletIndex* index_meta = _schema->inverted_index(*column); if (index_meta) { std::string remote_inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( @@ -613,14 +605,11 @@ Status BetaRowset::add_to_binlog() { linked_success_files.push_back(binlog_file); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : _schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - auto index_id = index.index_id(); + for (const auto& index : _schema->inverted_indexes()) { + auto index_id = index->index_id(); auto index_file = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(seg_file), index_id, - index.get_index_suffix()); + index->get_index_suffix()); auto binlog_index_file = (std::filesystem::path(binlog_dir) / std::filesystem::path(index_file).filename()) .string(); @@ -661,7 +650,7 @@ Status BetaRowset::calc_file_crc(uint32_t* crc_value, int64_t* file_count) { file_paths.emplace_back(seg_path); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { for (auto& column : _schema->columns()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); + const TabletIndex* index_meta = _schema->inverted_index(*column); if (index_meta) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( @@ -805,7 +794,7 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, } else { rapidjson::Value indices(rapidjson::kArrayType); for (auto column : _rowset_meta->tablet_schema()->columns()) { - const auto* index_meta = _rowset_meta->tablet_schema()->get_inverted_index(*column); + const auto* index_meta = _rowset_meta->tablet_schema()->inverted_index(*column); if (index_meta == nullptr) { continue; } diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 037fae316e9157..198b4e8595ed20 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -547,8 +547,8 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u } // rename remaining inverted index files for (auto column : _context.tablet_schema->columns()) { - if (_context.tablet_schema->has_inverted_index(*column)) { - const auto* index_info = _context.tablet_schema->get_inverted_index(*column); + if (const auto& index_info = _context.tablet_schema->inverted_index(*column); + index_info != nullptr) { auto index_id = index_info->index_id(); if (_context.tablet_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index cb0fda83e60777..df85c11cd356fe 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -108,6 +108,7 @@ struct RowsetWriterContext { std::shared_ptr schema_lock; int64_t compaction_level = 0; + int64_t storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE; // For local rowset std::string tablet_path; diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index f901c786062c95..330174952b622c 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -165,8 +165,7 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e } // Delete inverted index files for (auto&& column : schema->columns()) { - if (schema->has_inverted_index(*column)) { - const auto* index_info = schema->get_inverted_index(*column); + if (const auto* index_info = schema->inverted_index(*column); index_info != nullptr) { auto index_id = index_info->index_id(); if (schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index 62f209db5ad4a5..2d66b940a3893b 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -63,7 +63,7 @@ struct ColumnWriterOptions { bool need_inverted_index = false; uint8_t gram_size; uint16_t gram_bf_size; - std::vector indexes; + std::vector indexes; // unused const TabletIndex* inverted_index = nullptr; InvertedIndexFileWriter* inverted_index_file_writer; std::string to_string() const { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index 8729bd0c590276..50874d0db5c72b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -75,6 +75,23 @@ const int32_t MAX_LEAF_COUNT = 1024; const float MAXMBSortInHeap = 512.0 * 8; const int DIMS = 1; +bool InvertedIndexColumnWriter::check_support_inverted_index(const TabletColumn& column) { + // bellow types are not supported in inverted index for extracted columns + static std::set invalid_types = { + FieldType::OLAP_FIELD_TYPE_DOUBLE, + FieldType::OLAP_FIELD_TYPE_JSONB, + FieldType::OLAP_FIELD_TYPE_ARRAY, + FieldType::OLAP_FIELD_TYPE_FLOAT, + }; + if (column.is_extracted_column() && (invalid_types.contains(column.type()))) { + return false; + } + if (column.is_variant_type()) { + return false; + } + return true; +} + template class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { public: diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_writer.h index 63c1e219e649e8..da90752db09168 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h @@ -33,7 +33,6 @@ #include "io/fs/local_file_system.h" #include "olap/olap_common.h" #include "olap/options.h" -#include "olap/tablet_schema.h" namespace doris { class CollectionValue; @@ -41,6 +40,7 @@ class CollectionValue; class Field; class TabletIndex; +class TabletColumn; namespace segment_v2 { class InvertedIndexFileWriter; @@ -74,22 +74,7 @@ class InvertedIndexColumnWriter { // check if the column is valid for inverted index, some columns // are generated from variant, but not all of them are supported - static bool check_support_inverted_index(const TabletColumn& column) { - // bellow types are not supported in inverted index for extracted columns - static std::set invalid_types = { - FieldType::OLAP_FIELD_TYPE_DOUBLE, - FieldType::OLAP_FIELD_TYPE_JSONB, - FieldType::OLAP_FIELD_TYPE_ARRAY, - FieldType::OLAP_FIELD_TYPE_FLOAT, - }; - if (column.is_extracted_column() && (invalid_types.contains(column.type()))) { - return false; - } - if (column.is_variant_type()) { - return false; - } - return true; - } + static bool check_support_inverted_index(const TabletColumn& column); private: DISALLOW_COPY_AND_ASSIGN(InvertedIndexColumnWriter); diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 93ec03df452b6c..33d1a24ece3947 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -25,6 +25,7 @@ namespace segment_v2 { static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store page size: 16KB +static constexpr size_t STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536; struct PageBuilderOptions { size_t data_page_size = DEFAULT_PAGE_SIZE; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 8f10eae03aeba1..1426792139c28b 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1057,16 +1057,17 @@ Status SegmentIterator::_init_inverted_index_iterators() { return Status::OK(); } for (auto cid : _schema->column_ids()) { + // Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase. if (_inverted_index_iterators[cid] == nullptr) { - // Not check type valid, since we need to get inverted index for related variant type when reading the segment. - // If check type valid, we can not get inverted index for variant type, and result nullptr.The result for calling - // get_inverted_index with variant suffix should return corresponding inverted index meta. - bool check_inverted_index_by_type = false; - // Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase. + // In the _opts.tablet_schema, the sub-column type information for the variant is FieldType::OLAP_FIELD_TYPE_VARIANT. + // This is because the sub-column is created in create_materialized_variant_column. + // We use this column to locate the metadata for the inverted index, which requires a unique_id and path. + const auto& column = _opts.tablet_schema->column(cid); + int32_t col_unique_id = + column.is_extracted_column() ? column.parent_unique_id() : column.unique_id(); RETURN_IF_ERROR(_segment->new_inverted_index_iterator( - _opts.tablet_schema->column(cid), - _segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid), - check_inverted_index_by_type), + column, + _segment->_tablet_schema->inverted_index(col_unique_id, column.suffix_path()), _opts, &_inverted_index_iterators[cid])); } } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 7f6c06fb057902..db95d3b62e73d9 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -207,22 +207,21 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (_opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load()) { skip_inverted_index = true; } - // indexes for this column - opts.indexes = schema->get_indexes_for_column(column); + if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { opts.need_zone_map = false; opts.need_bloom_filter = false; opts.need_bitmap_index = false; } - opts.inverted_index_file_writer = _inverted_index_file_writer; - for (const auto* index : opts.indexes) { - if (!skip_inverted_index && index->index_type() == IndexType::INVERTED) { - opts.inverted_index = index; - opts.need_inverted_index = true; - DCHECK(_inverted_index_file_writer != nullptr); - // TODO support multiple inverted index - break; - } + + // indexes for this column + if (const auto& index = schema->inverted_index(column); + index != nullptr && !skip_inverted_index) { + opts.inverted_index = index; + opts.need_inverted_index = true; + DCHECK(_inverted_index_file_writer != nullptr); + opts.inverted_index_file_writer = _inverted_index_file_writer; + // TODO support multiple inverted index } #define CHECK_FIELD_TYPE(TYPE, type_name) \ if (column.type() == FieldType::OLAP_FIELD_TYPE_##TYPE) { \ @@ -246,6 +245,35 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co #undef CHECK_FIELD_TYPE + if (_opts.rowset_ctx != nullptr) { + int64_t storage_page_size = _opts.rowset_ctx->storage_page_size; + // storage_page_size must be between 4KB and 10MB. + if (storage_page_size >= 4096 && storage_page_size <= 10485760) { + opts.data_page_size = storage_page_size; + } + } + DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size", { + auto table_id = DebugPoints::instance()->get_debug_param_or_default( + "VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id", + INT_MIN); + auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default( + "VerticalSegmentWriter._create_column_writer.storage_page_size", + "storage_page_size", INT_MIN); + if (table_id == INT_MIN || target_data_page_size == INT_MIN) { + return Status::Error( + "Debug point parameters missing: either 'table_id' or 'storage_page_size' not " + "set."); + } + if (table_id == _tablet_schema->table_id() && + opts.data_page_size != target_data_page_size) { + return Status::Error( + "Mismatch in 'storage_page_size': expected size does not match the current " + "data page size. " + "Expected: " + + std::to_string(target_data_page_size) + + ", Actual: " + std::to_string(opts.data_page_size) + "."); + } + }) if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 805d9b65d19c53..0ac9f349769cd4 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -201,23 +201,20 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo tablet_schema->skip_write_index_on_load()) { skip_inverted_index = true; } - // indexes for this column - opts.indexes = tablet_schema->get_indexes_for_column(column); + if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { opts.need_zone_map = false; opts.need_bloom_filter = false; opts.need_bitmap_index = false; } - for (const auto* index : opts.indexes) { - if (!skip_inverted_index && index->index_type() == IndexType::INVERTED) { - opts.inverted_index = index; - opts.need_inverted_index = true; - DCHECK(_inverted_index_file_writer != nullptr); - // TODO support multiple inverted index - break; - } + if (const auto& index = tablet_schema->inverted_index(column); + index != nullptr && !skip_inverted_index) { + opts.inverted_index = index; + opts.need_inverted_index = true; + DCHECK(_inverted_index_file_writer != nullptr); + opts.inverted_index_file_writer = _inverted_index_file_writer; + // TODO support multiple inverted index } - opts.inverted_index_file_writer = _inverted_index_file_writer; #define CHECK_FIELD_TYPE(TYPE, type_name) \ if (column.type() == FieldType::OLAP_FIELD_TYPE_##TYPE) { \ @@ -241,6 +238,35 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo #undef CHECK_FIELD_TYPE + if (_opts.rowset_ctx != nullptr) { + int64_t storage_page_size = _opts.rowset_ctx->storage_page_size; + // storage_page_size must be between 4KB and 10MB. + if (storage_page_size >= 4096 && storage_page_size <= 10485760) { + opts.data_page_size = storage_page_size; + } + } + DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size", { + auto table_id = DebugPoints::instance()->get_debug_param_or_default( + "VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id", + INT_MIN); + auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default( + "VerticalSegmentWriter._create_column_writer.storage_page_size", + "storage_page_size", INT_MIN); + if (table_id == INT_MIN || target_data_page_size == INT_MIN) { + return Status::Error( + "Debug point parameters missing: either 'table_id' or 'storage_page_size' not " + "set."); + } + if (table_id == _tablet_schema->table_id() && + opts.data_page_size != target_data_page_size) { + return Status::Error( + "Mismatch in 'storage_page_size': expected size does not match the current " + "data page size. " + "Expected: " + + std::to_string(target_data_page_size) + + ", Actual: " + std::to_string(opts.data_page_size) + "."); + } + }) if (column.is_row_store_column()) { // smaller page size for row store column auto page_size = _tablet_schema->row_store_page_size(); diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 112986d31d9bb8..93b1051246179e 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -232,6 +232,7 @@ Status RowsetBuilder::init() { context.mow_context = mow_context; context.write_file_cache = _req.write_file_cache; context.partial_update_info = _partial_update_info; + context.storage_page_size = _tablet->tablet_meta()->storage_page_size(); _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false)); _pending_rs_guard = _engine.pending_local_rowsets().add(context.rowset_id); diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 3b77ac19e2e5d2..5ef85dbaf11c19 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -81,6 +81,7 @@ #include "vec/columns/column.h" #include "vec/columns/column_nullable.h" #include "vec/common/assert_cast.h" +#include "vec/common/schema_util.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" #include "vec/exprs/vexpr.h" @@ -1367,13 +1368,9 @@ Status SchemaChangeJob::parse_request(const SchemaChangeParams& sc_params, *sc_directly = true; return Status::OK(); } else if (column_mapping->ref_column_idx >= 0) { - const auto& column_new = new_tablet_schema->column(i); - const auto& column_old = base_tablet_schema->column(column_mapping->ref_column_idx); // index changed - if (column_new.is_bf_column() != column_old.is_bf_column() || - column_new.has_bitmap_index() != column_old.has_bitmap_index() || - new_tablet_schema->has_inverted_index(column_new) != - base_tablet_schema->has_inverted_index(column_old)) { + if (vectorized::schema_util::has_schema_index_diff( + new_tablet_schema, base_tablet_schema, i, column_mapping->ref_column_idx)) { *sc_directly = true; return Status::OK(); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 2cfa9a8e8b763d..67205835b53947 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -698,11 +698,8 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (tablet_schema.get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : tablet_schema.indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - auto index_id = index.index_id(); + for (const auto& index : tablet_schema.inverted_indexes()) { + auto index_id = index->index_id(); auto index_file = ref_tablet->get_segment_index_filepath( rowset_id, segment_index, index_id); auto snapshot_segment_index_file_path = diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index b1d4c9dfb891f6..0bcec58b4aabb0 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1272,7 +1272,7 @@ std::vector Tablet::pick_candidate_rowsets_to_build_inverted_in std::shared_lock rlock(_meta_lock); auto has_alter_inverted_index = [&](RowsetSharedPtr rowset) -> bool { for (const auto& index_id : alter_index_uids) { - if (rowset->tablet_schema()->has_inverted_index_with_index_id(index_id, "")) { + if (rowset->tablet_schema()->has_inverted_index_with_index_id(index_id)) { return true; } } @@ -2650,12 +2650,9 @@ void Tablet::gc_binlogs(int64_t version) { // add binlog segment files and index files for (int64_t i = 0; i < num_segments; ++i) { wait_for_deleted_binlog_files.emplace_back(get_segment_filepath(rowset_id, i)); - for (const auto& index : this->tablet_schema()->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } + for (const auto& index : this->tablet_schema()->inverted_indexes()) { wait_for_deleted_binlog_files.emplace_back( - get_segment_index_filepath(rowset_id, i, index.index_id())); + get_segment_index_filepath(rowset_id, i, index->index_id())); } } }; @@ -2804,12 +2801,8 @@ int64_t Tablet::get_inverted_index_file_szie(const RowsetMetaSharedPtr& rs_meta) if (rs_meta->tablet_schema()->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - auto indices = rs_meta->tablet_schema()->indexes(); + const auto& indices = rs_meta->tablet_schema()->inverted_indexes(); for (auto& index : indices) { - // only get file_size for inverted index - if (index.index_type() != IndexType::INVERTED) { - continue; - } for (int seg_id = 0; seg_id < rs_meta->num_segments(); ++seg_id) { std::string segment_path = get_segment_path(rs_meta, seg_id); int64_t file_size = 0; @@ -2817,7 +2810,7 @@ int64_t Tablet::get_inverted_index_file_szie(const RowsetMetaSharedPtr& rs_meta) std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(segment_path), - index.index_id(), index.get_index_suffix()); + index->index_id(), index->get_index_suffix()); auto st = fs->file_size(inverted_index_file_path, &file_size); if (!st.ok()) { file_size = 0; diff --git a/be/src/olap/tablet_column_object_pool.cpp b/be/src/olap/tablet_column_object_pool.cpp new file mode 100644 index 00000000000000..6e07fb4e831e60 --- /dev/null +++ b/be/src/olap/tablet_column_object_pool.cpp @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/tablet_column_object_pool.h" + +#include +#include + +#include "olap/tablet_schema.h" + +namespace doris { + +bvar::Adder g_tablet_column_cache_count("tablet_column_cache_count"); +bvar::Adder g_tablet_column_cache_hit_count("tablet_column_cache_hit_count"); + +std::pair TabletColumnObjectPool::insert(const std::string& key) { + auto* lru_handle = lookup(key); + TabletColumnPtr tablet_column_ptr; + if (lru_handle) { + auto* value = (CacheValue*)LRUCachePolicy::value(lru_handle); + tablet_column_ptr = value->tablet_column; + VLOG_DEBUG << "reuse column "; + g_tablet_column_cache_hit_count << 1; + } else { + auto* value = new CacheValue; + tablet_column_ptr = std::make_shared(); + ColumnPB pb; + pb.ParseFromString(key); + tablet_column_ptr->init_from_pb(pb); + VLOG_DEBUG << "create column "; + value->tablet_column = tablet_column_ptr; + lru_handle = LRUCachePolicy::insert(key, value, 1, 0, CachePriority::NORMAL); + g_tablet_column_cache_count << 1; + } + DCHECK(lru_handle != nullptr); + return {lru_handle, tablet_column_ptr}; +} + +TabletColumnObjectPool::CacheValue::~CacheValue() { + g_tablet_column_cache_count << -1; +} + +} // namespace doris diff --git a/be/src/olap/tablet_column_object_pool.h b/be/src/olap/tablet_column_object_pool.h new file mode 100644 index 00000000000000..1eead6a25c9609 --- /dev/null +++ b/be/src/olap/tablet_column_object_pool.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "olap/tablet_fwd.h" +#include "olap/tablet_schema.h" +#include "runtime/exec_env.h" +#include "runtime/memory/lru_cache_policy.h" + +namespace doris { + +// TabletColumnObjectPool is a cache for TabletColumn objects. It is used to reduce memory consumption +// when there are a large number of identical TabletColumns in the cluster, which usually occurs +// when VARIANT type columns are modified and added, each Rowset has an individual TabletSchema. +// Excessive TabletSchemas can lead to significant memory overhead. Reusing memory for identical +// TabletColumns would greatly reduce this memory consumption. + +class TabletColumnObjectPool : public LRUCachePolicy { +public: + TabletColumnObjectPool(size_t capacity) + : LRUCachePolicy(CachePolicy::CacheType::TABLET_COLUMN_OBJECT_POOL, capacity, + LRUCacheType::NUMBER, config::tablet_schema_cache_recycle_interval) {} + + static TabletColumnObjectPool* create_global_column_cache(size_t capacity) { + auto* res = new TabletColumnObjectPool(capacity); + return res; + } + + static TabletColumnObjectPool* instance() { + return ExecEnv::GetInstance()->get_tablet_column_object_pool(); + } + + std::pair insert(const std::string& key); + +private: + class CacheValue : public LRUCacheValueBase { + public: + ~CacheValue() override; + TabletColumnPtr tablet_column; + }; +}; + +} // namespace doris diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 91f3b7dd8169bf..d153069babdfcc 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -97,7 +97,8 @@ TabletMetaSharedPtr TabletMeta::create( request.time_series_compaction_file_count_threshold, request.time_series_compaction_time_threshold_seconds, request.time_series_compaction_empty_rowsets_threshold, - request.time_series_compaction_level_threshold, inverted_index_file_storage_format); + request.time_series_compaction_level_threshold, inverted_index_file_storage_format, + request.storage_page_size); } TabletMeta::TabletMeta() @@ -118,7 +119,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id int64_t time_series_compaction_time_threshold_seconds, int64_t time_series_compaction_empty_rowsets_threshold, int64_t time_series_compaction_level_threshold, - TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format) + TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format, + int64_t storage_page_size) : _tablet_uid(0, 0), _schema(new TabletSchema), _delete_bitmap(new DeleteBitmap(tablet_id)) { @@ -150,6 +152,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id time_series_compaction_empty_rowsets_threshold); tablet_meta_pb.set_time_series_compaction_level_threshold( time_series_compaction_level_threshold); + tablet_meta_pb.set_storage_page_size(storage_page_size); TabletSchemaPB* schema = tablet_meta_pb.mutable_schema(); schema->set_num_short_key_columns(tablet_schema.short_key_column_count); schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block); @@ -377,7 +380,8 @@ TabletMeta::TabletMeta(const TabletMeta& b) b._time_series_compaction_time_threshold_seconds), _time_series_compaction_empty_rowsets_threshold( b._time_series_compaction_empty_rowsets_threshold), - _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold) {}; + _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold), + _storage_page_size(b._storage_page_size) {}; void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, ColumnPB* column) { @@ -685,6 +689,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { tablet_meta_pb.time_series_compaction_empty_rowsets_threshold(); _time_series_compaction_level_threshold = tablet_meta_pb.time_series_compaction_level_threshold(); + _storage_page_size = tablet_meta_pb.storage_page_size(); } void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { @@ -776,6 +781,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { time_series_compaction_empty_rowsets_threshold()); tablet_meta_pb->set_time_series_compaction_level_threshold( time_series_compaction_level_threshold()); + tablet_meta_pb->set_storage_page_size(storage_page_size()); } int64_t TabletMeta::mem_size() const { @@ -983,6 +989,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) { return false; if (a._time_series_compaction_level_threshold != b._time_series_compaction_level_threshold) return false; + if (a._storage_page_size != b._storage_page_size) return false; return true; } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index d56e529e42bf4b..0d9645e01901e5 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -114,7 +114,8 @@ class TabletMeta : public MetadataAdder { int64_t time_series_compaction_empty_rowsets_threshold = 5, int64_t time_series_compaction_level_threshold = 1, TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format = - TInvertedIndexFileStorageFormat::V2); + TInvertedIndexFileStorageFormat::V2, + int64_t storage_page_size = 65536); // If need add a filed in TableMeta, filed init copy in copy construct function TabletMeta(const TabletMeta& tablet_meta); TabletMeta(TabletMeta&& tablet_meta) = delete; @@ -293,6 +294,11 @@ class TabletMeta : public MetadataAdder { int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; } + void set_storage_page_size(int64_t storage_page_size) { + _storage_page_size = storage_page_size; + } + int64_t storage_page_size() const { return _storage_page_size; } + private: Status _save_meta(DataDir* data_dir); @@ -353,6 +359,8 @@ class TabletMeta : public MetadataAdder { // cloud int64_t _ttl_seconds = 0; + int64_t _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE; + mutable std::shared_mutex _meta_lock; }; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 4041afac78ee13..36610f909748f2 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -38,8 +38,10 @@ #include "exec/tablet_info.h" #include "olap/inverted_index_parser.h" #include "olap/olap_define.h" +#include "olap/tablet_column_object_pool.h" #include "olap/types.h" #include "olap/utils.h" +#include "runtime/memory/lru_cache_policy.h" #include "runtime/thread_context.h" #include "tablet_meta.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" @@ -853,7 +855,9 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const { TabletSchema::TabletSchema() = default; -TabletSchema::~TabletSchema() = default; +TabletSchema::~TabletSchema() { + clear_column_cache_handlers(); +} int64_t TabletSchema::get_metadata_size() const { return sizeof(TabletSchema) + _vl_field_mem_size; @@ -902,14 +906,13 @@ void TabletColumn::append_sparse_column(TabletColumn column) { _num_sparse_columns++; } -void TabletSchema::append_index(TabletIndex index) { +void TabletSchema::append_index(TabletIndex&& index) { _indexes.push_back(std::move(index)); } void TabletSchema::update_index(const TabletColumn& col, TabletIndex index) { int32_t col_unique_id = col.unique_id(); - const std::string& suffix_path = - col.has_path_info() ? escape_for_path_name(col.path_info_ptr()->get_path()) : ""; + const std::string& suffix_path = escape_for_path_name(col.suffix_path()); for (size_t i = 0; i < _indexes.size(); i++) { for (int32_t id : _indexes[i].col_unique_ids()) { if (id == col_unique_id && _indexes[i].get_index_suffix() == suffix_path) { @@ -948,9 +951,18 @@ void TabletSchema::clear_columns() { _num_null_columns = 0; _num_key_columns = 0; _cols.clear(); + clear_column_cache_handlers(); +} + +void TabletSchema::clear_column_cache_handlers() { + for (auto* cache_handle : _column_cache_handlers) { + TabletColumnObjectPool::instance()->release(cache_handle); + } + _column_cache_handlers.clear(); } -void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns) { +void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns, + bool reuse_cache_column) { _keys_type = schema.keys_type(); _num_columns = 0; _num_variant_columns = 0; @@ -961,25 +973,34 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _field_name_to_index.clear(); _field_id_to_index.clear(); _cluster_key_idxes.clear(); + clear_column_cache_handlers(); for (const auto& i : schema.cluster_key_idxes()) { _cluster_key_idxes.push_back(i); } for (auto& column_pb : schema.column()) { - TabletColumn column; - column.init_from_pb(column_pb); - if (ignore_extracted_columns && column.is_extracted_column()) { + TabletColumnPtr column; + if (reuse_cache_column) { + auto pair = TabletColumnObjectPool::instance()->insert( + deterministic_string_serialize(column_pb)); + column = pair.second; + _column_cache_handlers.push_back(pair.first); + } else { + column = std::make_shared(); + column->init_from_pb(column_pb); + } + if (ignore_extracted_columns && column->is_extracted_column()) { continue; } - if (column.is_key()) { + if (column->is_key()) { _num_key_columns++; } - if (column.is_nullable()) { + if (column->is_nullable()) { _num_null_columns++; } - if (column.is_variant_type()) { + if (column->is_variant_type()) { ++_num_variant_columns; } - _cols.emplace_back(std::make_shared(std::move(column))); + _cols.emplace_back(std::move(column)); _vl_field_mem_size += sizeof(StringRef) + sizeof(char) * _cols.back()->name().size() + sizeof(size_t); _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns); @@ -1098,6 +1119,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _version_col_idx = -1; _skip_bitmap_col_idx = -1; _cluster_key_idxes.clear(); + clear_column_cache_handlers(); for (const auto& i : ori_tablet_schema._cluster_key_idxes) { _cluster_key_idxes.push_back(i); } @@ -1337,28 +1359,6 @@ Result TabletSchema::column(const std::string& field_name) return _cols[it->second].get(); } -std::vector TabletSchema::get_indexes_for_column( - const TabletColumn& col) const { - std::vector indexes_for_column; - // Some columns (Float, Double, JSONB ...) from the variant do not support index, but they are listed in TabltetIndex. - if (!segment_v2::InvertedIndexColumnWriter::check_support_inverted_index(col)) { - return indexes_for_column; - } - int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - const std::string& suffix_path = - col.has_path_info() ? escape_for_path_name(col.path_info_ptr()->get_path()) : ""; - // TODO use more efficient impl - for (size_t i = 0; i < _indexes.size(); i++) { - for (int32_t id : _indexes[i].col_unique_ids()) { - if (id == col_unique_id && _indexes[i].get_index_suffix() == suffix_path) { - indexes_for_column.push_back(&(_indexes[i])); - } - } - } - - return indexes_for_column; -} - void TabletSchema::update_tablet_columns(const TabletSchema& tablet_schema, const std::vector& t_columns) { copy_from(tablet_schema); @@ -1370,49 +1370,17 @@ void TabletSchema::update_tablet_columns(const TabletSchema& tablet_schema, } } -bool TabletSchema::has_inverted_index(const TabletColumn& col) const { - // TODO use more efficient impl - int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - const std::string& suffix_path = - col.has_path_info() ? escape_for_path_name(col.path_info_ptr()->get_path()) : ""; - for (size_t i = 0; i < _indexes.size(); i++) { - if (_indexes[i].index_type() == IndexType::INVERTED) { - for (int32_t id : _indexes[i].col_unique_ids()) { - if (id == col_unique_id && _indexes[i].get_index_suffix() == suffix_path) { - return true; - } - } - } - } - - return false; -} - -bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id, - const std::string& suffix_name) const { +bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id) const { for (size_t i = 0; i < _indexes.size(); i++) { - if (_indexes[i].index_type() == IndexType::INVERTED && - _indexes[i].get_index_suffix() == suffix_name && _indexes[i].index_id() == index_id) { + if (_indexes[i].index_type() == IndexType::INVERTED && _indexes[i].index_id() == index_id) { return true; } } return false; } -const TabletIndex* TabletSchema::get_inverted_index_with_index_id( - int64_t index_id, const std::string& suffix_name) const { - for (size_t i = 0; i < _indexes.size(); i++) { - if (_indexes[i].index_type() == IndexType::INVERTED && - _indexes[i].get_index_suffix() == suffix_name && _indexes[i].index_id() == index_id) { - return &(_indexes[i]); - } - } - - return nullptr; -} - -const TabletIndex* TabletSchema::get_inverted_index(int32_t col_unique_id, - const std::string& suffix_path) const { +const TabletIndex* TabletSchema::inverted_index(int32_t col_unique_id, + const std::string& suffix_path) const { for (size_t i = 0; i < _indexes.size(); i++) { if (_indexes[i].index_type() == IndexType::INVERTED) { for (int32_t id : _indexes[i].col_unique_ids()) { @@ -1426,19 +1394,15 @@ const TabletIndex* TabletSchema::get_inverted_index(int32_t col_unique_id, return nullptr; } -const TabletIndex* TabletSchema::get_inverted_index(const TabletColumn& col, - bool check_valid) const { - // With check_valid set to true by default +const TabletIndex* TabletSchema::inverted_index(const TabletColumn& col) const { // Some columns(Float, Double, JSONB ...) from the variant do not support inverted index - if (check_valid && !segment_v2::InvertedIndexColumnWriter::check_support_inverted_index(col)) { + if (!segment_v2::InvertedIndexColumnWriter::check_support_inverted_index(col)) { return nullptr; } // TODO use more efficient impl // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - const std::string& suffix_path = - col.has_path_info() ? escape_for_path_name(col.path_info_ptr()->get_path()) : ""; - return get_inverted_index(col_unique_id, suffix_path); + return inverted_index(col_unique_id, escape_for_path_name(col.suffix_path())); } bool TabletSchema::has_ngram_bf_index(int32_t col_unique_id) const { @@ -1573,13 +1537,4 @@ bool operator!=(const TabletSchema& a, const TabletSchema& b) { return !(a == b); } -std::string TabletSchema::deterministic_string_serialize(const TabletSchemaPB& schema_pb) { - std::string output; - google::protobuf::io::StringOutputStream string_output_stream(&output); - google::protobuf::io::CodedOutputStream output_stream(&string_output_stream); - output_stream.SetSerializationDeterministic(true); - schema_pb.SerializeToCodedStream(&output_stream); - return output; -} - } // namespace doris diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ebe2c63c7f30d2..2ca75fad356ee1 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -40,6 +40,7 @@ #include "olap/rowset/segment_v2/options.h" #include "runtime/define_primitive_type.h" #include "runtime/descriptors.h" +#include "runtime/memory/lru_cache_policy.h" #include "util/string_util.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/common/string_ref.h" @@ -164,6 +165,9 @@ class TabletColumn : public MetadataAdder { bool is_extracted_column() const { return _column_path != nullptr && !_column_path->empty() && _parent_col_unique_id > 0; }; + std::string suffix_path() const { + return is_extracted_column() ? _column_path->get_path() : ""; + } bool is_nested_subcolumn() const { return _column_path != nullptr && _column_path->has_nested_part(); } @@ -224,13 +228,16 @@ class TabletColumn : public MetadataAdder { bool _has_bitmap_index = false; bool _visible = true; - int32_t _parent_col_unique_id = -1; + std::vector _sub_columns; uint32_t _sub_column_count = 0; bool _result_is_nullable = false; int _be_exec_version = -1; - vectorized::PathInDataPtr _column_path; + + // The extracted sub-columns from "variant" contain the following information: + int32_t _parent_col_unique_id = -1; // "variant" -> col_unique_id + vectorized::PathInDataPtr _column_path; // the path of the sub-columns themselves // Record information about columns merged into a sparse column within a variant // `{"id": 100, "name" : "jack", "point" : 3.9}` @@ -298,13 +305,25 @@ class TabletSchema : public MetadataAdder { TabletSchema(); virtual ~TabletSchema(); - void init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns = false); + // Init from pb + // ignore_extracted_columns: ignore the extracted columns from variant column + // reuse_cached_column: reuse the cached column in the schema if they are the same, to reduce memory usage + void init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns = false, + bool reuse_cached_column = false); // Notice: Use deterministic way to serialize protobuf, // since serialize Map in protobuf may could lead to un-deterministic by default - static std::string deterministic_string_serialize(const TabletSchemaPB& schema_pb); + template + static std::string deterministic_string_serialize(const PbType& pb) { + std::string output; + google::protobuf::io::StringOutputStream string_output_stream(&output); + google::protobuf::io::CodedOutputStream output_stream(&string_output_stream); + output_stream.SetSerializationDeterministic(true); + pb.SerializeToCodedStream(&output_stream); + return output; + } void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; void append_column(TabletColumn column, ColumnType col_type = ColumnType::NORMAL); - void append_index(TabletIndex index); + void append_index(TabletIndex&& index); void update_index(const TabletColumn& column, TabletIndex index); void remove_index(int64_t index_id); void clear_index(); @@ -376,7 +395,15 @@ class TabletSchema : public MetadataAdder { void set_row_store_page_size(long page_size) { _row_store_page_size = page_size; } long row_store_page_size() const { return _row_store_page_size; } - const std::vector& indexes() const { return _indexes; } + const std::vector inverted_indexes() const { + std::vector inverted_indexes; + for (const auto& index : _indexes) { + if (index.index_type() == IndexType::INVERTED) { + inverted_indexes.emplace_back(&index); + } + } + return inverted_indexes; + } bool has_inverted_index() const { for (const auto& index : _indexes) { if (index.index_type() == IndexType::INVERTED) { @@ -385,17 +412,15 @@ class TabletSchema : public MetadataAdder { } return false; } - std::vector get_indexes_for_column(const TabletColumn& col) const; - bool has_inverted_index(const TabletColumn& col) const; - bool has_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_path) const; - const TabletIndex* get_inverted_index_with_index_id(int64_t index_id, - const std::string& suffix_name) const; - // check_valid: check if this column supports inverted index + bool has_inverted_index_with_index_id(int64_t index_id) const; + // Check whether this column supports inverted index // Some columns (Float, Double, JSONB ...) from the variant do not support index, but they are listed in TabletIndex. - // If returned, the index file will not be found. - const TabletIndex* get_inverted_index(const TabletColumn& col, bool check_valid = true) const; - const TabletIndex* get_inverted_index(int32_t col_unique_id, - const std::string& suffix_path) const; + const TabletIndex* inverted_index(const TabletColumn& col) const; + + // Regardless of whether this column supports inverted index + // TabletIndex information will be returned as long as it exists. + const TabletIndex* inverted_index(int32_t col_unique_id, + const std::string& suffix_path = "") const; bool has_ngram_bf_index(int32_t col_unique_id) const; const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const; void update_indexes_from_thrift(const std::vector& indexes); @@ -507,10 +532,13 @@ class TabletSchema : public MetadataAdder { friend bool operator==(const TabletSchema& a, const TabletSchema& b); friend bool operator!=(const TabletSchema& a, const TabletSchema& b); + void clear_column_cache_handlers(); + KeysType _keys_type = DUP_KEYS; SortType _sort_type = SortType::LEXICAL; size_t _sort_col_num = 0; std::vector _cols; + std::vector _column_cache_handlers; std::vector _indexes; std::unordered_map _field_name_to_index; diff --git a/be/src/olap/tablet_schema_cache.cpp b/be/src/olap/tablet_schema_cache.cpp index e339c947bb97a4..fd238fa5affb3f 100644 --- a/be/src/olap/tablet_schema_cache.cpp +++ b/be/src/olap/tablet_schema_cache.cpp @@ -18,30 +18,45 @@ #include "olap/tablet_schema_cache.h" #include +#include +#include #include "bvar/bvar.h" #include "olap/tablet_schema.h" +#include "util/sha.h" bvar::Adder g_tablet_schema_cache_count("tablet_schema_cache_count"); bvar::Adder g_tablet_schema_cache_columns_count("tablet_schema_cache_columns_count"); +bvar::Adder g_tablet_schema_cache_hit_count("tablet_schema_cache_hit_count"); namespace doris { +// to reduce the memory consumption of the serialized TabletSchema as key. +// use sha256 to prevent from hash collision +static std::string get_key_signature(const std::string& origin) { + SHA256Digest digest; + digest.reset(origin.data(), origin.length()); + return std::string {digest.digest().data(), digest.digest().length()}; +} + std::pair TabletSchemaCache::insert(const std::string& key) { - auto* lru_handle = lookup(key); + std::string key_signature = get_key_signature(key); + auto* lru_handle = lookup(key_signature); TabletSchemaSPtr tablet_schema_ptr; if (lru_handle) { auto* value = (CacheValue*)LRUCachePolicy::value(lru_handle); tablet_schema_ptr = value->tablet_schema; + g_tablet_schema_cache_hit_count << 1; } else { auto* value = new CacheValue; tablet_schema_ptr = std::make_shared(); TabletSchemaPB pb; pb.ParseFromString(key); - tablet_schema_ptr->init_from_pb(pb); + // We should reuse the memory of the same TabletColumn object, set reuse_cached_column to true + tablet_schema_ptr->init_from_pb(pb, false, true); value->tablet_schema = tablet_schema_ptr; - lru_handle = LRUCachePolicy::insert(key, value, tablet_schema_ptr->num_columns(), 0, - CachePriority::NORMAL); + lru_handle = LRUCachePolicy::insert(key_signature, value, tablet_schema_ptr->num_columns(), + 0, CachePriority::NORMAL); g_tablet_schema_cache_count << 1; g_tablet_schema_cache_columns_count << tablet_schema_ptr->num_columns(); } diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 21be34a334dd8d..a300e6e0f09fa3 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -407,11 +407,8 @@ Status EngineStorageMigrationTask::_copy_index_and_data_files( if (tablet_schema.get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : tablet_schema.indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - auto index_id = index.index_id(); + for (const auto& index : tablet_schema.inverted_indexes()) { + auto index_id = index->index_id(); auto index_file = _tablet->get_segment_index_filepath(rowset_id, segment_index, index_id); auto snapshot_segment_index_file_path = diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 8f8c3f7ad8004e..5e44f049f4707d 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -94,13 +94,19 @@ Status IndexBuilder::update_inverted_index_info() { auto column_name = t_inverted_index.columns[0]; auto column_idx = output_rs_tablet_schema->field_index(column_name); if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx - << "]"; - continue; + if (!t_inverted_index.column_unique_ids.empty()) { + auto column_unique_id = t_inverted_index.column_unique_ids[0]; + column_idx = output_rs_tablet_schema->field_index(column_unique_id); + } + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name + << " referenced_column=" << column_idx << "]"; + continue; + } } auto column = output_rs_tablet_schema->column(column_idx); - const auto* index_meta = output_rs_tablet_schema->get_inverted_index(column); + const auto* index_meta = output_rs_tablet_schema->inverted_index(column); if (index_meta == nullptr) { LOG(ERROR) << "failed to find column: " << column_name << " index_id: " << t_inverted_index.index_id; @@ -136,12 +142,7 @@ Status IndexBuilder::update_inverted_index_info() { return Status::Error( "indexes count cannot be negative"); } - int32_t indexes_size = 0; - for (auto index : output_rs_tablet_schema->indexes()) { - if (index.index_type() == IndexType::INVERTED) { - indexes_size++; - } - } + int32_t indexes_size = output_rs_tablet_schema->inverted_indexes().size(); if (indexes_count != indexes_size) { return Status::Error( "indexes count not equal to expected"); @@ -159,11 +160,11 @@ Status IndexBuilder::update_inverted_index_info() { LOG(WARNING) << "referenced column was missing. " << "[column=" << t_inverted_index.columns[0] << " referenced_column=" << column_uid << "]"; - output_rs_tablet_schema->append_index(index); + output_rs_tablet_schema->append_index(std::move(index)); continue; } const TabletColumn& col = output_rs_tablet_schema->column_by_uid(column_uid); - const TabletIndex* exist_index = output_rs_tablet_schema->get_inverted_index(col); + const TabletIndex* exist_index = output_rs_tablet_schema->inverted_index(col); if (exist_index && exist_index->index_id() != index.index_id()) { LOG(WARNING) << fmt::format( "column: {} has a exist inverted index, but the index id not equal " @@ -173,7 +174,7 @@ Status IndexBuilder::update_inverted_index_info() { without_index_uids.insert(exist_index->index_id()); output_rs_tablet_schema->remove_index(exist_index->index_id()); } - output_rs_tablet_schema->append_index(index); + output_rs_tablet_schema->append_index(std::move(index)); } } // construct input rowset reader @@ -381,8 +382,10 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta auto column_name = inverted_index.columns[0]; auto column_idx = output_rowset_schema->field_index(column_name); if (column_idx < 0) { - column_idx = - output_rowset_schema->field_index(inverted_index.column_unique_ids[0]); + if (!inverted_index.column_unique_ids.empty()) { + column_idx = output_rowset_schema->field_index( + inverted_index.column_unique_ids[0]); + } if (column_idx < 0) { LOG(WARNING) << "referenced column was missing. " << "[column=" << column_name @@ -391,14 +394,15 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta } } auto column = output_rowset_schema->column(column_idx); + // variant column is not support for building index if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { continue; } - DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id, "")); + DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id)); _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); std::unique_ptr field(FieldFactory::create(column)); - const auto* index_meta = output_rowset_schema->get_inverted_index(column); + const auto* index_meta = output_rowset_schema->inverted_index(column); std::unique_ptr inverted_index_builder; try { RETURN_IF_ERROR(segment_v2::InvertedIndexColumnWriter::create( @@ -513,9 +517,16 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema, auto column_name = inverted_index.columns[0]; auto column_idx = tablet_schema->field_index(column_name); if (column_idx < 0) { - LOG(WARNING) << "referenced column was missing. " - << "[column=" << column_name << " referenced_column=" << column_idx << "]"; - continue; + if (!inverted_index.column_unique_ids.empty()) { + auto column_unique_id = inverted_index.column_unique_ids[0]; + column_idx = tablet_schema->field_index(column_unique_id); + } + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + continue; + } } auto column = tablet_schema->column(column_idx); auto writer_sign = std::make_pair(segment_idx, index_id); diff --git a/be/src/pipeline/exec/multi_cast_data_streamer.cpp b/be/src/pipeline/exec/multi_cast_data_streamer.cpp index 3e629093e23b97..25c939695f90ef 100644 --- a/be/src/pipeline/exec/multi_cast_data_streamer.cpp +++ b/be/src/pipeline/exec/multi_cast_data_streamer.cpp @@ -23,16 +23,14 @@ namespace doris::pipeline { #include "common/compile_check_begin.h" -MultiCastBlock::MultiCastBlock(vectorized::Block* block, int used_count, int un_finish_copy, - size_t mem_size) - : _used_count(used_count), _un_finish_copy(un_finish_copy), _mem_size(mem_size) { +MultiCastBlock::MultiCastBlock(vectorized::Block* block, int un_finish_copy, size_t mem_size) + : _un_finish_copy(un_finish_copy), _mem_size(mem_size) { _block = vectorized::Block::create_unique(block->get_columns_with_type_and_name()); block->clear(); } Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* block, bool* eos) { int* un_finish_copy = nullptr; - int use_count = 0; { std::lock_guard l(_mutex); auto& pos_to_pull = _sender_pos_to_read[sender_idx]; @@ -43,8 +41,6 @@ Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* blo _cumulative_mem_size -= pos_to_pull->_mem_size; - pos_to_pull->_used_count--; - use_count = pos_to_pull->_used_count; un_finish_copy = &pos_to_pull->_un_finish_copy; pos_to_pull++; @@ -56,12 +52,7 @@ Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* blo *eos = _eos and pos_to_pull == end; } - if (use_count == 0) { - // will clear _multi_cast_blocks - _wait_copy_block(block, *un_finish_copy); - } else { - _copy_block(block, *un_finish_copy); - } + _copy_block(block, *un_finish_copy); return Status::OK(); } @@ -71,21 +62,13 @@ void MultiCastDataStreamer::_copy_block(vectorized::Block* block, int& un_finish for (int i = 0; i < block->columns(); ++i) { block->get_by_position(i).column = block->get_by_position(i).column->clone_resized(rows); } - std::unique_lock l(_mutex); un_finish_copy--; if (un_finish_copy == 0) { - l.unlock(); - _cv.notify_one(); + _multi_cast_blocks.pop_front(); } } -void MultiCastDataStreamer::_wait_copy_block(vectorized::Block* block, int& un_finish_copy) { - std::unique_lock l(_mutex); - _cv.wait(l, [&]() { return un_finish_copy == 0; }); - _multi_cast_blocks.pop_front(); -} - Status MultiCastDataStreamer::push(RuntimeState* state, doris::vectorized::Block* block, bool eos) { auto rows = block->rows(); COUNTER_UPDATE(_process_rows, rows); @@ -96,8 +79,7 @@ Status MultiCastDataStreamer::push(RuntimeState* state, doris::vectorized::Block { std::lock_guard l(_mutex); - _multi_cast_blocks.emplace_back(block, _cast_sender_count, _cast_sender_count - 1, - block_mem_size); + _multi_cast_blocks.emplace_back(block, _cast_sender_count, block_mem_size); // last elem auto end = std::prev(_multi_cast_blocks.end()); for (int i = 0; i < _sender_pos_to_read.size(); ++i) { diff --git a/be/src/pipeline/exec/multi_cast_data_streamer.h b/be/src/pipeline/exec/multi_cast_data_streamer.h index 07e64016363f65..51a73cf0c2b053 100644 --- a/be/src/pipeline/exec/multi_cast_data_streamer.h +++ b/be/src/pipeline/exec/multi_cast_data_streamer.h @@ -23,10 +23,11 @@ namespace doris::pipeline { class Dependency; struct MultiCastBlock { - MultiCastBlock(vectorized::Block* block, int used_count, int need_copy, size_t mem_size); + MultiCastBlock(vectorized::Block* block, int need_copy, size_t mem_size); std::unique_ptr _block; - int _used_count; + // Each block is copied during pull. If _un_finish_copy == 0, + // it indicates that this block has been fully used and can be released. int _un_finish_copy; size_t _mem_size; }; @@ -69,14 +70,10 @@ class MultiCastDataStreamer { void _block_reading(int sender_idx); void _copy_block(vectorized::Block* block, int& un_finish_copy); - - void _wait_copy_block(vectorized::Block* block, int& un_finish_copy); - const RowDescriptor& _row_desc; RuntimeProfile* _profile = nullptr; std::list _multi_cast_blocks; std::vector::iterator> _sender_pos_to_read; - std::condition_variable _cv; std::mutex _mutex; bool _eos = false; int _cast_sender_count = 0; diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h index e1b81d8dae3b07..6053b1a2f48e87 100644 --- a/be/src/pipeline/exec/operator.h +++ b/be/src/pipeline/exec/operator.h @@ -836,9 +836,9 @@ class StatefulOperatorX : public OperatorX { template requires(std::is_base_of_v) -class AsyncWriterSink : public PipelineXSinkLocalState { +class AsyncWriterSink : public PipelineXSinkLocalState { public: - using Base = PipelineXSinkLocalState; + using Base = PipelineXSinkLocalState; AsyncWriterSink(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state), _async_writer_dependency(nullptr) { _finish_dependency = diff --git a/be/src/pipeline/exec/result_file_sink_operator.cpp b/be/src/pipeline/exec/result_file_sink_operator.cpp index 7c9c38ece5c4e9..bc4e4c88d14ca7 100644 --- a/be/src/pipeline/exec/result_file_sink_operator.cpp +++ b/be/src/pipeline/exec/result_file_sink_operator.cpp @@ -95,6 +95,7 @@ Status ResultFileSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& i state->fragment_instance_id(), p._buf_size, &_sender, state->execution_timeout(), state->batch_size())); } + _sender->set_dependency(state->fragment_instance_id(), _dependency->shared_from_this()); // create writer _writer.reset(new (std::nothrow) vectorized::VFileResultWriter( diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index a8213b31ba8f47..400495437d44b5 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -216,10 +216,6 @@ Status PipelineTask::_open() { return Status::OK(); } -void PipelineTask::set_task_queue(TaskQueue* task_queue) { - _task_queue = task_queue; -} - bool PipelineTask::_wait_to_start() { // Before task starting, we should make sure // 1. Execution dependency is ready (which is controlled by FE 2-phase commit) diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h index febc9634c49f23..3b4627f589dc54 100644 --- a/be/src/pipeline/pipeline_task.h +++ b/be/src/pipeline/pipeline_task.h @@ -41,7 +41,7 @@ class PipelineFragmentContext; namespace doris::pipeline { -class TaskQueue; +class MultiCoreTaskQueue; class PriorityTaskQueue; class Dependency; @@ -159,8 +159,8 @@ class PipelineTask { } } - void set_task_queue(TaskQueue* task_queue); - TaskQueue* get_task_queue() { return _task_queue; } + void set_task_queue(MultiCoreTaskQueue* task_queue) { _task_queue = task_queue; } + MultiCoreTaskQueue* get_task_queue() { return _task_queue; } static constexpr auto THREAD_TIME_SLICE = 100'000'000ULL; @@ -257,7 +257,7 @@ class PipelineTask { uint32_t _schedule_time = 0; std::unique_ptr _block; PipelineFragmentContext* _fragment_context = nullptr; - TaskQueue* _task_queue = nullptr; + MultiCoreTaskQueue* _task_queue = nullptr; // used for priority queue // it may be visited by different thread but there is no race condition diff --git a/be/src/pipeline/query_cache/query_cache.h b/be/src/pipeline/query_cache/query_cache.h index a905831b530578..827c516ad75f07 100644 --- a/be/src/pipeline/query_cache/query_cache.h +++ b/be/src/pipeline/query_cache/query_cache.h @@ -37,6 +37,7 @@ #include "runtime/memory/mem_tracker.h" #include "util/slice.h" #include "util/time.h" +#include "vec/core/block.h" namespace doris { diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp index ade960650d74c2..ea812ca9b12dd6 100644 --- a/be/src/pipeline/task_queue.cpp +++ b/be/src/pipeline/task_queue.cpp @@ -28,7 +28,6 @@ namespace doris::pipeline { #include "common/compile_check_begin.h" -TaskQueue::~TaskQueue() = default; PipelineTask* SubTaskQueue::try_take(bool is_steal) { if (_queue.empty()) { @@ -133,44 +132,35 @@ Status PriorityTaskQueue::push(PipelineTask* task) { MultiCoreTaskQueue::~MultiCoreTaskQueue() = default; -MultiCoreTaskQueue::MultiCoreTaskQueue(int core_size) : TaskQueue(core_size), _closed(false) { - _prio_task_queue_list = - std::make_shared>>(core_size); - for (int i = 0; i < core_size; i++) { - (*_prio_task_queue_list)[i] = std::make_unique(); - } -} +MultiCoreTaskQueue::MultiCoreTaskQueue(int core_size) + : _prio_task_queues(core_size), _closed(false), _core_size(core_size) {} void MultiCoreTaskQueue::close() { if (_closed) { return; } _closed = true; - for (int i = 0; i < _core_size; ++i) { - (*_prio_task_queue_list)[i]->close(); - } - std::atomic_store(&_prio_task_queue_list, - std::shared_ptr>>(nullptr)); + // close all priority task queue + std::ranges::for_each(_prio_task_queues, + [](auto& prio_task_queue) { prio_task_queue.close(); }); } PipelineTask* MultiCoreTaskQueue::take(int core_id) { PipelineTask* task = nullptr; - auto prio_task_queue_list = - std::atomic_load_explicit(&_prio_task_queue_list, std::memory_order_relaxed); while (!_closed) { - DCHECK(prio_task_queue_list->size() > core_id) - << " list size: " << prio_task_queue_list->size() << " core_id: " << core_id + DCHECK(_prio_task_queues.size() > core_id) + << " list size: " << _prio_task_queues.size() << " core_id: " << core_id << " _core_size: " << _core_size << " _next_core: " << _next_core.load(); - task = (*prio_task_queue_list)[core_id]->try_take(false); + task = _prio_task_queues[core_id].try_take(false); if (task) { task->set_core_id(core_id); break; } - task = _steal_take(core_id, *prio_task_queue_list); + task = _steal_take(core_id); if (task) { break; } - task = (*prio_task_queue_list)[core_id]->take(WAIT_CORE_TASK_TIMEOUT_MS /* timeout_ms */); + task = _prio_task_queues[core_id].take(WAIT_CORE_TASK_TIMEOUT_MS /* timeout_ms */); if (task) { task->set_core_id(core_id); break; @@ -182,8 +172,7 @@ PipelineTask* MultiCoreTaskQueue::take(int core_id) { return task; } -PipelineTask* MultiCoreTaskQueue::_steal_take( - int core_id, std::vector>& prio_task_queue_list) { +PipelineTask* MultiCoreTaskQueue::_steal_take(int core_id) { DCHECK(core_id < _core_size); int next_id = core_id; for (int i = 1; i < _core_size; ++i) { @@ -192,7 +181,7 @@ PipelineTask* MultiCoreTaskQueue::_steal_take( next_id = 0; } DCHECK(next_id < _core_size); - auto task = prio_task_queue_list[next_id]->try_take(true); + auto task = _prio_task_queues[next_id].try_take(true); if (task) { task->set_core_id(next_id); return task; @@ -212,17 +201,13 @@ Status MultiCoreTaskQueue::push_back(PipelineTask* task) { Status MultiCoreTaskQueue::push_back(PipelineTask* task, int core_id) { DCHECK(core_id < _core_size); task->put_in_runnable_queue(); - auto prio_task_queue_list = - std::atomic_load_explicit(&_prio_task_queue_list, std::memory_order_relaxed); - return (*prio_task_queue_list)[core_id]->push(task); + return _prio_task_queues[core_id].push(task); } void MultiCoreTaskQueue::update_statistics(PipelineTask* task, int64_t time_spent) { task->inc_runtime_ns(time_spent); - auto prio_task_queue_list = - std::atomic_load_explicit(&_prio_task_queue_list, std::memory_order_relaxed); - (*prio_task_queue_list)[task->get_core_id()]->inc_sub_queue_runtime(task->get_queue_level(), - time_spent); + _prio_task_queues[task->get_core_id()].inc_sub_queue_runtime(task->get_queue_level(), + time_spent); } } // namespace doris::pipeline \ No newline at end of file diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h index c9e74248c7ac17..1651eb50cac4ab 100644 --- a/be/src/pipeline/task_queue.h +++ b/be/src/pipeline/task_queue.h @@ -34,30 +34,6 @@ namespace doris::pipeline { #include "common/compile_check_begin.h" -class TaskQueue { -public: - TaskQueue(int core_size) : _core_size(core_size) {} - virtual ~TaskQueue(); - virtual void close() = 0; - // Get the task by core id. - // TODO: To think the logic is useful? - virtual PipelineTask* take(int core_id) = 0; - - // push from scheduler - virtual Status push_back(PipelineTask* task) = 0; - - // push from worker - virtual Status push_back(PipelineTask* task, int core_id) = 0; - - virtual void update_statistics(PipelineTask* task, int64_t time_spent) {} - - int cores() const { return _core_size; } - -protected: - int _core_size; - static constexpr auto WAIT_CORE_TASK_TIMEOUT_MS = 100; -}; - class SubTaskQueue { friend class PriorityTaskQueue; @@ -127,31 +103,35 @@ class PriorityTaskQueue { }; // Need consider NUMA architecture -class MultiCoreTaskQueue : public TaskQueue { +class MultiCoreTaskQueue { public: explicit MultiCoreTaskQueue(int core_size); - ~MultiCoreTaskQueue() override; + ~MultiCoreTaskQueue(); - void close() override; + void close(); // Get the task by core id. - PipelineTask* take(int core_id) override; + PipelineTask* take(int core_id); // TODO combine these methods to `push_back(task, core_id = -1)` - Status push_back(PipelineTask* task) override; + Status push_back(PipelineTask* task); - Status push_back(PipelineTask* task, int core_id) override; + Status push_back(PipelineTask* task, int core_id); - void update_statistics(PipelineTask* task, int64_t time_spent) override; + void update_statistics(PipelineTask* task, int64_t time_spent); + + int cores() const { return _core_size; } private: - PipelineTask* _steal_take( - int core_id, std::vector>& prio_task_queue_list); + PipelineTask* _steal_take(int core_id); - std::shared_ptr>> _prio_task_queue_list; + std::vector _prio_task_queues; std::atomic _next_core = 0; std::atomic _closed; + + int _core_size; + static constexpr auto WAIT_CORE_TASK_TIMEOUT_MS = 100; }; #include "common/compile_check_end.h" } // namespace doris::pipeline diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp index 5a4f8819bcb307..45898e764175b2 100644 --- a/be/src/pipeline/task_scheduler.cpp +++ b/be/src/pipeline/task_scheduler.cpp @@ -51,7 +51,7 @@ TaskScheduler::~TaskScheduler() { } Status TaskScheduler::start() { - int cores = _task_queue->cores(); + int cores = _task_queue.cores(); RETURN_IF_ERROR(ThreadPoolBuilder(_name) .set_min_threads(cores) .set_max_threads(cores) @@ -67,7 +67,7 @@ Status TaskScheduler::start() { } Status TaskScheduler::schedule_task(PipelineTask* task) { - return _task_queue->push_back(task); + return _task_queue.push_back(task); } // after _close_task, task maybe destructed. @@ -99,17 +99,17 @@ void _close_task(PipelineTask* task, Status exec_status) { void TaskScheduler::_do_work(int index) { while (_markers[index]) { - auto* task = _task_queue->take(index); + auto* task = _task_queue.take(index); if (!task) { continue; } if (task->is_running()) { - static_cast(_task_queue->push_back(task, index)); + static_cast(_task_queue.push_back(task, index)); continue; } task->log_detail_if_need(); task->set_running(true); - task->set_task_queue(_task_queue.get()); + task->set_task_queue(&_task_queue); auto* fragment_ctx = task->fragment_context(); bool canceled = fragment_ctx->is_canceled(); @@ -189,9 +189,7 @@ void TaskScheduler::_do_work(int index) { void TaskScheduler::stop() { if (!_shutdown) { - if (_task_queue) { - _task_queue->close(); - } + _task_queue.close(); if (_fix_thread_pool) { for (size_t i = 0; i < _markers.size(); ++i) { _markers[i] = false; diff --git a/be/src/pipeline/task_scheduler.h b/be/src/pipeline/task_scheduler.h index 4caceca20d4a44..bdb5bec1776f58 100644 --- a/be/src/pipeline/task_scheduler.h +++ b/be/src/pipeline/task_scheduler.h @@ -31,6 +31,7 @@ #include "gutil/ref_counted.h" #include "pipeline_task.h" #include "runtime/workload_group/workload_group.h" +#include "task_queue.h" #include "util/thread.h" namespace doris { @@ -39,13 +40,11 @@ class ThreadPool; } // namespace doris namespace doris::pipeline { -class TaskQueue; class TaskScheduler { public: - TaskScheduler(std::shared_ptr task_queue, std::string name, - CgroupCpuCtl* cgroup_cpu_ctl) - : _task_queue(std::move(task_queue)), + TaskScheduler(int core_num, std::string name, CgroupCpuCtl* cgroup_cpu_ctl) + : _task_queue(core_num), _shutdown(false), _name(std::move(name)), _cgroup_cpu_ctl(cgroup_cpu_ctl) {} @@ -62,7 +61,7 @@ class TaskScheduler { private: std::unique_ptr _fix_thread_pool; - std::shared_ptr _task_queue; + MultiCoreTaskQueue _task_queue; std::vector _markers; bool _shutdown; std::string _name; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index fdbd6507d13472..b1617744eac6ba 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -101,6 +101,7 @@ class FrontendServiceClient; class FileMetaCache; class GroupCommitMgr; class TabletSchemaCache; +class TabletColumnObjectPool; class UserFunctionCache; class SchemaCache; class StoragePageCache; @@ -275,6 +276,9 @@ class ExecEnv { void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } + void set_tablet_column_object_pool(TabletColumnObjectPool* c) { + this->_tablet_column_object_pool = c; + } void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { @@ -300,6 +304,7 @@ class ExecEnv { std::map get_running_frontends(); TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } + TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } SchemaCache* schema_cache() { return _schema_cache; } StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } SegmentLoader* segment_loader() { return _segment_loader; } @@ -439,6 +444,7 @@ class ExecEnv { // these redundancy header could introduce potential bug, at least, more header means slow compile. // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. TabletSchemaCache* _tablet_schema_cache = nullptr; + TabletColumnObjectPool* _tablet_column_object_pool = nullptr; std::unique_ptr _storage_engine; SchemaCache* _schema_cache = nullptr; StoragePageCache* _storage_page_cache = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 3cddcd60b8bd8e..e43524b2d2a00b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -53,6 +53,7 @@ #include "olap/schema_cache.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" +#include "olap/tablet_column_object_pool.h" #include "olap/tablet_schema_cache.h" #include "olap/wal/wal_manager.h" #include "pipeline/pipeline_tracing.h" @@ -339,6 +340,9 @@ Status ExecEnv::_init(const std::vector& store_paths, _tablet_schema_cache = TabletSchemaCache::create_global_schema_cache(config::tablet_schema_cache_capacity); + _tablet_column_object_pool = TabletColumnObjectPool::create_global_column_cache( + config::tablet_schema_cache_capacity); + // Storage engine doris::EngineOptions options; options.store_paths = store_paths; @@ -381,9 +385,8 @@ Status ExecEnv::init_pipeline_task_scheduler() { LOG_INFO("pipeline executors_size set ").tag("size", executors_size); // TODO pipeline workload group combie two blocked schedulers. - auto t_queue = std::make_shared(executors_size); _without_group_task_scheduler = - new pipeline::TaskScheduler(t_queue, "PipeNoGSchePool", nullptr); + new pipeline::TaskScheduler(executors_size, "PipeNoGSchePool", nullptr); RETURN_IF_ERROR(_without_group_task_scheduler->start()); _runtime_filter_timer_queue = new doris::pipeline::RuntimeFilterTimerQueue(); diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h index 666d32bdb56e4d..e7e1c73e7cbb41 100644 --- a/be/src/runtime/memory/cache_policy.h +++ b/be/src/runtime/memory/cache_policy.h @@ -48,7 +48,8 @@ class CachePolicy { CLOUD_TXN_DELETE_BITMAP_CACHE = 17, NONE = 18, // not be used FOR_UT_CACHE_NUMBER = 19, - QUERY_CACHE = 20 + QUERY_CACHE = 20, + TABLET_COLUMN_OBJECT_POOL = 21, }; static std::string type_string(CacheType type) { @@ -93,6 +94,8 @@ class CachePolicy { return "ForUTCacheNumber"; case CacheType::QUERY_CACHE: return "QueryCache"; + case CacheType::TABLET_COLUMN_OBJECT_POOL: + return "TabletColumnObjectPool"; default: LOG(FATAL) << "not match type of cache policy :" << static_cast(type); } @@ -119,7 +122,8 @@ class CachePolicy { {"CreateTabletRRIdxCache", CacheType::CREATE_TABLET_RR_IDX_CACHE}, {"CloudTabletCache", CacheType::CLOUD_TABLET_CACHE}, {"CloudTxnDeleteBitmapCache", CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE}, - {"ForUTCacheNumber", CacheType::FOR_UT_CACHE_NUMBER}}; + {"ForUTCacheNumber", CacheType::FOR_UT_CACHE_NUMBER}, + {"TabletColumnObjectPool", CacheType::TABLET_COLUMN_OBJECT_POOL}}; static CacheType string_to_type(std::string type) { if (StringToType.contains(type)) { diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index 84016132da9b5a..c6a3c07adda1dd 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -468,9 +468,8 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e if (executors_size <= 0) { executors_size = CpuInfo::num_cores(); } - auto task_queue = std::make_shared(executors_size); std::unique_ptr pipeline_task_scheduler = - std::make_unique(std::move(task_queue), "Pipe_" + tg_name, + std::make_unique(executors_size, "Pipe_" + tg_name, cg_cpu_ctl_ptr); Status ret = pipeline_task_scheduler->start(); if (ret.ok()) { diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index e6fdfaa87657f8..86d47add0dadad 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -353,11 +353,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg* arg) { std::vector segment_index_file_names; auto tablet_schema = rowset_meta->tablet_schema(); if (tablet_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (const auto& index : tablet_schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } - auto index_id = index.index_id(); + for (const auto& index : tablet_schema->inverted_indexes()) { + auto index_id = index->index_id(); for (int64_t segment_index = 0; segment_index < num_segments; ++segment_index) { auto get_segment_index_file_size_url = fmt::format( "{}?method={}&tablet_id={}&rowset_id={}&segment_index={}&segment_index_id={" @@ -379,7 +376,7 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg* arg) { rowset_meta->rowset_id().to_string(), segment_index); segment_index_file_names.push_back(InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(segment_path), index_id, - index.get_index_suffix())); + index->get_index_suffix())); status = HttpClient::execute_with_retry(max_retry, 1, get_segment_index_file_size_cb); diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index d13c0c091b9ced..d1788b0948a6f2 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include "common/config.h" diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index b2f4cdc3ce7885..18058469ee4566 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -401,15 +401,15 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { S3Conf ret { .bucket = info.bucket(), .prefix = info.prefix(), - .client_conf { - .endpoint = info.endpoint(), - .region = info.region(), - .ak = info.ak(), - .sk = info.sk(), - .token {}, - .bucket = info.bucket(), - .provider = io::ObjStorageType::AWS, - }, + .client_conf {.endpoint = info.endpoint(), + .region = info.region(), + .ak = info.ak(), + .sk = info.sk(), + .token {}, + .bucket = info.bucket(), + .provider = io::ObjStorageType::AWS, + .use_virtual_addressing = + info.has_use_path_style() ? !info.use_path_style() : true}, .sse_enabled = info.sse_enabled(), }; diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index acf27cac848649..3d392e2addbae5 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -1311,14 +1311,14 @@ rapidjson::Value* find_leaf_node_by_path(rapidjson::Value& json, const PathInDat // 2. nested array with only nulls, eg. [null. null],todo: think a better way to deal distinguish array null value and real null value. // 3. empty root jsonb value(not null) // 4. type is nothing -bool skip_empty_json(const ColumnNullable* nullable, const DataTypePtr& type, int row, - const PathInData& path) { +bool skip_empty_json(const ColumnNullable* nullable, const DataTypePtr& type, + TypeIndex base_type_id, int row, const PathInData& path) { // skip nulls if (nullable && nullable->is_null_at(row)) { return true; } // check if it is empty nested json array, then skip - if (type->equals(*ColumnObject::NESTED_TYPE)) { + if (base_type_id == TypeIndex::VARIANT && type->equals(*ColumnObject::NESTED_TYPE)) { Field field = (*nullable)[row]; if (field.get_type() == Field::Types::Array) { const auto& array = field.get(); @@ -1338,7 +1338,7 @@ bool skip_empty_json(const ColumnNullable* nullable, const DataTypePtr& type, in return true; } // skip nothing type - if (WhichDataType(remove_nullable(get_base_type_of_array(type))).is_nothing()) { + if (base_type_id == TypeIndex::Nothing) { return true; } return false; @@ -1346,17 +1346,19 @@ bool skip_empty_json(const ColumnNullable* nullable, const DataTypePtr& type, in Status find_and_set_leave_value(const IColumn* column, const PathInData& path, const DataTypeSerDeSPtr& type_serde, const DataTypePtr& type, - rapidjson::Value& root, + TypeIndex base_type_index, rapidjson::Value& root, rapidjson::Document::AllocatorType& allocator, Arena& mem_pool, int row) { +#ifndef NDEBUG // sanitize type and column if (column->get_name() != type->create_column()->get_name()) { return Status::InternalError( "failed to set value for path {}, expected type {}, but got {} at row {}", path.get_path(), type->get_name(), column->get_name(), row); } +#endif const auto* nullable = check_and_get_column(column); - if (skip_empty_json(nullable, type, row, path)) { + if (skip_empty_json(nullable, type, base_type_index, row, path)) { return Status::OK(); } // TODO could cache the result of leaf nodes with it's path info @@ -1476,11 +1478,12 @@ Status ColumnObject::serialize_one_row_to_json_format(int row, rapidjson::String VLOG_DEBUG << "dump structure " << JsonFunctions::print_json_value(*doc_structure); #endif for (const auto& subcolumn : subcolumns) { - RETURN_IF_ERROR(find_and_set_leave_value(subcolumn->data.get_finalized_column_ptr(), - subcolumn->path, - subcolumn->data.get_least_common_type_serde(), - subcolumn->data.get_least_common_type(), root, - doc_structure->GetAllocator(), mem_pool, row)); + RETURN_IF_ERROR(find_and_set_leave_value( + subcolumn->data.get_finalized_column_ptr(), subcolumn->path, + subcolumn->data.get_least_common_type_serde(), + subcolumn->data.get_least_common_type(), + subcolumn->data.least_common_type.get_base_type_id(), root, + doc_structure->GetAllocator(), mem_pool, row)); if (subcolumn->path.empty() && !root.IsObject()) { // root was modified, only handle root node break; @@ -1549,10 +1552,11 @@ Status ColumnObject::merge_sparse_to_root_column() { ++null_count; continue; } - bool succ = find_and_set_leave_value(column, subcolumn->path, - subcolumn->data.get_least_common_type_serde(), - subcolumn->data.get_least_common_type(), root, - doc_structure->GetAllocator(), mem_pool, i); + bool succ = find_and_set_leave_value( + column, subcolumn->path, subcolumn->data.get_least_common_type_serde(), + subcolumn->data.get_least_common_type(), + subcolumn->data.least_common_type.get_base_type_id(), root, + doc_structure->GetAllocator(), mem_pool, i); if (succ && subcolumn->path.empty() && !root.IsObject()) { // root was modified, only handle root node break; diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index adaee6e9fe6a5d..9692c02fda31e9 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -360,6 +360,7 @@ void update_least_sparse_column(const std::vector& schemas, void inherit_column_attributes(const TabletColumn& source, TabletColumn& target, TabletSchemaSPtr& target_schema) { + DCHECK(target.is_extracted_column()); if (target.type() != FieldType::OLAP_FIELD_TYPE_TINYINT && target.type() != FieldType::OLAP_FIELD_TYPE_ARRAY && target.type() != FieldType::OLAP_FIELD_TYPE_DOUBLE && @@ -368,18 +369,18 @@ void inherit_column_attributes(const TabletColumn& source, TabletColumn& target, target.set_is_bf_column(source.is_bf_column()); } target.set_aggregation_method(source.aggregation()); - const auto* source_index_meta = target_schema->get_inverted_index(source.unique_id(), ""); + const auto* source_index_meta = target_schema->inverted_index(source.unique_id()); if (source_index_meta != nullptr) { // add index meta TabletIndex index_info = *source_index_meta; index_info.set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path()); - // get_inverted_index: No need to check, just inherit directly - const auto* target_index_meta = target_schema->get_inverted_index(target, false); + const auto* target_index_meta = target_schema->inverted_index( + target.parent_unique_id(), target.path_info_ptr()->get_path()); if (target_index_meta != nullptr) { // already exist target_schema->update_index(target, index_info); } else { - target_schema->append_index(index_info); + target_schema->append_index(std::move(index_info)); } } } @@ -591,4 +592,20 @@ Status extract(ColumnPtr source, const PathInData& path, MutableColumnPtr& dst) return Status::OK(); } +bool has_schema_index_diff(const TabletSchema* new_schema, const TabletSchema* old_schema, + int32_t new_col_idx, int32_t old_col_idx) { + const auto& column_new = new_schema->column(new_col_idx); + const auto& column_old = old_schema->column(old_col_idx); + + if (column_new.is_bf_column() != column_old.is_bf_column() || + column_new.has_bitmap_index() != column_old.has_bitmap_index()) { + return true; + } + + bool new_schema_has_inverted_index = new_schema->inverted_index(column_new); + bool old_schema_has_inverted_index = old_schema->inverted_index(column_old); + + return new_schema_has_inverted_index != old_schema_has_inverted_index; +} + } // namespace doris::vectorized::schema_util diff --git a/be/src/vec/common/schema_util.h b/be/src/vec/common/schema_util.h index 080e6331dc1dd0..8ceb97a915630d 100644 --- a/be/src/vec/common/schema_util.h +++ b/be/src/vec/common/schema_util.h @@ -121,4 +121,7 @@ Status extract(ColumnPtr source, const PathInData& path, MutableColumnPtr& dst); std::string dump_column(DataTypePtr type, const ColumnPtr& col); +bool has_schema_index_diff(const TabletSchema* new_schema, const TabletSchema* old_schema, + int32_t new_col_idx, int32_t old_col_idx); + } // namespace doris::vectorized::schema_util diff --git a/be/src/vec/data_types/data_type_string.cpp b/be/src/vec/data_types/data_type_string.cpp index d2c2ae2c0b03a2..878e6c319a103b 100644 --- a/be/src/vec/data_types/data_type_string.cpp +++ b/be/src/vec/data_types/data_type_string.cpp @@ -27,6 +27,8 @@ #include #include "agent/be_exec_version_manager.h" +#include "common/exception.h" +#include "common/status.h" #include "vec/columns/column.h" #include "vec/columns/column_const.h" #include "vec/columns/column_string.h" @@ -81,7 +83,7 @@ bool DataTypeString::equals(const IDataType& rhs) const { int64_t DataTypeString::get_uncompressed_serialized_bytes(const IColumn& column, int be_exec_version) const { if (be_exec_version >= USE_CONST_SERDE) { - auto size = sizeof(bool) + sizeof(size_t) + sizeof(size_t); + int64_t size = sizeof(bool) + sizeof(size_t) + sizeof(size_t); bool is_const_column = is_column_const(column); const IColumn* string_column = &column; if (is_const_column) { @@ -99,9 +101,15 @@ int64_t DataTypeString::get_uncompressed_serialized_bytes(const IColumn& column, upper_int32(offsets_size))); } size += sizeof(size_t); - if (auto bytes = data_column.get_chars().size(); bytes <= SERIALIZED_MEM_SIZE_LIMIT) { + if (size_t bytes = data_column.get_chars().size(); bytes <= SERIALIZED_MEM_SIZE_LIMIT) { size += bytes; } else { + if (bytes > LZ4_MAX_INPUT_SIZE) { + throw Exception(ErrorCode::BUFFER_OVERFLOW, + "LZ4_compressBound meet invalid input size, input_size={}, " + "LZ4_MAX_INPUT_SIZE={}", + bytes, LZ4_MAX_INPUT_SIZE); + } size += sizeof(size_t) + std::max(bytes, (size_t)LZ4_compressBound(bytes)); } return size; diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 6b6639f2feb244..e2161d8a6dc48a 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -857,28 +857,79 @@ Status OrcReader::set_fill_columns( if (_colname_to_value_range == nullptr || !_init_search_argument(_colname_to_value_range)) { _lazy_read_ctx.can_lazy_read = false; } + try { + _row_reader_options.range(_range_start_offset, _range_size); + _row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz); + _row_reader_options.include(_read_cols); + _row_reader_options.setEnableLazyDecoding(true); - if (!_lazy_read_ctx.can_lazy_read) { - for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { - _lazy_read_ctx.partition_columns.emplace(kv.first, kv.second); + uint64_t number_of_stripes = _reader->getNumberOfStripes(); + auto all_stripes_needed = _reader->getNeedReadStripes(_row_reader_options); + + int64_t range_end_offset = _range_start_offset + _range_size; + + // If you set "orc_tiny_stripe_threshold_bytes" = 0, the use tiny stripes merge io optimization will not be used. + int64_t orc_tiny_stripe_threshold_bytes = 8L * 1024L * 1024L; + int64_t orc_once_max_read_bytes = 8L * 1024L * 1024L; + int64_t orc_max_merge_distance_bytes = 1L * 1024L * 1024L; + + if (_state != nullptr) { + orc_tiny_stripe_threshold_bytes = + _state->query_options().orc_tiny_stripe_threshold_bytes; + orc_once_max_read_bytes = _state->query_options().orc_once_max_read_bytes; + orc_max_merge_distance_bytes = _state->query_options().orc_max_merge_distance_bytes; } - for (auto& kv : _lazy_read_ctx.predicate_missing_columns) { - _lazy_read_ctx.missing_columns.emplace(kv.first, kv.second); + + bool all_tiny_stripes = true; + std::vector tiny_stripe_ranges; + + for (uint64_t i = 0; i < number_of_stripes; i++) { + std::unique_ptr strip_info = _reader->getStripe(i); + uint64_t strip_start_offset = strip_info->getOffset(); + uint64_t strip_end_offset = strip_start_offset + strip_info->getLength(); + + if (strip_start_offset >= range_end_offset || strip_end_offset < _range_start_offset || + !all_stripes_needed[i]) { + continue; + } + if (strip_info->getLength() > orc_tiny_stripe_threshold_bytes) { + all_tiny_stripes = false; + break; + } + + tiny_stripe_ranges.emplace_back(strip_start_offset, strip_end_offset); } - } + if (all_tiny_stripes && number_of_stripes > 0) { + std::vector prefetch_merge_ranges = + io::PrefetchRange::merge_adjacent_seq_ranges(tiny_stripe_ranges, + orc_max_merge_distance_bytes, + orc_once_max_read_bytes); + auto range_finder = + std::make_shared(std::move(prefetch_merge_ranges)); - _fill_all_columns = true; + auto* orc_input_stream_ptr = static_cast(_reader->getStream()); + orc_input_stream_ptr->set_all_tiny_stripes(); + auto& orc_file_reader = orc_input_stream_ptr->get_file_reader(); + auto orc_inner_reader = orc_input_stream_ptr->get_inner_reader(); + orc_file_reader = std::make_shared(_profile, orc_inner_reader, + range_finder); + } - // create orc row reader - try { - _row_reader_options.range(_range_start_offset, _range_size); - _row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz); - _row_reader_options.include(_read_cols); + if (!_lazy_read_ctx.can_lazy_read) { + for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { + _lazy_read_ctx.partition_columns.emplace(kv.first, kv.second); + } + for (auto& kv : _lazy_read_ctx.predicate_missing_columns) { + _lazy_read_ctx.missing_columns.emplace(kv.first, kv.second); + } + } + + _fill_all_columns = true; + // create orc row reader if (_lazy_read_ctx.can_lazy_read) { _row_reader_options.filter(_lazy_read_ctx.predicate_orc_columns); _orc_filter = std::unique_ptr(new ORCFilterImpl(this)); } - _row_reader_options.setEnableLazyDecoding(true); if (!_lazy_read_ctx.conjuncts.empty()) { _string_dict_filter = std::make_unique(this); } @@ -2415,6 +2466,9 @@ MutableColumnPtr OrcReader::_convert_dict_column_to_string_column( void ORCFileInputStream::beforeReadStripe( std::unique_ptr current_strip_information, std::vector selected_columns) { + if (_is_all_tiny_stripes) { + return; + } if (_file_reader != nullptr) { _file_reader->collect_profile_before_close(); } diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index 4aad5637ef544e..0807f4949e5850 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -34,6 +34,7 @@ #include "common/status.h" #include "exec/olap_common.h" #include "io/file_factory.h" +#include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" #include "io/fs/file_reader_writer_fwd.h" #include "olap/olap_common.h" @@ -642,7 +643,11 @@ class ORCFileInputStream : public orc::InputStream, public ProfileCollector { _io_ctx(io_ctx), _profile(profile) {} - ~ORCFileInputStream() override = default; + ~ORCFileInputStream() override { + if (_file_reader != nullptr) { + _file_reader->collect_profile_before_close(); + } + } uint64_t getLength() const override { return _file_reader->size(); } @@ -655,6 +660,12 @@ class ORCFileInputStream : public orc::InputStream, public ProfileCollector { void beforeReadStripe(std::unique_ptr current_strip_information, std::vector selected_columns) override; + void set_all_tiny_stripes() { _is_all_tiny_stripes = true; } + + io::FileReaderSPtr& get_file_reader() { return _file_reader; } + + io::FileReaderSPtr& get_inner_reader() { return _inner_reader; } + protected: void _collect_profile_at_runtime() override {}; void _collect_profile_before_close() override; @@ -663,10 +674,10 @@ class ORCFileInputStream : public orc::InputStream, public ProfileCollector { const std::string& _file_name; io::FileReaderSPtr _inner_reader; io::FileReaderSPtr _file_reader; + bool _is_all_tiny_stripes = false; // Owned by OrcReader OrcReader::Statistics* _statistics = nullptr; const io::IOContext* _io_ctx = nullptr; RuntimeProfile* _profile = nullptr; }; - } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/table/paimon_jni_reader.cpp b/be/src/vec/exec/format/table/paimon_jni_reader.cpp index a9ec243cf460b8..30358eace1aae2 100644 --- a/be/src/vec/exec/format/table/paimon_jni_reader.cpp +++ b/be/src/vec/exec/format/table/paimon_jni_reader.cpp @@ -61,6 +61,9 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d std::to_string(range.table_format_params.paimon_params.last_update_time); params["required_fields"] = join(column_names, ","); params["columns_types"] = join(column_types, "#"); + if (range.table_format_params.paimon_params.__isset.paimon_table) { + params["paimon_table"] = range.table_format_params.paimon_params.paimon_table; + } // Used to create paimon option for (auto& kv : range.table_format_params.paimon_params.paimon_options) { diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index 0bef185351dc87..332eaed37d4483 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -33,6 +33,7 @@ #include "vec/exprs/table_function/vexplode_map.h" #include "vec/exprs/table_function/vexplode_numbers.h" #include "vec/exprs/table_function/vexplode_split.h" +#include "vec/exprs/table_function/vposexplode.h" #include "vec/utils/util.hpp" namespace doris::vectorized { @@ -61,6 +62,7 @@ const std::unordered_map()}, {"explode_map", TableFunctionCreator {}}, {"explode_json_object", TableFunctionCreator {}}, + {"posexplode", TableFunctionCreator {}}, {"explode", TableFunctionCreator {}}}; Status TableFunctionFactory::get_fn(const TFunction& t_fn, ObjectPool* pool, TableFunction** fn) { diff --git a/be/src/vec/exprs/table_function/vposexplode.cpp b/be/src/vec/exprs/table_function/vposexplode.cpp new file mode 100644 index 00000000000000..20d04a219f831a --- /dev/null +++ b/be/src/vec/exprs/table_function/vposexplode.cpp @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exprs/table_function/vposexplode.h" + +#include + +#include +#include + +#include "common/status.h" +#include "vec/columns/column.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" + +namespace doris::vectorized { + +VPosExplodeTableFunction::VPosExplodeTableFunction() { + _fn_name = "posexplode"; +} + +Status VPosExplodeTableFunction::process_init(Block* block, RuntimeState* state) { + CHECK(_expr_context->root()->children().size() == 1) + << "VPosExplodeTableFunction only support 1 child but has " + << _expr_context->root()->children().size(); + + int value_column_idx = -1; + RETURN_IF_ERROR(_expr_context->root()->children()[0]->execute(_expr_context.get(), block, + &value_column_idx)); + + _collection_column = + block->get_by_position(value_column_idx).column->convert_to_full_column_if_const(); + + if (!extract_column_array_info(*_collection_column, _array_detail)) { + return Status::NotSupported("column type {} not supported now, only support array", + block->get_by_position(value_column_idx).column->get_name()); + } + if (is_column_nullable(*_collection_column)) { + _array_data_column = + assert_cast( + assert_cast(*_collection_column).get_nested_column()) + .get_data_ptr(); + } else { + _array_data_column = assert_cast(*_collection_column).get_data_ptr(); + } + return Status::OK(); +} + +void VPosExplodeTableFunction::process_row(size_t row_idx) { + DCHECK(row_idx < _collection_column->size()); + TableFunction::process_row(row_idx); + + if (!_array_detail.array_nullmap_data || !_array_detail.array_nullmap_data[row_idx]) { + _collection_offset = (*_array_detail.offsets_ptr)[row_idx - 1]; + _cur_size = (*_array_detail.offsets_ptr)[row_idx] - _collection_offset; + } +} + +void VPosExplodeTableFunction::process_close() { + _collection_column = nullptr; + _array_data_column = nullptr; + _array_detail.reset(); + _collection_offset = 0; +} + +void VPosExplodeTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { + // now we only support array column explode to struct column + size_t pos = _collection_offset + _cur_offset; + // if current is empty array row, also append a default value + if (current_empty()) { + column->insert_many_defaults(length); + return; + } + ColumnStruct* ret = nullptr; + // this _is_nullable is whole output column's nullable + if (_is_nullable) { + ret = assert_cast( + assert_cast(column.get())->get_nested_column_ptr().get()); + assert_cast( + assert_cast(column.get())->get_null_map_column_ptr().get()) + ->insert_many_defaults(length); + } else if (column->is_column_struct()) { + ret = assert_cast(column.get()); + } else { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support array column explode to struct column"); + } + if (!ret || ret->tuple_size() != 2) { + throw Exception( + ErrorCode::INTERNAL_ERROR, + "only support array column explode to two column, but given: ", ret->tuple_size()); + } + auto& pose_column_nullable = assert_cast(ret->get_column(0)); + pose_column_nullable.get_null_map_column().insert_many_defaults(length); + assert_cast(pose_column_nullable.get_nested_column()) + .insert_many_vals(_cur_offset, length); + ret->get_column(1).insert_many_from(*_array_data_column, pos, length); +} + +int VPosExplodeTableFunction::get_value(MutableColumnPtr& column, int max_step) { + max_step = std::min(max_step, (int)(_cur_size - _cur_offset)); + size_t pos = _collection_offset + _cur_offset; + if (current_empty()) { + column->insert_default(); + max_step = 1; + } else { + ColumnStruct* struct_column = nullptr; + if (_is_nullable) { + auto* nullable_column = assert_cast(column.get()); + struct_column = + assert_cast(nullable_column->get_nested_column_ptr().get()); + auto* nullmap_column = + assert_cast(nullable_column->get_null_map_column_ptr().get()); + // here nullmap_column insert max_step many defaults as if array[row_idx] is NULL + // will be not update value, _cur_size = 0, means current_empty; + // so here could insert directly + nullmap_column->insert_many_defaults(max_step); + } else { + struct_column = assert_cast(column.get()); + } + if (!struct_column || struct_column->tuple_size() != 2) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support array column explode to two column, but given: ", + struct_column->tuple_size()); + } + auto& pose_column_nullable = assert_cast(struct_column->get_column(0)); + pose_column_nullable.get_null_map_column().insert_many_defaults(max_step); + assert_cast(pose_column_nullable.get_nested_column()) + .insert_range_of_integer(_cur_offset, _cur_offset + max_step); + struct_column->get_column(1).insert_range_from(*_array_data_column, pos, max_step); + } + forward(max_step); + return max_step; +} +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/vposexplode.h b/be/src/vec/exprs/table_function/vposexplode.h new file mode 100644 index 00000000000000..4e021fd58da918 --- /dev/null +++ b/be/src/vec/exprs/table_function/vposexplode.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "vec/columns/column_map.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/exprs/table_function/table_function.h" +#include "vec/functions/array/function_array_utils.h" + +namespace doris::vectorized { + +class VPosExplodeTableFunction : public TableFunction { + ENABLE_FACTORY_CREATOR(VPosExplodeTableFunction); + +public: + VPosExplodeTableFunction(); + + ~VPosExplodeTableFunction() override = default; + + Status process_init(Block* block, RuntimeState* state) override; + void process_row(size_t row_idx) override; + void process_close() override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; + +private: + ColumnPtr _collection_column; + ColumnPtr _array_data_column; + ColumnArrayExecutionData _array_detail; + size_t _collection_offset; // start offset of array[row_idx] +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/array/function_array_distance.h b/be/src/vec/functions/array/function_array_distance.h index fa05196e5e1c2c..5284a3838adfce 100644 --- a/be/src/vec/functions/array/function_array_distance.h +++ b/be/src/vec/functions/array/function_array_distance.h @@ -123,7 +123,7 @@ class FunctionArrayDistance : public IFunction { // prepare return data auto dst = ColumnFloat64::create(input_rows_count); auto& dst_data = dst->get_data(); - auto dst_null_column = ColumnUInt8::create(input_rows_count); + auto dst_null_column = ColumnUInt8::create(input_rows_count, 0); auto& dst_null_data = dst_null_column->get_data(); const auto& offsets1 = *arr1.offsets_ptr; diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h index 4a9bffdbb3cc1d..60a353b9eb0b7b 100644 --- a/be/src/vec/functions/array/function_array_element.h +++ b/be/src/vec/functions/array/function_array_element.h @@ -97,7 +97,7 @@ class FunctionArrayElement : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { - auto dst_null_column = ColumnUInt8::create(input_rows_count); + auto dst_null_column = ColumnUInt8::create(input_rows_count, 0); UInt8* dst_null_map = dst_null_column->get_data().data(); const UInt8* src_null_map = nullptr; ColumnsWithTypeAndName args; diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index ec31277c36c855..25a4397309557a 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -217,9 +217,9 @@ class FunctionArrayIndex : public IFunction { const auto& right_chars = reinterpret_cast(right_column).get_chars(); // prepare return data - auto dst = ColumnVector::create(offsets.size()); + auto dst = ColumnVector::create(offsets.size(), 0); auto& dst_data = dst->get_data(); - auto dst_null_column = ColumnUInt8::create(offsets.size()); + auto dst_null_column = ColumnUInt8::create(offsets.size(), 0); auto& dst_null_data = dst_null_column->get_data(); // process @@ -286,9 +286,9 @@ class FunctionArrayIndex : public IFunction { const auto& right_data = reinterpret_cast(right_column).get_data(); // prepare return data - auto dst = ColumnVector::create(offsets.size()); + auto dst = ColumnVector::create(offsets.size(), 0); auto& dst_data = dst->get_data(); - auto dst_null_column = ColumnUInt8::create(offsets.size()); + auto dst_null_column = ColumnUInt8::create(offsets.size(), 0); auto& dst_null_data = dst_null_column->get_data(); // process diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/functions/function_fake.cpp index 6a4f6275e1a89b..646da600b50c13 100644 --- a/be/src/vec/functions/function_fake.cpp +++ b/be/src/vec/functions/function_fake.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -83,6 +84,25 @@ struct FunctionExplodeMap { static std::string get_error_msg() { return "Fake function do not support execute"; } }; +template +struct FunctionPoseExplode { + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + DCHECK(is_array(arguments[0])) << arguments[0]->get_name() << " not supported"; + DataTypes fieldTypes(2); + fieldTypes[0] = make_nullable(std::make_shared()); + fieldTypes[1] = + check_and_get_data_type(arguments[0].get())->get_nested_type(); + auto struct_type = std::make_shared(fieldTypes); + if constexpr (AlwaysNullable) { + return make_nullable(struct_type); + } else { + return arguments[0]->is_nullable() ? make_nullable(struct_type) : struct_type; + } + } + static DataTypes get_variadic_argument_types() { return {}; } + static std::string get_error_msg() { return "Fake function do not support execute"; } +}; + // explode json-object: expands json-object to struct with a pair of key and value in column string struct FunctionExplodeJsonObject { static DataTypePtr get_return_type_impl(const DataTypes& arguments) { @@ -138,6 +158,12 @@ void register_table_function_expand_outer_default(SimpleFunctionFactory& factory COMBINATOR_SUFFIX_OUTER); }; +template +void register_table_function_with_impl(SimpleFunctionFactory& factory, const std::string& name, + const std::string& suffix = "") { + factory.register_function>(name + suffix); +}; + void register_function_fake(SimpleFunctionFactory& factory) { register_function(factory, "esquery"); @@ -158,6 +184,9 @@ void register_function_fake(SimpleFunctionFactory& factory) { register_table_function_expand_outer_default( factory, "explode_json_array_double"); register_table_function_expand_outer_default(factory, "explode_bitmap"); + register_table_function_with_impl>(factory, "posexplode"); + register_table_function_with_impl>(factory, "posexplode", + COMBINATOR_SUFFIX_OUTER); register_table_function_expand_outer_default(factory, "explode_variant_array"); } diff --git a/be/src/vec/runtime/vdata_stream_mgr.cpp b/be/src/vec/runtime/vdata_stream_mgr.cpp index 573b842f2e68cc..382a6d0e6e3ece 100644 --- a/be/src/vec/runtime/vdata_stream_mgr.cpp +++ b/be/src/vec/runtime/vdata_stream_mgr.cpp @@ -72,7 +72,7 @@ std::shared_ptr VDataStreamMgr::create_recvr( fragment_instance_id, dest_node_id, num_senders, is_merging, profile)); uint32_t hash_value = get_hash_value(fragment_instance_id, dest_node_id); - std::lock_guard l(_lock); + std::unique_lock l(_lock); _fragment_stream_set.insert(std::make_pair(fragment_instance_id, dest_node_id)); _receiver_map.insert(std::make_pair(hash_value, recvr)); return recvr; @@ -84,7 +84,7 @@ Status VDataStreamMgr::find_recvr(const TUniqueId& fragment_instance_id, PlanNod << ", node=" << node_id; uint32_t hash_value = get_hash_value(fragment_instance_id, node_id); // Create lock guard and not own lock currently and will lock conditionally - std::unique_lock recvr_lock(_lock, std::defer_lock); + std::shared_lock recvr_lock(_lock, std::defer_lock); if (acquire_lock) { recvr_lock.lock(); } @@ -160,7 +160,7 @@ Status VDataStreamMgr::deregister_recvr(const TUniqueId& fragment_instance_id, P << ", node=" << node_id; uint32_t hash_value = get_hash_value(fragment_instance_id, node_id); { - std::lock_guard l(_lock); + std::unique_lock l(_lock); auto range = _receiver_map.equal_range(hash_value); while (range.first != range.second) { const std::shared_ptr& recvr = range.first->second; @@ -194,7 +194,7 @@ void VDataStreamMgr::cancel(const TUniqueId& fragment_instance_id, Status exec_s VLOG_QUERY << "cancelling all streams for fragment=" << print_id(fragment_instance_id); std::vector> recvrs; { - std::lock_guard l(_lock); + std::shared_lock l(_lock); FragmentStreamSet::iterator i = _fragment_stream_set.lower_bound(std::make_pair(fragment_instance_id, 0)); while (i != _fragment_stream_set.end() && i->first == fragment_instance_id) { diff --git a/be/src/vec/runtime/vdata_stream_mgr.h b/be/src/vec/runtime/vdata_stream_mgr.h index 30fa1c0e318f9e..81f9d19c6b6aeb 100644 --- a/be/src/vec/runtime/vdata_stream_mgr.h +++ b/be/src/vec/runtime/vdata_stream_mgr.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -71,7 +72,7 @@ class VDataStreamMgr { void cancel(const TUniqueId& fragment_instance_id, Status exec_status); private: - std::mutex _lock; + std::shared_mutex _lock; using StreamMap = std::unordered_multimap>; StreamMap _receiver_map; diff --git a/be/test/io/fs/buffered_reader_test.cpp b/be/test/io/fs/buffered_reader_test.cpp index d0a504162d3bb7..658c98ba51465b 100644 --- a/be/test/io/fs/buffered_reader_test.cpp +++ b/be/test/io/fs/buffered_reader_test.cpp @@ -114,6 +114,34 @@ class MockOffsetFileReader : public io::FileReader { io::Path _path = "/tmp/mock"; }; +class TestingRangeCacheFileReader : public io::FileReader { +public: + TestingRangeCacheFileReader(std::shared_ptr delegate) : _delegate(delegate) {}; + + ~TestingRangeCacheFileReader() override = default; + + Status close() override { return _delegate->close(); } + + const io::Path& path() const override { return _delegate->path(); } + + size_t size() const override { return _delegate->size(); } + + bool closed() const override { return _delegate->closed(); } + + const io::PrefetchRange& last_read_range() const { return *_last_read_range; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext* io_ctx) override { + _last_read_range = std::make_unique(offset, offset + result.size); + return _delegate->read_at_impl(offset, result, bytes_read, io_ctx); + } + +private: + std::shared_ptr _delegate; + std::unique_ptr _last_read_range; +}; + TEST_F(BufferedReaderTest, normal_use) { // buffered_reader_test_file 950 bytes io::FileReaderSPtr local_reader; @@ -398,4 +426,84 @@ TEST_F(BufferedReaderTest, test_merged_io) { } } +TEST_F(BufferedReaderTest, test_range_cache_file_reader) { + io::FileReaderSPtr offset_reader = std::make_shared(128 * 1024 * 1024); + auto testing_reader = std::make_shared(offset_reader); + + int64_t orc_max_merge_distance = 1L * 1024L * 1024L; + int64_t orc_once_max_read_size = 8L * 1024L * 1024L; + + { + std::vector tiny_stripe_ranges = { + io::PrefetchRange(3, 33), + io::PrefetchRange(33, 63), + io::PrefetchRange(63, 8L * 1024L * 1024L + 63), + }; + std::vector prefetch_merge_ranges = + io::PrefetchRange::merge_adjacent_seq_ranges( + tiny_stripe_ranges, orc_max_merge_distance, orc_once_max_read_size); + auto range_finder = + std::make_shared(std::move(prefetch_merge_ranges)); + io::RangeCacheFileReader range_cache_file_reader(nullptr, testing_reader, range_finder); + char data[1]; + Slice result(data, 1); + size_t bytes_read; + EXPECT_TRUE(range_cache_file_reader.read_at(3, result, &bytes_read, nullptr).ok()); + EXPECT_EQ(io::PrefetchRange(3, 63), testing_reader->last_read_range()); + + EXPECT_TRUE(range_cache_file_reader.read_at(63, result, &bytes_read, nullptr).ok()); + EXPECT_EQ(io::PrefetchRange(63, 8 * 1024L * 1024L + 63), testing_reader->last_read_range()); + EXPECT_TRUE(range_cache_file_reader.close().ok()); + } + + { + std::vector tiny_stripe_ranges = { + io::PrefetchRange(3, 33), + io::PrefetchRange(33, 63), + io::PrefetchRange(63, 8L * 1024L * 1024L + 63), + }; + std::vector prefetch_merge_ranges = + io::PrefetchRange::merge_adjacent_seq_ranges( + tiny_stripe_ranges, orc_max_merge_distance, orc_once_max_read_size); + auto range_finder = + std::make_shared(std::move(prefetch_merge_ranges)); + io::RangeCacheFileReader range_cache_file_reader(nullptr, testing_reader, range_finder); + char data[1]; + Slice result(data, 1); + size_t bytes_read; + EXPECT_TRUE(range_cache_file_reader.read_at(62, result, &bytes_read, nullptr).ok()); + EXPECT_EQ(io::PrefetchRange(3, 63), testing_reader->last_read_range()); + + EXPECT_TRUE(range_cache_file_reader.read_at(63, result, &bytes_read, nullptr).ok()); + EXPECT_EQ(io::PrefetchRange(63, 8L * 1024L * 1024L + 63), + testing_reader->last_read_range()); + EXPECT_TRUE(range_cache_file_reader.close().ok()); + } + + { + std::vector tiny_stripe_ranges = { + io::PrefetchRange(3, 3), + io::PrefetchRange(4, 1048576L * 5L + 4), + io::PrefetchRange(1048576L * 5L + 4, 1048576L * 3L + 1048576L * 5L + 4), + }; + std::vector prefetch_merge_ranges = + io::PrefetchRange::merge_adjacent_seq_ranges( + tiny_stripe_ranges, orc_max_merge_distance, orc_once_max_read_size); + auto range_finder = + std::make_shared(std::move(prefetch_merge_ranges)); + io::RangeCacheFileReader range_cache_file_reader(nullptr, testing_reader, range_finder); + char data[1]; + Slice result(data, 1); + size_t bytes_read; + EXPECT_TRUE(range_cache_file_reader.read_at(3, result, &bytes_read, nullptr).ok()); + EXPECT_EQ(io::PrefetchRange(3, 1 + 1048576 * 5 + 3), testing_reader->last_read_range()); + + EXPECT_TRUE(range_cache_file_reader.read_at(4 + 1048576 * 5, result, &bytes_read, nullptr) + .ok()); + EXPECT_EQ(io::PrefetchRange(4 + 1048576 * 5, 3 * 1048576 + 4 + 1048576 * 5), + testing_reader->last_read_range()); + EXPECT_TRUE(range_cache_file_reader.close().ok()); + } +} + } // end namespace doris diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp index 922a77fcaa4e47..5e3370847e94b9 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp @@ -406,7 +406,7 @@ TEST_F(IndexCompactionTest, write_index_test) { // read col key const auto& key = _tablet_schema->column_by_uid(0); - const auto* key_index = _tablet_schema->get_inverted_index(key); + const auto* key_index = _tablet_schema->inverted_index(key); EXPECT_TRUE(key_index != nullptr); std::vector query_data {99, 66, 56, 87, 85, 96, 20000}; std::vector query_result {21, 25, 22, 18, 14, 18, 0}; @@ -414,7 +414,7 @@ TEST_F(IndexCompactionTest, write_index_test) { // read col v3 const auto& v3_column = _tablet_schema->column_by_uid(3); - const auto* v3_index = _tablet_schema->get_inverted_index(v3_column); + const auto* v3_index = _tablet_schema->inverted_index(v3_column); EXPECT_TRUE(v3_index != nullptr); std::vector query_data3 {99, 66, 56, 87, 85, 96, 10000}; std::vector query_result3 {12, 20, 25, 23, 16, 24, 0}; @@ -422,7 +422,7 @@ TEST_F(IndexCompactionTest, write_index_test) { // read col v1 const auto& v1_column = _tablet_schema->column_by_uid(1); - const auto* v1_index = _tablet_schema->get_inverted_index(v1_column); + const auto* v1_index = _tablet_schema->inverted_index(v1_column); EXPECT_TRUE(v1_index != nullptr); std::vector query_data1 {"good", "maybe", "great", "null"}; std::vector query_result1 {197, 191, 194, 0}; @@ -431,7 +431,7 @@ TEST_F(IndexCompactionTest, write_index_test) { // read col v2 const auto& v2_column = _tablet_schema->column_by_uid(2); - const auto* v2_index = _tablet_schema->get_inverted_index(v2_column); + const auto* v2_index = _tablet_schema->inverted_index(v2_column); EXPECT_TRUE(v2_index != nullptr); std::vector query_data2 {"musicstream.com", "http", "https", "null"}; std::vector query_result2 {191, 799, 1201, 0}; diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp new file mode 100644 index 00000000000000..321d43fa87206c --- /dev/null +++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_with_deleted_term.cpp @@ -0,0 +1,671 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "CLucene/StdHeader.h" +#include "CLucene/config/repl_wchar.h" +#include "json2pb/json_to_pb.h" +#include "json2pb/pb_to_json.h" +#include "olap/base_compaction.h" +#include "olap/rowset/beta_rowset.h" +#include "olap/rowset/beta_rowset_writer.h" +#include "olap/rowset/rowset_factory.h" +#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" +#include "olap/storage_engine.h" + +namespace doris { + +using namespace doris::vectorized; + +constexpr static uint32_t MAX_PATH_LEN = 1024; +constexpr static std::string_view dest_dir = "/ut_dir/inverted_index_test"; +constexpr static std::string_view tmp_dir = "./ut_dir/tmp"; +static int64_t inc_id = 1000; + +struct DataRow { + int key; + std::string word; + std::string url; + int num; +}; + +static std::vector read_data(const std::string file_name) { + std::ifstream file(file_name); + EXPECT_TRUE(file.is_open()); + + std::string line; + std::vector data; + + while (std::getline(file, line)) { + std::stringstream ss(line); + std::string item; + DataRow row; + EXPECT_TRUE(std::getline(ss, item, ',')); + row.key = std::stoi(item); + EXPECT_TRUE(std::getline(ss, item, ',')); + row.word = item; + EXPECT_TRUE(std::getline(ss, item, ',')); + row.url = item; + EXPECT_TRUE(std::getline(ss, item, ',')); + row.num = std::stoi(item); + data.emplace_back(std::move(row)); + } + + file.close(); + return data; +} + +static bool query_bkd(const TabletIndex* index, + std::shared_ptr& inverted_index_file_reader, + const std::vector& query_data, const std::vector& query_result) { + const auto& idx_reader = BkdIndexReader::create_shared(index, inverted_index_file_reader); + const auto& index_searcher_builder = std::make_unique(); + auto dir = inverted_index_file_reader->open(index); + EXPECT_TRUE(dir.has_value()); + auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release()); + EXPECT_TRUE(searcher_result.has_value()); + auto bkd_searcher = std::get_if(&searcher_result.value()); + EXPECT_TRUE(bkd_searcher != nullptr); + idx_reader->_type_info = get_scalar_type_info((FieldType)(*bkd_searcher)->type); + EXPECT_TRUE(idx_reader->_type_info != nullptr); + idx_reader->_value_key_coder = get_key_coder(idx_reader->_type_info->type()); + + for (int i = 0; i < query_data.size(); i++) { + vectorized::Field param_value = Int32(query_data[i]); + std::unique_ptr query_param = nullptr; + EXPECT_TRUE(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + PrimitiveType::TYPE_INT, ¶m_value, query_param) + .ok()); + auto result = std::make_shared(); + EXPECT_TRUE(idx_reader + ->invoke_bkd_query(query_param->get_value(), + InvertedIndexQueryType::EQUAL_QUERY, *bkd_searcher, + result) + .ok()); + EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i]; + } + return true; +} + +static bool query_string(const TabletIndex* index, + std::shared_ptr& inverted_index_file_reader, + const std::string& column_name, const std::vector& query_data, + const std::vector& query_result) { + const auto& idx_reader = + StringTypeInvertedIndexReader::create_shared(index, inverted_index_file_reader); + const auto& index_searcher_builder = std::make_unique(); + auto dir = inverted_index_file_reader->open(index); + EXPECT_TRUE(dir.has_value()); + auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release()); + EXPECT_TRUE(searcher_result.has_value()); + auto string_searcher = std::get_if(&searcher_result.value()); + EXPECT_TRUE(string_searcher != nullptr); + std::wstring column_name_ws = StringUtil::string_to_wstring(column_name); + + for (int i = 0; i < query_data.size(); i++) { + TQueryOptions queryOptions; + auto query = QueryFactory::create(InvertedIndexQueryType::EQUAL_QUERY, *string_searcher, + queryOptions); + EXPECT_TRUE(query != nullptr); + InvertedIndexQueryInfo query_info; + query_info.field_name = column_name_ws; + query_info.terms.emplace_back(query_data[i]); + query->add(query_info); + auto result = std::make_shared(); + query->search(*result); + EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i]; + } + return true; +} + +static bool query_fulltext(const TabletIndex* index, + std::shared_ptr& inverted_index_file_reader, + const std::string& column_name, + const std::vector& query_data, + const std::vector& query_result) { + const auto& idx_reader = FullTextIndexReader::create_shared(index, inverted_index_file_reader); + const auto& index_searcher_builder = std::make_unique(); + auto dir = inverted_index_file_reader->open(index); + EXPECT_TRUE(dir.has_value()); + auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release()); + EXPECT_TRUE(searcher_result.has_value()); + auto string_searcher = std::get_if(&searcher_result.value()); + EXPECT_TRUE(string_searcher != nullptr); + std::wstring column_name_ws = StringUtil::string_to_wstring(column_name); + + for (int i = 0; i < query_data.size(); i++) { + TQueryOptions queryOptions; + auto query = QueryFactory::create(InvertedIndexQueryType::MATCH_ANY_QUERY, *string_searcher, + queryOptions); + EXPECT_TRUE(query != nullptr); + InvertedIndexQueryInfo query_info; + query_info.field_name = column_name_ws; + query_info.terms.emplace_back(query_data[i]); + query->add(query_info); + auto result = std::make_shared(); + query->search(*result); + EXPECT_EQ(query_result[i], result->cardinality()) << query_data[i]; + } + return true; +} + +static void check_terms_stats(lucene::store::Directory* dir) { + IndexReader* r = IndexReader::open(dir); + + printf("Max Docs: %d\n", r->maxDoc()); + printf("Num Docs: %d\n", r->numDocs()); + + int64_t ver = r->getCurrentVersion(dir); + printf("Current Version: %f\n", (float_t)ver); + + TermEnum* te = r->terms(); + int32_t nterms; + for (nterms = 0; te->next(); nterms++) { + /* empty */ + std::string token = + lucene_wcstoutf8string(te->term(false)->text(), te->term(false)->textLength()); + std::string field = lucene_wcstoutf8string(te->term(false)->field(), + lenOfString(te->term(false)->field())); + + printf("Field: %s ", field.c_str()); + printf("Term: %s ", token.c_str()); + printf("Freq: %d\n", te->docFreq()); + if (false) { + TermDocs* td = r->termDocs(te->term()); + while (td->next()) { + printf("DocID: %d ", td->doc()); + printf("TermFreq: %d\n", td->freq()); + } + _CLLDELETE(td); + } + } + printf("Term count: %d\n\n", nterms); + te->close(); + _CLLDELETE(te); + + r->close(); + _CLLDELETE(r); +} +static Status check_idx_file_correctness(lucene::store::Directory* index_reader, + lucene::store::Directory* tmp_index_reader) { + lucene::index::IndexReader* idx_reader = lucene::index::IndexReader::open(index_reader); + lucene::index::IndexReader* tmp_idx_reader = lucene::index::IndexReader::open(tmp_index_reader); + + // compare numDocs + if (idx_reader->numDocs() != tmp_idx_reader->numDocs()) { + return Status::InternalError( + "index compaction correctness check failed, numDocs not equal, idx_numDocs={}, " + "tmp_idx_numDocs={}", + idx_reader->numDocs(), tmp_idx_reader->numDocs()); + } + + lucene::index::TermEnum* term_enum = idx_reader->terms(); + lucene::index::TermEnum* tmp_term_enum = tmp_idx_reader->terms(); + lucene::index::TermDocs* term_docs = nullptr; + lucene::index::TermDocs* tmp_term_docs = nullptr; + + // iterate TermEnum + while (term_enum->next() && tmp_term_enum->next()) { + std::string token = lucene_wcstoutf8string(term_enum->term(false)->text(), + term_enum->term(false)->textLength()); + std::string field = lucene_wcstoutf8string(term_enum->term(false)->field(), + lenOfString(term_enum->term(false)->field())); + std::string tmp_token = lucene_wcstoutf8string(tmp_term_enum->term(false)->text(), + tmp_term_enum->term(false)->textLength()); + std::string tmp_field = + lucene_wcstoutf8string(tmp_term_enum->term(false)->field(), + lenOfString(tmp_term_enum->term(false)->field())); + // compare token and field + if (field != tmp_field) { + return Status::InternalError( + "index compaction correctness check failed, fields not equal, field={}, " + "tmp_field={}", + field, field); + } + if (token != tmp_token) { + return Status::InternalError( + "index compaction correctness check failed, tokens not equal, token={}, " + "tmp_token={}", + token, tmp_token); + } + + // get term's docId and freq + term_docs = idx_reader->termDocs(term_enum->term(false)); + tmp_term_docs = tmp_idx_reader->termDocs(tmp_term_enum->term(false)); + + // compare term's docId and freq + while (term_docs->next() && tmp_term_docs->next()) { + if (term_docs->doc() != tmp_term_docs->doc() || + term_docs->freq() != tmp_term_docs->freq()) { + return Status::InternalError( + "index compaction correctness check failed, docId or freq not equal, " + "docId={}, tmp_docId={}, freq={}, tmp_freq={}", + term_docs->doc(), tmp_term_docs->doc(), term_docs->freq(), + tmp_term_docs->freq()); + } + } + + // check if there are remaining docs + if (term_docs->next() || tmp_term_docs->next()) { + return Status::InternalError( + "index compaction correctness check failed, number of docs not equal for " + "term={}, tmp_term={}", + token, tmp_token); + } + if (term_docs) { + term_docs->close(); + _CLLDELETE(term_docs); + } + if (tmp_term_docs) { + tmp_term_docs->close(); + _CLLDELETE(tmp_term_docs); + } + } + + // check if there are remaining terms + if (term_enum->next() || tmp_term_enum->next()) { + return Status::InternalError( + "index compaction correctness check failed, number of terms not equal"); + } + if (term_enum) { + term_enum->close(); + _CLLDELETE(term_enum); + } + if (tmp_term_enum) { + tmp_term_enum->close(); + _CLLDELETE(tmp_term_enum); + } + if (idx_reader) { + idx_reader->close(); + _CLLDELETE(idx_reader); + } + if (tmp_idx_reader) { + tmp_idx_reader->close(); + _CLLDELETE(tmp_idx_reader); + } + return Status::OK(); +} + +static RowsetSharedPtr do_compaction(std::vector rowsets, + StorageEngine* engine_ref, TabletSharedPtr tablet, + bool is_index_compaction) { + config::inverted_index_compaction_enable = is_index_compaction; + // only base compaction can handle delete predicate + BaseCompaction compaction(*engine_ref, tablet); + compaction._input_rowsets = std::move(rowsets); + compaction.build_basic_info(); + + std::vector input_rs_readers; + input_rs_readers.reserve(compaction._input_rowsets.size()); + for (auto& rowset : compaction._input_rowsets) { + RowsetReaderSharedPtr rs_reader; + EXPECT_TRUE(rowset->create_reader(&rs_reader).ok()); + input_rs_readers.push_back(std::move(rs_reader)); + } + + RowsetWriterContext ctx; + EXPECT_TRUE(compaction.construct_output_rowset_writer(ctx).ok()); + + if (is_index_compaction) { + EXPECT_TRUE(ctx.columns_to_do_index_compaction.size() == 2); + // col v1 + EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(1)); + // col v2 + EXPECT_TRUE(ctx.columns_to_do_index_compaction.contains(2)); + } + + compaction._stats.rowid_conversion = compaction._rowid_conversion.get(); + EXPECT_TRUE(Merger::vertical_merge_rowsets(tablet, compaction.compaction_type(), + *(compaction._cur_tablet_schema), input_rs_readers, + compaction._output_rs_writer.get(), 100000, 5, + &compaction._stats) + .ok()); + const auto& dst_writer = + dynamic_cast(compaction._output_rs_writer.get()); + for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) { + EXPECT_FALSE(idx_file_writer->_closed); + } + Status st = compaction.do_inverted_index_compaction(); + EXPECT_TRUE(st.ok()) << st.to_string(); + + st = compaction._output_rs_writer->build(compaction._output_rowset); + EXPECT_TRUE(st.ok()) << st.to_string(); + + for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) { + EXPECT_TRUE(idx_file_writer->_closed); + } + EXPECT_TRUE(compaction._output_rowset->num_segments() == 1); + + return compaction._output_rowset; +} + +class IndexCompactionDeleteTest : public ::testing::Test { +protected: + void SetUp() override { + // absolute dir + char buffer[MAX_PATH_LEN]; + EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); + _curreent_dir = std::string(buffer); + _absolute_dir = _curreent_dir + std::string(dest_dir); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(_absolute_dir).ok()); + + // tmp dir + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok()); + std::vector paths; + paths.emplace_back(std::string(tmp_dir), 1024000000); + auto tmp_file_dirs = std::make_unique(paths); + Status st = tmp_file_dirs->init(); + EXPECT_TRUE(st.ok()) << st.to_json(); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + + // storage engine + doris::EngineOptions options; + auto engine = std::make_unique(options); + _engine_ref = engine.get(); + _data_dir = std::make_unique(*_engine_ref, _absolute_dir); + static_cast(_data_dir->update_capacity()); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + + // tablet_schema + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0, + "INT", "key"); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, + "STRING", "v1"); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10002, "v2_index", 2, + "STRING", "v2", true); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10003, "v3_index", 3, "INT", + "v3"); + + _tablet_schema.reset(new TabletSchema); + _tablet_schema->init_from_pb(schema_pb); + + // tablet + TabletMetaSharedPtr tablet_meta(new TabletMeta(_tablet_schema)); + + _tablet.reset(new Tablet(*_engine_ref, tablet_meta, _data_dir.get())); + EXPECT_TRUE(_tablet->init().ok()); + } + void TearDown() override { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_absolute_dir).ok()); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); + _engine_ref = nullptr; + ExecEnv::GetInstance()->set_storage_engine(nullptr); + } + + void init_rs_meta(RowsetMetaSharedPtr& rs_meta, int64_t start, int64_t end) { + std::string json_rowset_meta = R"({ + "rowset_id": 540081, + "tablet_id": 15673, + "partition_id": 10000, + "tablet_schema_hash": 567997577, + "rowset_type": "BETA_ROWSET", + "rowset_state": "VISIBLE", + "empty": false + })"; + RowsetMetaPB rowset_meta_pb; + json2pb::JsonToProtoMessage(json_rowset_meta, &rowset_meta_pb); + rowset_meta_pb.set_start_version(start); + rowset_meta_pb.set_end_version(end); + rs_meta->init_from_pb(rowset_meta_pb); + } + + RowsetSharedPtr create_delete_predicate_rowset(const TabletSchemaSPtr& schema, std::string pred, + int64_t version) { + DeletePredicatePB del_pred; + del_pred.add_sub_predicates(pred); + del_pred.set_version(1); + RowsetMetaSharedPtr rsm(new RowsetMeta()); + init_rs_meta(rsm, version, version); + RowsetId id; + id.init(version); + rsm->set_rowset_id(id); + rsm->set_delete_predicate(std::move(del_pred)); + rsm->set_tablet_schema(schema); + return std::make_shared(schema, rsm, ""); + } + + void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id, + const std::string& index_name, int32_t col_unique_id, + const std::string& column_type, const std::string& column_name, + bool parser = false) { + column_pb->set_unique_id(col_unique_id); + column_pb->set_name(column_name); + column_pb->set_type(column_type); + column_pb->set_is_key(false); + column_pb->set_is_nullable(true); + tablet_index->set_index_id(index_id); + tablet_index->set_index_name(index_name); + tablet_index->set_index_type(IndexType::INVERTED); + tablet_index->add_col_unique_id(col_unique_id); + if (parser) { + auto* properties = tablet_index->mutable_properties(); + (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_UNICODE; + } + } + + void check_meta_and_file(RowsetSharedPtr output_rowset) { + CHECK_EQ(output_rowset->num_segments(), 1); + // check rowset meta and file + int seg_id = 0; + // meta + const auto& index_info = output_rowset->_rowset_meta->inverted_index_file_info(seg_id); + EXPECT_TRUE(index_info.has_index_size()); + const auto& fs = output_rowset->_rowset_meta->fs(); + const auto& file_name = fmt::format("{}/{}_{}.idx", output_rowset->tablet_path(), + output_rowset->rowset_id().to_string(), seg_id); + int64_t file_size = 0; + EXPECT_TRUE(fs->file_size(file_name, &file_size).ok()); + EXPECT_EQ(index_info.index_size(), file_size); + + // file + const auto& seg_path = output_rowset->segment_path(seg_id); + EXPECT_TRUE(seg_path.has_value()); + const auto& index_file_path_prefix = + InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); + auto inverted_index_file_reader = std::make_shared( + fs, std::string(index_file_path_prefix), + _tablet_schema->get_inverted_index_storage_format(), index_info); + EXPECT_TRUE(inverted_index_file_reader->init().ok()); + const auto& dirs = inverted_index_file_reader->get_all_directories(); + EXPECT_TRUE(dirs.has_value()); + EXPECT_EQ(dirs.value().size(), 4); + + // read col key + const auto& key = _tablet_schema->column_by_uid(0); + const auto* key_index = _tablet_schema->inverted_index(key); + EXPECT_TRUE(key_index != nullptr); + std::vector query_data {99, 66, 56, 87, 85, 96, 20000}; + std::vector query_result {19, 21, 21, 16, 14, 18, 0}; + EXPECT_TRUE(query_bkd(key_index, inverted_index_file_reader, query_data, query_result)); + + // read col v3 + const auto& v3_column = _tablet_schema->column_by_uid(3); + const auto* v3_index = _tablet_schema->inverted_index(v3_column); + EXPECT_TRUE(v3_index != nullptr); + std::vector query_data3 {99, 66, 56, 87, 85, 96, 10000}; + std::vector query_result3 {12, 18, 22, 21, 16, 20, 0}; + EXPECT_TRUE(query_bkd(v3_index, inverted_index_file_reader, query_data3, query_result3)); + + // read col v1 + const auto& v1_column = _tablet_schema->column_by_uid(1); + const auto* v1_index = _tablet_schema->inverted_index(v1_column); + EXPECT_TRUE(v1_index != nullptr); + std::vector query_data1 {"good", "maybe", "great", "null"}; + std::vector query_result1 {197, 191, 0, 0}; + EXPECT_TRUE(query_string(v1_index, inverted_index_file_reader, "1", query_data1, + query_result1)); + + // read col v2 + const auto& v2_column = _tablet_schema->column_by_uid(2); + const auto* v2_index = _tablet_schema->inverted_index(v2_column); + EXPECT_TRUE(v2_index != nullptr); + std::vector query_data2 {"musicstream.com", "http", "https", "null"}; + std::vector query_result2 {176, 719, 1087, 0}; + EXPECT_TRUE(query_fulltext(v2_index, inverted_index_file_reader, "2", query_data2, + query_result2)); + } + + RowsetWriterContext rowset_writer_context() { + RowsetWriterContext context; + RowsetId rowset_id; + rowset_id.init(inc_id); + context.rowset_id = rowset_id; + context.rowset_type = BETA_ROWSET; + context.data_dir = _data_dir.get(); + context.rowset_state = VISIBLE; + context.tablet_schema = _tablet_schema; + context.tablet_path = _tablet->tablet_path(); + context.version = Version(inc_id, inc_id); + context.max_rows_per_segment = 200; + inc_id++; + return context; + } + + IndexCompactionDeleteTest() = default; + ~IndexCompactionDeleteTest() override = default; + +private: + TabletSchemaSPtr _tablet_schema = nullptr; + StorageEngine* _engine_ref = nullptr; + std::unique_ptr _data_dir = nullptr; + TabletSharedPtr _tablet = nullptr; + std::string _absolute_dir; + std::string _curreent_dir; +}; + +TEST_F(IndexCompactionDeleteTest, delete_index_test) { + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_tablet->tablet_path()).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(_tablet->tablet_path()).ok()); + std::string data_file1 = + _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data1.csv"; + std::string data_file2 = + _curreent_dir + "/be/test/olap/rowset/segment_v2/inverted_index/data/data2.csv"; + + std::vector> data; + data.emplace_back(read_data(data_file1)); + data.emplace_back(read_data(data_file2)); + + std::vector rowsets(data.size()); + for (int i = 0; i < data.size(); i++) { + const auto& res = + RowsetFactory::create_rowset_writer(*_engine_ref, rowset_writer_context(), false); + EXPECT_TRUE(res.has_value()) << res.error(); + const auto& rowset_writer = res.value(); + + Block block = _tablet_schema->create_block(); + auto columns = block.mutate_columns(); + for (const auto& row : data[i]) { + vectorized::Field key = Int32(row.key); + vectorized::Field v1 = row.word; + vectorized::Field v2 = row.url; + vectorized::Field v3 = Int32(row.num); + columns[0]->insert(key); + columns[1]->insert(v1); + columns[2]->insert(v2); + columns[3]->insert(v3); + } + EXPECT_TRUE(rowset_writer->add_block(&block).ok()); + EXPECT_TRUE(rowset_writer->flush().ok()); + const auto& dst_writer = dynamic_cast(rowset_writer.get()); + + // inverted index file writer + for (const auto& [seg_id, idx_file_writer] : dst_writer->_idx_files.get_file_writers()) { + EXPECT_TRUE(idx_file_writer->_closed); + } + + EXPECT_TRUE(rowset_writer->build(rowsets[i]).ok()); + EXPECT_TRUE(_tablet->add_rowset(rowsets[i]).ok()); + EXPECT_TRUE(rowsets[i]->num_segments() == 5); + + // check rowset meta and file + for (int seg_id = 0; seg_id < rowsets[i]->num_segments(); seg_id++) { + const auto& index_info = rowsets[i]->_rowset_meta->inverted_index_file_info(seg_id); + EXPECT_TRUE(index_info.has_index_size()); + const auto& fs = rowsets[i]->_rowset_meta->fs(); + const auto& file_name = fmt::format("{}/{}_{}.idx", rowsets[i]->tablet_path(), + rowsets[i]->rowset_id().to_string(), seg_id); + int64_t file_size = 0; + EXPECT_TRUE(fs->file_size(file_name, &file_size).ok()); + EXPECT_EQ(index_info.index_size(), file_size); + + const auto& seg_path = rowsets[i]->segment_path(seg_id); + EXPECT_TRUE(seg_path.has_value()); + const auto& index_file_path_prefix = + InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); + auto inverted_index_file_reader = std::make_shared( + fs, std::string(index_file_path_prefix), + _tablet_schema->get_inverted_index_storage_format(), index_info); + EXPECT_TRUE(inverted_index_file_reader->init().ok()); + const auto& dirs = inverted_index_file_reader->get_all_directories(); + EXPECT_TRUE(dirs.has_value()); + EXPECT_EQ(dirs.value().size(), 4); + } + } + + // create delete predicate rowset and add to tablet + auto delete_rowset = create_delete_predicate_rowset(_tablet_schema, "v1='great'", inc_id++); + EXPECT_TRUE(_tablet->add_rowset(delete_rowset).ok()); + EXPECT_TRUE(_tablet->rowset_map().size() == 3); + rowsets.push_back(delete_rowset); + EXPECT_TRUE(rowsets.size() == 3); + + auto output_rowset_index = do_compaction(rowsets, _engine_ref, _tablet, true); + const auto& seg_path = output_rowset_index->segment_path(0); + EXPECT_TRUE(seg_path.has_value()); + const auto& index_file_path_prefix = + InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value()); + auto inverted_index_file_reader_index = std::make_shared( + output_rowset_index->_rowset_meta->fs(), std::string(index_file_path_prefix), + _tablet_schema->get_inverted_index_storage_format()); + EXPECT_TRUE(inverted_index_file_reader_index->init().ok()); + + auto output_rowset_normal = do_compaction(rowsets, _engine_ref, _tablet, false); + const auto& seg_path_normal = output_rowset_normal->segment_path(0); + EXPECT_TRUE(seg_path_normal.has_value()); + const auto& index_file_path_prefix_normal = + InvertedIndexDescriptor::get_index_file_path_prefix(seg_path_normal.value()); + auto inverted_index_file_reader_normal = std::make_shared( + output_rowset_normal->_rowset_meta->fs(), std::string(index_file_path_prefix_normal), + _tablet_schema->get_inverted_index_storage_format()); + EXPECT_TRUE(inverted_index_file_reader_normal->init().ok()); + + // check index file terms + auto dir_idx_compaction = inverted_index_file_reader_index->_open(10001, ""); + auto dir_normal_compaction = inverted_index_file_reader_normal->_open(10001, ""); + check_terms_stats(dir_idx_compaction->get()); + check_terms_stats(dir_normal_compaction->get()); + auto st = check_idx_file_correctness(dir_idx_compaction->get(), dir_normal_compaction->get()); + EXPECT_TRUE(st.ok()) << st.to_string(); + + // check meta and file + check_meta_and_file(output_rowset_index); + check_meta_and_file(output_rowset_normal); +} + +} // namespace doris diff --git a/be/test/olap/tablet_index_test.cpp b/be/test/olap/tablet_index_test.cpp new file mode 100644 index 00000000000000..7842f9af18d51d --- /dev/null +++ b/be/test/olap/tablet_index_test.cpp @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "olap/tablet_schema.h" +#include "vec/common/schema_util.h" + +namespace doris { + +class TabletIndexTest : public testing::Test {}; + +void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id, + const std::string& index_name, int32_t col_unique_id, + const std::string& column_type, const std::string& column_name, + const IndexType& index_type, bool is_bf_column) { + column_pb->set_unique_id(col_unique_id); + column_pb->set_name(column_name); + column_pb->set_type(column_type); + column_pb->set_is_nullable(true); + column_pb->set_is_bf_column(is_bf_column); + tablet_index->set_index_id(index_id); + tablet_index->set_index_name(index_name); + tablet_index->set_index_type(index_type); + tablet_index->add_col_unique_id(col_unique_id); + if (index_type == IndexType::NGRAM_BF) { + auto* properties = tablet_index->mutable_properties(); + (*properties)["gram_size"] = "5"; + (*properties)["bf_size"] = "1024"; + } +} + +TEST_F(TabletIndexTest, test_inverted_index) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0, "INT", + "key", IndexType::INVERTED, true); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, "STRING", + "v1", IndexType::INVERTED, false); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10002, "v2_index", 2, "STRING", + "v2", IndexType::NGRAM_BF, true); + + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + EXPECT_TRUE(tablet_schema->has_inverted_index()); + EXPECT_EQ(tablet_schema->inverted_indexes().size(), 2); + EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(0)) != nullptr); + EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(1)) != nullptr); + EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(2)) == nullptr); + EXPECT_TRUE(tablet_schema->inverted_index(3) == nullptr); + EXPECT_TRUE(tablet_schema->inverted_index(4, "v1.a") == nullptr); +} + +TEST_F(TabletIndexTest, test_schema_index_diff) { + TabletSchemaPB new_schema_pb; + new_schema_pb.set_keys_type(KeysType::DUP_KEYS); + new_schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + + construct_column(new_schema_pb.add_column(), new_schema_pb.add_index(), 10000, "key_index", 0, + "INT", "key", IndexType::INVERTED, true); + construct_column(new_schema_pb.add_column(), new_schema_pb.add_index(), 10001, "v1_index", 1, + "STRING", "v1", IndexType::INVERTED, false); + construct_column(new_schema_pb.add_column(), new_schema_pb.add_index(), 10002, "v2_index", 2, + "STRING", "v2", IndexType::NGRAM_BF, true); + + TabletSchemaSPtr new_tablet_schema = std::make_shared(); + new_tablet_schema->init_from_pb(new_schema_pb); + + TabletSchemaPB old_schema_pb; + old_schema_pb.set_keys_type(KeysType::DUP_KEYS); + old_schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + + construct_column(old_schema_pb.add_column(), old_schema_pb.add_index(), 10000, "key_index", 0, + "INT", "key", IndexType::INVERTED, true); + construct_column(old_schema_pb.add_column(), old_schema_pb.add_index(), 10001, "v1_index", 1, + "STRING", "v1", IndexType::INVERTED, true); + construct_column(old_schema_pb.add_column(), old_schema_pb.add_index(), 10002, "v2_index", 2, + "STRING", "v2", IndexType::INVERTED, true); + + TabletSchemaSPtr old_tablet_schema = std::make_shared(); + old_tablet_schema->init_from_pb(old_schema_pb); + + EXPECT_FALSE(vectorized::schema_util::has_schema_index_diff(new_tablet_schema.get(), + old_tablet_schema.get(), 0, 0)); + EXPECT_TRUE(vectorized::schema_util::has_schema_index_diff(new_tablet_schema.get(), + old_tablet_schema.get(), 1, 1)); + EXPECT_TRUE(vectorized::schema_util::has_schema_index_diff(new_tablet_schema.get(), + old_tablet_schema.get(), 2, 2)); +} + +} // namespace doris diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index 5207279a291f0f..59933db80e5bb9 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -31,6 +31,7 @@ #include "olap/page_cache.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" +#include "olap/tablet_column_object_pool.h" #include "olap/tablet_schema_cache.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" @@ -66,6 +67,9 @@ int main(int argc, char** argv) { doris::ExecEnv::GetInstance()->set_tablet_schema_cache( doris::TabletSchemaCache::create_global_schema_cache( doris::config::tablet_schema_cache_capacity)); + doris::ExecEnv::GetInstance()->set_tablet_column_object_pool( + doris::TabletColumnObjectPool::create_global_column_cache( + doris::config::tablet_schema_cache_capacity)); LOG(INFO) << "init config " << st; doris::Status s = doris::config::set_config("enable_stacktrace", "false"); if (!s.ok()) { diff --git a/be/test/vec/jsonb/serialize_test.cpp b/be/test/vec/jsonb/serialize_test.cpp index 3845c689e1e381..82d8c4f394ab5a 100644 --- a/be/test/vec/jsonb/serialize_test.cpp +++ b/be/test/vec/jsonb/serialize_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,8 @@ #include #include +#include "agent/be_exec_version_manager.h" +#include "common/exception.h" #include "gen_cpp/descriptors.pb.h" #include "gtest/gtest_pred_impl.h" #include "olap/hll.h" @@ -263,6 +266,20 @@ TEST(BlockSerializeTest, Map) { EXPECT_EQ(block.dump_data(), new_block.dump_data()); } +TEST(BlockSerializeTest, Bigstr) { + DataTypePtr s = std::make_shared(); + MutableColumnPtr col = ColumnString::create(); + std::string bigdata; + bigdata.resize(std::numeric_limits::max() - 5); + col->insert_data(bigdata.data(), bigdata.length()); + try { + s->get_uncompressed_serialized_bytes(*col, BeExecVersionManager::get_newest_version()); + } catch (std::exception e) { + return; + } + assert(false); +} + TEST(BlockSerializeTest, Struct) { TabletSchema schema; TabletColumn struct_col; diff --git a/cloud/conf/doris_cloud.conf b/cloud/conf/doris_cloud.conf index ed52fda56779b6..f8b50d1a924886 100644 --- a/cloud/conf/doris_cloud.conf +++ b/cloud/conf/doris_cloud.conf @@ -15,33 +15,18 @@ # specific language governing permissions and limitations # under the License. -# // meta_service brpc_listen_port = 5000 -brpc_num_threads = -1 -brpc_idle_timeout_sec = 30 fdb_cluster = xxx:yyy@127.0.0.1:4500 -fdb_cluster_file_path = ./conf/fdb.cluster +# http request to meta-service needs this token for auth http_token = greedisgood9999 +# use CIDR to mask if there are multiple NICs +# priority_networks = 192.168.0.1/24 -# // doris txn config -label_keep_max_second = 259200 -expired_txn_scan_key_nums = 1000 - -# // logging +# logging log_dir = ./log/ -# info warn error +# candidates: info warn error log_level = info log_size_mb = 1024 log_filenum_quota = 10 -log_immediate_flush = false # log_verbose_modules = * -# // recycler config -recycle_interval_seconds = 3600 -retention_seconds = 259200 -recycle_concurrency = 16 -# recycle_whitelist = -# recycle_blacklist = - -# //max stage num -max_num_stages = 40 diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index da6ae113cd7349..04b0b4a3382671 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -22,7 +22,8 @@ namespace doris::cloud::config { CONF_Int32(brpc_listen_port, "5000"); -CONF_Int32(brpc_num_threads, "-1"); +CONF_Int32(brpc_num_threads, "64"); +// connections without data transmission for so many seconds will be closed // Set -1 to disable it. CONF_Int32(brpc_idle_timeout_sec, "-1"); CONF_String(hostname, ""); @@ -65,10 +66,10 @@ CONF_mInt64(dropped_partition_retention_seconds, "10800"); // 3h CONF_Strings(recycle_whitelist, ""); // Comma seprated list // These instances will not be recycled, only effective when whitelist is empty. CONF_Strings(recycle_blacklist, ""); // Comma seprated list -CONF_mInt32(instance_recycler_worker_pool_size, "8"); +CONF_mInt32(instance_recycler_worker_pool_size, "32"); CONF_Bool(enable_checker, "false"); // The parallelism for parallel recycle operation -CONF_Int32(recycle_pool_parallelism, "10"); +CONF_Int32(recycle_pool_parallelism, "40"); // Currently only used for recycler test CONF_Bool(enable_inverted_check, "false"); // interval for scanning instances to do checks and inspections diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index dfa06ca5fa08dc..876a8817b85f0e 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1782,6 +1782,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont // lock_id > 0 : load // lock_id = -1 : compaction // lock_id = -2 : schema change + // lock_id = -3 : compaction update delete bitmap without lock if (request->lock_id() > 0) { std::string pending_val; if (!delete_bitmap_keys.SerializeToString(&pending_val)) { @@ -1794,6 +1795,15 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont fdb_txn_size = fdb_txn_size + pending_key.size() + pending_val.size(); LOG(INFO) << "xxx update delete bitmap put pending_key=" << hex(pending_key) << " lock_id=" << request->lock_id() << " value_size: " << pending_val.size(); + } else if (request->lock_id() == -3) { + // delete existing key + for (size_t i = 0; i < request->rowset_ids_size(); ++i) { + auto& start_key = delete_bitmap_keys.delete_bitmap_keys(i); + std::string end_key {start_key}; + encode_int64(INT64_MAX, &end_key); + txn->remove(start_key, end_key); + LOG(INFO) << "xxx remove existing key=" << hex(start_key) << " tablet_id=" << tablet_id; + } } // 4. Update delete bitmap for curent txn @@ -1838,7 +1848,8 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont total_key++; total_size += key.size() + val.size(); VLOG_DEBUG << "xxx update delete bitmap put delete_bitmap_key=" << hex(key) - << " lock_id=" << request->lock_id() << " value_size: " << val.size(); + << " lock_id=" << request->lock_id() << " key_size: " << key.size() + << " value_size: " << val.size(); } err = txn->commit(); @@ -1942,6 +1953,11 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control last_ver = ver; last_seg_id = seg_id; } else { + TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_code", &code); + if (code != MetaServiceCode::OK) { + msg = "test get get_delete_bitmap fail,code=" + MetaServiceCode_Name(code); + return; + } response->mutable_segment_delete_bitmaps()->rbegin()->append(v); } } diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index bb1f6197f1d3b0..9e9ff38c2ecda0 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -92,6 +92,12 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { + if (res->status().code() != MetaServiceCode::OK) { + res->clear_rowset_ids(); + res->clear_segment_ids(); + res->clear_versions(); + res->clear_segment_delete_bitmaps(); + } LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " delete_bitmap_size=" << res->segment_delete_bitmaps_size(); diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 399e0964f4da1d..1f99974360045e 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -701,6 +701,9 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptrset_ak(cipher_ak_sk_pair.first); new_vault.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second); new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info); + if (obj_info.has_use_path_style()) { + new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style()); + } auto new_vault_info = new_vault.DebugString(); val = new_vault.SerializeAsString(); diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index ee90e604e1c5f6..e6b9e9dddeed7c 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -4768,6 +4768,94 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) { ASSERT_EQ(get_delete_bitmap_res.versions(100), 3); ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(100), "abcd4"); } + + // update existing delete bitmap key + { + //first update new key + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(112); + update_delete_bitmap_req.set_partition_id(123); + update_delete_bitmap_req.set_lock_id(888); + update_delete_bitmap_req.set_initiator(-1); + update_delete_bitmap_req.set_tablet_id(333); + std::string large_value = generate_random_string(300 * 1000 * 3); + update_delete_bitmap_req.add_rowset_ids("456"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(2); + update_delete_bitmap_req.add_segment_delete_bitmaps(large_value); + meta_service->update_delete_bitmap( + reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, nullptr); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); + + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(333); + + get_delete_bitmap_req.add_rowset_ids("456"); + get_delete_bitmap_req.add_begin_versions(2); + get_delete_bitmap_req.add_end_versions(2); + + meta_service->get_delete_bitmap(reinterpret_cast(&cntl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + ASSERT_EQ(get_delete_bitmap_res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(get_delete_bitmap_res.rowset_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.versions_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + + ASSERT_EQ(get_delete_bitmap_res.rowset_ids(0), "456"); + ASSERT_EQ(get_delete_bitmap_res.segment_ids(0), 0); + ASSERT_EQ(get_delete_bitmap_res.versions(0), 2); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), large_value); + } + + { + //compaction update delete bitmap without lock + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(112); + update_delete_bitmap_req.set_partition_id(123); + update_delete_bitmap_req.set_unlock(true); + update_delete_bitmap_req.set_lock_id(-3); + update_delete_bitmap_req.set_initiator(-1); + update_delete_bitmap_req.set_tablet_id(333); + std::string large_value = generate_random_string(300 * 1000); + update_delete_bitmap_req.add_rowset_ids("456"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(2); + update_delete_bitmap_req.add_segment_delete_bitmaps(large_value); + meta_service->update_delete_bitmap( + reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, nullptr); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); + + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(333); + + get_delete_bitmap_req.add_rowset_ids("456"); + get_delete_bitmap_req.add_begin_versions(2); + get_delete_bitmap_req.add_end_versions(2); + + meta_service->get_delete_bitmap(reinterpret_cast(&cntl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + ASSERT_EQ(get_delete_bitmap_res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(get_delete_bitmap_res.rowset_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.versions_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + + ASSERT_EQ(get_delete_bitmap_res.rowset_ids(0), "456"); + ASSERT_EQ(get_delete_bitmap_res.segment_ids(0), 0); + ASSERT_EQ(get_delete_bitmap_res.versions(0), 2); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), large_value); + } } TEST(MetaServiceTest, UpdateDeleteBitmapWithException) { @@ -5036,6 +5124,77 @@ TEST(MetaServiceTest, DeleteBimapCommitTxnTest) { } } +TEST(MetaServiceTest, GetDeleteBitmapWithRetryTest) { + auto meta_service = get_meta_service(); + SyncPoint::get_instance()->enable_processing(); + size_t index = 0; + SyncPoint::get_instance()->set_call_back("get_delete_bitmap_code", [&](auto&& args) { + LOG(INFO) << "GET_DELETE_BITMAP_CODE,index=" << index; + if (++index < 2) { + *doris::try_any_cast(args[0]) = MetaServiceCode::KV_TXN_TOO_OLD; + } + }); + + // get delete bitmap update lock + brpc::Controller cntl; + GetDeleteBitmapUpdateLockRequest get_lock_req; + GetDeleteBitmapUpdateLockResponse get_lock_res; + get_lock_req.set_cloud_unique_id("test_cloud_unique_id"); + get_lock_req.set_table_id(100); + get_lock_req.add_partition_ids(123); + get_lock_req.set_expiration(5); + get_lock_req.set_lock_id(888); + get_lock_req.set_initiator(-1); + meta_service->get_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &get_lock_req, + &get_lock_res, nullptr); + ASSERT_EQ(get_lock_res.status().code(), MetaServiceCode::OK); + + //first update new key + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(100); + update_delete_bitmap_req.set_partition_id(123); + update_delete_bitmap_req.set_lock_id(888); + update_delete_bitmap_req.set_initiator(-1); + update_delete_bitmap_req.set_tablet_id(333); + std::string large_value = generate_random_string(300 * 1000 * 3); + update_delete_bitmap_req.add_rowset_ids("456"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(2); + update_delete_bitmap_req.add_segment_delete_bitmaps(large_value); + meta_service->update_delete_bitmap(reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, + nullptr); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); + + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(333); + + get_delete_bitmap_req.add_rowset_ids("456"); + get_delete_bitmap_req.add_begin_versions(2); + get_delete_bitmap_req.add_end_versions(2); + + meta_service->get_delete_bitmap(reinterpret_cast(&cntl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + ASSERT_EQ(get_delete_bitmap_res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(get_delete_bitmap_res.rowset_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.versions_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + + ASSERT_EQ(get_delete_bitmap_res.rowset_ids(0), "456"); + ASSERT_EQ(get_delete_bitmap_res.segment_ids(0), 0); + ASSERT_EQ(get_delete_bitmap_res.versions(0), 2); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), large_value); + + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + TEST(MetaServiceTest, GetVersion) { auto service = get_meta_service(); diff --git a/conf/ldap.conf b/conf/ldap.conf index f783c53ea96da9..b501a729d7e8cc 100644 --- a/conf/ldap.conf +++ b/conf/ldap.conf @@ -30,6 +30,7 @@ # ldap_user_basedn - Search base for users. # ldap_user_filter - User lookup filter, the placeholder {login} will be replaced by the user supplied login. # ldap_group_basedn - Search base for groups. +# ldap_group_filter - Group lookup filter, the placeholder {login} will be replaced by the user supplied login. example : "(&(memberUid={login}))" ## step2: Restart fe, and use root or admin account to log in to doris. ## step3: Execute sql statement to set ldap admin password: # set ldap_admin_password = 'password'; diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run67.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run67.hql new file mode 100644 index 00000000000000..f84cc11f040cda --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run67.hql @@ -0,0 +1,11 @@ +use `default`; + +CREATE TABLE `orc_tiny_stripes`( + col1 bigint, + col2 string, + col3 bigint +) +STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc/orc_tiny_stripes'; + +msck repair table orc_tiny_stripes; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_60_3.orc b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_60_3.orc new file mode 100644 index 00000000000000..34c95840549584 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_60_3.orc differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_6_1.orc b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_6_1.orc new file mode 100644 index 00000000000000..2735240826b242 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/output_6_1.orc differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_60_3.orc b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_60_3.orc new file mode 100644 index 00000000000000..6a3d16f9b445f9 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_60_3.orc differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_6_1.orc b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_6_1.orc new file mode 100644 index 00000000000000..7fce353b526ed5 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc/orc_tiny_stripes/random_output_6_1.orc differ diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 7bd9fa631c8da3..e85d465f66308b 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -42,13 +42,19 @@ public class PaimonJniScanner extends JniScanner { private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class); + @Deprecated private static final String PAIMON_OPTION_PREFIX = "paimon."; + @Deprecated private static final String HADOOP_OPTION_PREFIX = "hadoop."; private final Map params; + @Deprecated private final Map paimonOptionParams; + @Deprecated private final Map hadoopOptionParams; + @Deprecated private final String dbName; + @Deprecated private final String tblName; private final String paimonSplit; private final String paimonPredicate; @@ -58,9 +64,13 @@ public class PaimonJniScanner extends JniScanner { private List paimonAllFieldNames; private List paimonDataTypeList; + @Deprecated private long ctlId; + @Deprecated private long dbId; + @Deprecated private long tblId; + @Deprecated private long lastUpdateTime; private RecordReader.RecordIterator recordIterator = null; private final ClassLoader classLoader; @@ -214,16 +224,20 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { } private void initTable() { - PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId, - paimonOptionParams, hadoopOptionParams, dbName, tblName); - TableExt tableExt = PaimonTableCache.getTable(key); - if (tableExt.getCreateTime() < lastUpdateTime) { - LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}", key, tableExt.getCreateTime(), - lastUpdateTime); - PaimonTableCache.invalidateTableCache(key); - tableExt = PaimonTableCache.getTable(key); + if (params.containsKey("paimon_table")) { + table = PaimonUtils.deserialize(params.get("paimon_table")); + } else { + PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId, + paimonOptionParams, hadoopOptionParams, dbName, tblName); + TableExt tableExt = PaimonTableCache.getTable(key); + if (tableExt.getCreateTime() < lastUpdateTime) { + LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}", key, tableExt.getCreateTime(), + lastUpdateTime); + PaimonTableCache.invalidateTableCache(key); + tableExt = PaimonTableCache.getTable(key); + } + this.table = tableExt.getTable(); } - this.table = tableExt.getTable(); paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index dd0aca5923e74a..e48e4304eafa22 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1208,6 +1208,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int report_queue_size = 100; + // if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc + @ConfField(mutable = true, masterOnly = true, description = { + "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。", + "The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, " + + "the default value is 10000." + }) + public static int report_resend_batch_task_num_per_rpc = 10000; + /** * If set to true, metric collector will be run as a daemon timer to collect metrics at fix interval */ @@ -2622,6 +2630,13 @@ public class Config extends ConfigBase { }) public static boolean enable_java_udf = true; + @ConfField(mutable = true, masterOnly = true, description = { + "开启后，可以在导入时，利用创建的全局java_udf函数处理数据, 默认为false。", + "When enabled, data can be processed using the globally created java_udf function during import." + + " The default setting is false." + }) + public static boolean enable_udf_in_load = false; + @ConfField(description = { "是否忽略 Image 文件中未知的模块。如果为 true，不在 PersistMetaModules.MODULE_NAMES 中的元数据模块将被忽略并跳过。" + "默认为 false，如果 Image 文件中包含未知的模块，Doris 将会抛出异常。" diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java index a6fb10f261d597..f174a4ef663683 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LdapConfig.java @@ -66,6 +66,12 @@ public class LdapConfig extends ConfigBase { @ConfigBase.ConfField public static String ldap_group_basedn = ""; + /** + * Group lookup filter, the placeholder {login} will be replaced by the user supplied login. + */ + @ConfigBase.ConfField + public static String ldap_group_filter = ""; + /** * The user LDAP information cache time. * After timeout, the user information will be retrieved from the LDAP service again. diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ThriftUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/ThriftUtils.java new file mode 100644 index 00000000000000..92881cec5261bf --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ThriftUtils.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.apache.thrift.TBase; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +// Utility functions for thrift +public class ThriftUtils { + // Get the size of the binary message of the thrift object + public static long getBinaryMessageSize(TBase thriftObject) { + TSizeTransport trans = new TSizeTransport(); + TBinaryProtocol protocol = new TBinaryProtocol(trans); + try { + thriftObject.write(protocol); + } catch (TException e) { + return -1; + } + return trans.getSize(); + } + + // A transport class that only records the size of the message + private static class TSizeTransport extends TTransport { + private long size = 0; + + public long getSize() { + return size; + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + size += len; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException { + } + + @Override + public void close() { + } + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + throw new UnsupportedOperationException("Unimplemented method 'read'"); + } + + @Override + public TConfiguration getConfiguration() { + return new TConfiguration(); + } + + @Override + public void updateKnownMessageSize(long size) throws TTransportException { + } + + @Override + public void checkReadBytesAvailable(long numBytes) throws TTransportException { + } + } +} diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 0b60b1ddc51b84..85d5f6f8f9807b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -50,12 +50,13 @@ statementBase | supportedCreateStatement #supportedCreateStatementAlias | supportedAlterStatement #supportedAlterStatementAlias | materializedViewStatement #materializedViewStatementAlias - | supportedJobStatement #supportedJobStatementAlias + | supportedJobStatement #supportedJobStatementAlias | constraintStatement #constraintStatementAlias | supportedDropStatement #supportedDropStatementAlias | supportedSetStatement #supportedSetStatementAlias | supportedUnsetStatement #supportedUnsetStatementAlias | supportedRefreshStatement #supportedRefreshStatementAlias + | supportedShowStatement #supportedShowStatementAlias | unsupportedStatement #unsupported ; @@ -190,6 +191,13 @@ supportedDropStatement : DROP CATALOG RECYCLE BIN WHERE idType=STRING_LITERAL EQ id=INTEGER_VALUE #dropCatalogRecycleBin ; +supportedShowStatement + : SHOW (GLOBAL | SESSION | LOCAL)? VARIABLES wildWhere? #showVariables + | SHOW VIEW + (FROM |IN) tableName=multipartIdentifier + ((FROM | IN) database=identifier)? #showView + ; + unsupportedOtherStatement : HELP mark=identifierOrText #help | INSTALL PLUGIN FROM source=identifierOrText properties=propertyClause? #installPlugin @@ -225,7 +233,6 @@ unsupportedShowStatement | SHOW STORAGE (VAULT | VAULTS) #showStorageVault | SHOW CREATE REPOSITORY FOR identifier #showCreateRepository | SHOW WHITELIST #showWhitelist - | SHOW (GLOBAL | SESSION | LOCAL)? VARIABLES wildWhere? #showVariables | SHOW OPEN TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere? #showOpenTables | SHOW TABLE STATUS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTableStatus | SHOW FULL? TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTables @@ -303,9 +310,6 @@ unsupportedShowStatement | SHOW (KEY | KEYS | INDEX | INDEXES) (FROM |IN) tableName=multipartIdentifier ((FROM | IN) database=multipartIdentifier)? #showIndex - | SHOW VIEW - (FROM |IN) tableName=multipartIdentifier - ((FROM | IN) database=multipartIdentifier)? #showView | SHOW TRANSACTION ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTransaction | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL #showQueryProfile | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL #showLoadProfile diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index b38921f4fc913c..37638319e3f431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -232,7 +232,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getEnableMowLightDelete(), null, tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested(), null); + tbl.variantEnableFlattenNested(), null, + tbl.storagePageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 07b8323951db74..92a74e46b922be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -277,7 +277,8 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getEnableMowLightDelete(), tbl.getInvertedIndexFileStorageFormat(), tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested(), clusterKeyIndexes); + tbl.variantEnableFlattenNested(), clusterKeyIndexes, + tbl.storagePageSize()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 49838446cd5864..8eb87050bf6d77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -270,7 +270,8 @@ protected void createRollupReplica() throws AlterCancelException { tbl.getRowStoreColumnsUniqueIds(tbl.getTableProperty().getCopiedRowStoreColumns()), objectPool, tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested()); + tbl.variantEnableFlattenNested(), + tbl.storagePageSize()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index c8754aa5d626b0..522433dfe898da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -330,7 +330,8 @@ protected void createShadowIndexReplica() throws AlterCancelException { tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested()); + tbl.variantEnableFlattenNested(), + tbl.storagePageSize()); createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId) .get(shadowTabletId), originSchemaHash); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 2a8ae667fdefe0..53bb2ba95acb25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -37,6 +37,7 @@ import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.datasource.InternalCatalog; @@ -2456,7 +2457,11 @@ public Function findUdf(FunctionName fnName, Analyzer analyzer) throws AnalysisE } Function fn = null; - String dbName = fnName.analyzeDb(analyzer); + String dbName = null; + // when enable_udf_in_load == true, and db is null, maybe it's load, should find global function + if (!(Config.enable_udf_in_load && fnName.getDb() == null)) { + dbName = fnName.analyzeDb(analyzer); + } if (!Strings.isNullOrEmpty(dbName)) { // check operation privilege if (!analyzer.isReplay() && !Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyColumnClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyColumnClause.java index eb136c540f2a9e..05a6b23d0d51b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyColumnClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyColumnClause.java @@ -46,6 +46,10 @@ public Column getColumn() { return column; } + public void setColumn(Column column) { + this.column = column; + } + public ColumnPosition getColPos() { return colPos; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 71acd3b217bd04..1fd7480c06c0e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -366,6 +366,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException { this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) { // do nothing, will be analyzed when creating alter job + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) { + throw new AnalysisException("You can not modify storage_page_size"); } else { throw new AnalysisException("Unknown table property: " + properties.keySet()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index e0a47d7db6d769..ff784b4546a609 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -52,6 +52,7 @@ public NullLiteral() { public static NullLiteral create(Type type) { NullLiteral l = new NullLiteral(); l.type = type; + l.analysisDone(); return l; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 7867be0c9f2cbf..09cb50bb9ed72b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -279,7 +279,7 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { return new FloatLiteral(Double.valueOf(value), targetType); } catch (NumberFormatException e) { // consistent with CastExpr's getResultValue() method - return new NullLiteral(); + return NullLiteral.create(targetType); } case DECIMALV2: case DECIMAL32: diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java index a8489361d15b7a..b0da4b6ec277cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java @@ -170,6 +170,8 @@ public void setTypeRead(boolean isTypeRead) { public abstract boolean isCancelled(); + public abstract boolean isFinished(); + public abstract Status updateRepo(Repository repo); public static AbstractJob read(DataInput in) throws IOException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 282b549c5df352..d585176e3d47a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -110,10 +110,10 @@ public class BackupHandler extends MasterDaemon implements Writable { private Env env; - // map to store backup info, key is label name, value is Pair, meta && info is bytes - // this map not present in persist && only in fe master memory + // map to store backup info, key is label name, value is the BackupJob + // this map not present in persist && only in fe memory // one table only keep one snapshot info, only keep last - private final Map localSnapshots = new HashMap<>(); + private final Map localSnapshots = new HashMap<>(); private ReadWriteLock localSnapshotsLock = new ReentrantReadWriteLock(); public BackupHandler() { @@ -168,6 +168,7 @@ private boolean init() { return false; } } + isInit = true; return true; } @@ -558,11 +559,15 @@ private void addBackupOrRestoreJob(long dbId, AbstractJob job) { return; } + List removedLabels = Lists.newArrayList(); jobLock.lock(); try { Deque jobs = dbIdToBackupOrRestoreJobs.computeIfAbsent(dbId, k -> Lists.newLinkedList()); while (jobs.size() >= Config.max_backup_restore_job_num_per_db) { - jobs.removeFirst(); + AbstractJob removedJob = jobs.removeFirst(); + if (removedJob instanceof BackupJob && ((BackupJob) removedJob).isLocalSnapshot()) { + removedLabels.add(removedJob.getLabel()); + } } AbstractJob lastJob = jobs.peekLast(); @@ -575,6 +580,17 @@ private void addBackupOrRestoreJob(long dbId, AbstractJob job) { } finally { jobLock.unlock(); } + + if (job.isFinished() && job instanceof BackupJob) { + // Save snapshot to local repo, when reload backupHandler from image. + BackupJob backupJob = (BackupJob) job; + if (backupJob.isLocalSnapshot()) { + addSnapshot(backupJob.getLabel(), backupJob); + } + } + for (String label : removedLabels) { + removeSnapshot(label); + } } private List getAllCurrentJobs() { @@ -813,22 +829,42 @@ public boolean report(TTaskType type, long jobId, long taskId, int finishedNum, return false; } - public void addSnapshot(String labelName, Snapshot snapshot) { + public void addSnapshot(String labelName, BackupJob backupJob) { + assert backupJob.isFinished(); + + LOG.info("add snapshot {} to local repo", labelName); localSnapshotsLock.writeLock().lock(); try { - localSnapshots.put(labelName, snapshot); + localSnapshots.put(labelName, backupJob); + } finally { + localSnapshotsLock.writeLock().unlock(); + } + } + + public void removeSnapshot(String labelName) { + LOG.info("remove snapshot {} from local repo", labelName); + localSnapshotsLock.writeLock().lock(); + try { + localSnapshots.remove(labelName); } finally { localSnapshotsLock.writeLock().unlock(); } } public Snapshot getSnapshot(String labelName) { + BackupJob backupJob; localSnapshotsLock.readLock().lock(); try { - return localSnapshots.get(labelName); + backupJob = localSnapshots.get(labelName); } finally { localSnapshotsLock.readLock().unlock(); } + + if (backupJob == null) { + return null; + } + + return backupJob.getSnapshot(); } public static BackupHandler read(DataInput in) throws IOException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index ab7bfd8a03f6f7..c7e00f132dc0bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -130,9 +130,6 @@ public enum BackupJobState { @SerializedName("prop") private Map properties = Maps.newHashMap(); - private byte[] metaInfoBytes = null; - private byte[] jobInfoBytes = null; - public BackupJob() { super(JobType.BACKUP); } @@ -344,11 +341,7 @@ public synchronized boolean finishSnapshotUploadTask(UploadTask task, TFinishTas @Override public synchronized void replayRun() { - LOG.info("replay run backup job: {}", this); - if (state == BackupJobState.FINISHED && repoId == Repository.KEEP_ON_LOCAL_REPO_ID) { - Snapshot snapshot = new Snapshot(label, metaInfoBytes, jobInfoBytes); - env.getBackupHandler().addSnapshot(label, snapshot); - } + // nothing to do } @Override @@ -366,6 +359,11 @@ public boolean isCancelled() { return state == BackupJobState.CANCELLED; } + @Override + public boolean isFinished() { + return state == BackupJobState.FINISHED; + } + @Override public synchronized Status updateRepo(Repository repo) { this.repo = repo; @@ -839,8 +837,6 @@ private void saveMetaInfo() { } backupMeta.writeToFile(metaInfoFile); localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); - // read meta info to metaInfoBytes - metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); // 3. save job info file Map tableCommitSeqMap = Maps.newHashMap(); @@ -867,8 +863,6 @@ private void saveMetaInfo() { } jobInfo.writeToFile(jobInfoFile); localJobInfoFilePath = jobInfoFile.getAbsolutePath(); - // read job info to jobInfoBytes - jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath()); } catch (Exception e) { status = new Status(ErrCode.COMMON_ERROR, "failed to save meta info and job info file: " + e.getMessage()); return; @@ -922,7 +916,6 @@ private void uploadMetaAndJobInfoFile() { } } - finishedTime = System.currentTimeMillis(); state = BackupJobState.FINISHED; @@ -931,8 +924,7 @@ private void uploadMetaAndJobInfoFile() { LOG.info("job is finished. {}", this); if (repoId == Repository.KEEP_ON_LOCAL_REPO_ID) { - Snapshot snapshot = new Snapshot(label, metaInfoBytes, jobInfoBytes); - env.getBackupHandler().addSnapshot(label, snapshot); + env.getBackupHandler().addSnapshot(label, this); return; } } @@ -1025,6 +1017,29 @@ private void cancelInternal() { LOG.info("finished to cancel backup job. current state: {}. {}", curState.name(), this); } + public boolean isLocalSnapshot() { + return repoId == Repository.KEEP_ON_LOCAL_REPO_ID; + } + + // read meta and job info bytes from disk, and return the snapshot + public synchronized Snapshot getSnapshot() { + if (state != BackupJobState.FINISHED || repoId != Repository.KEEP_ON_LOCAL_REPO_ID) { + return null; + } + + try { + File metaInfoFile = new File(localMetaInfoFilePath); + File jobInfoFile = new File(localJobInfoFilePath); + byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); + byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath()); + return new Snapshot(label, metaInfoBytes, jobInfoBytes); + } catch (IOException e) { + LOG.warn("failed to load meta info and job info file, meta info file {}, job info file {}: ", + localMetaInfoFilePath, localJobInfoFilePath, e); + return null; + } + } + public synchronized List getInfo() { List info = Lists.newArrayList(); info.add(String.valueOf(jobId)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 59582edb03ef6b..e4537489177a5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -396,6 +396,11 @@ public boolean isCancelled() { return state == RestoreJobState.CANCELLED; } + @Override + public boolean isFinished() { + return state == RestoreJobState.FINISHED; + } + @Override public synchronized Status updateRepo(Repository repo) { this.repo = repo; @@ -1348,7 +1353,8 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, localTbl.rowStorePageSize(), - localTbl.variantEnableFlattenNested()); + localTbl.variantEnableFlattenNested(), + localTbl.storagePageSize()); task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat()); task.setInRestoreMode(true); if (baseTabletRef != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index 8f9679a0b51a09..d342b01ff7ee1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -38,6 +38,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeVariantArray; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import com.google.common.collect.ImmutableList; @@ -71,7 +73,9 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { tableGenerating(ExplodeJsonArrayStringOuter.class, "explode_json_array_string_outer"), tableGenerating(ExplodeJsonArrayJson.class, "explode_json_array_json"), tableGenerating(ExplodeJsonArrayJsonOuter.class, "explode_json_array_json_outer"), - tableGenerating(ExplodeVariantArray.class, "explode_variant_array") + tableGenerating(ExplodeVariantArray.class, "explode_variant_array"), + tableGenerating(PosExplode.class, "posexplode"), + tableGenerating(PosExplodeOuter.class, "posexplode_outer") ); public static final BuiltinTableGeneratingFunctions INSTANCE = new BuiltinTableGeneratingFunctions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index 11a60360426bf1..61023caf546613 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -497,6 +497,19 @@ public List getViews() { return views; } + public List
getViewsOnIdOrder() { + List
tables = idToTable.values().stream() + .sorted(Comparator.comparing(Table::getId)) + .collect(Collectors.toList()); + List
views = new ArrayList<>(); + for (Table table : tables) { + if (table.getType() == TableType.VIEW) { + views.add(table); + } + } + return views; + } + /** * this method is used for get existed table list by table id list, if table not exist, just ignore it. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cd0c0e80d8f27e..e5ef8e0dc721ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1596,6 +1596,7 @@ private void transferToMaster() { // Set initial root password if master FE first time launch. auth.setInitialRootPassword(Config.initial_root_password); } else { + VariableMgr.forceUpdateVariables(); if (journalVersion <= FeMetaVersion.VERSION_114) { // if journal version is less than 114, which means it is upgraded from version before 2.0. // When upgrading from 1.2 to 2.0, @@ -3604,6 +3605,12 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(olapTable.rowStorePageSize()).append("\""); } + // storage page size + if (olapTable.storagePageSize() != PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE) { + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE).append("\" = \""); + sb.append(olapTable.storagePageSize()).append("\""); + } + // skip inverted index on load if (olapTable.skipWriteIndexOnLoad()) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java index a0be94b683435c..c09c47be22555c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java @@ -32,14 +32,21 @@ import org.apache.doris.load.loadv2.LoadJobScheduler; import org.apache.doris.load.loadv2.LoadManager; import org.apache.doris.load.routineload.RoutineLoadManager; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.stats.StatsErrorEstimator; +import org.apache.doris.nereids.trees.plans.distribute.DistributePlanner; +import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping; import org.apache.doris.planner.GroupCommitPlanner; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.ScanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.Coordinator; +import org.apache.doris.qe.NereidsCoordinator; import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.GlobalTransactionMgr; @@ -134,6 +141,9 @@ public BrokerLoadJob createBrokerLoadJob() { public Coordinator createCoordinator(ConnectContext context, Analyzer analyzer, Planner planner, StatsErrorEstimator statsErrorEstimator) { + if (planner instanceof NereidsPlanner && SessionVariable.canUseNereidsDistributePlanner()) { + return new NereidsCoordinator(context, analyzer, (NereidsPlanner) planner, statsErrorEstimator); + } return new Coordinator(context, analyzer, planner, statsErrorEstimator); } @@ -141,8 +151,23 @@ public Coordinator createCoordinator(ConnectContext context, Analyzer analyzer, public Coordinator createCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, List fragments, List scanNodes, String timezone, boolean loadZeroTolerance, boolean enableProfile) { - return new Coordinator(jobId, queryId, descTable, fragments, scanNodes, timezone, loadZeroTolerance, - enableProfile); + if (SessionVariable.canUseNereidsDistributePlanner()) { + ConnectContext connectContext = new ConnectContext(); + connectContext.setQueryId(queryId); + StatementContext statementContext = new StatementContext( + connectContext, new OriginStatement("", 0) + ); + DistributePlanner distributePlanner = new DistributePlanner(statementContext, fragments); + FragmentIdMapping distributedPlans = distributePlanner.plan(); + + return new NereidsCoordinator( + jobId, queryId, descTable, fragments, distributedPlans.valueList(), + scanNodes, timezone, loadZeroTolerance, enableProfile + ); + } + return new Coordinator( + jobId, queryId, descTable, fragments, scanNodes, timezone, loadZeroTolerance, enableProfile + ); } public GroupCommitPlanner createGroupCommitPlanner(Database db, OlapTable table, List targetColumnNames, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index 83b49475a095d2..a3ed061ed4885d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -119,6 +119,9 @@ public class EsTable extends Table implements GsonPostProcessable { // Periodically pull es metadata private EsMetaStateTracker esMetaStateTracker; + // column name -> elasticsearch field data type + private Map column2typeMap = new HashMap<>(); + public EsTable() { super(TableType.ELASTICSEARCH); } @@ -366,6 +369,6 @@ public void syncTableMetaData() { } public List genColumnsFromEs() { - return EsUtil.genColumnsFromEs(client, indexName, mappingType, false); + return EsUtil.genColumnsFromEs(client, indexName, mappingType, false, column2typeMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java index 5897e9396c83aa..2dc6a90d593c80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java @@ -19,6 +19,8 @@ import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterTableStmt; +import org.apache.doris.analysis.ColumnDef; +import org.apache.doris.analysis.ColumnNullableType; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DbName; @@ -26,10 +28,13 @@ import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.HashDistributionDesc; import org.apache.doris.analysis.KeysDesc; +import org.apache.doris.analysis.ModifyColumnClause; import org.apache.doris.analysis.ModifyPartitionClause; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TypeDef; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; @@ -43,6 +48,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -52,6 +58,7 @@ import java.util.Map; import java.util.Optional; + public class InternalSchemaInitializer extends Thread { private static final Logger LOG = LogManager.getLogger(InternalSchemaInitializer.class); @@ -64,6 +71,7 @@ public void run() { if (!FeConstants.enableInternalSchemaDb) { return; } + modifyColumnStatsTblSchema(); while (!created()) { try { FrontendNodeType feType = Env.getCurrentEnv().getFeType(); @@ -98,6 +106,79 @@ public void run() { modifyTblReplicaCount(database, AuditLoader.AUDIT_LOG_TABLE); } + public void modifyColumnStatsTblSchema() { + while (true) { + try { + Table table = findStatsTable(); + if (table == null) { + break; + } + table.writeLock(); + try { + doSchemaChange(table); + break; + } finally { + table.writeUnlock(); + } + } catch (Throwable t) { + LOG.warn("Failed to do schema change for stats table. Try again later.", t); + } + try { + Thread.sleep(Config.resource_not_ready_sleep_seconds * 1000); + } catch (InterruptedException t) { + // IGNORE + } + } + } + + public Table findStatsTable() { + // 1. check database exist + Optional dbOpt = Env.getCurrentEnv().getInternalCatalog().getDb(FeConstants.INTERNAL_DB_NAME); + if (!dbOpt.isPresent()) { + return null; + } + + // 2. check table exist + Database db = dbOpt.get(); + Optional
tableOp = db.getTable(StatisticConstants.TABLE_STATISTIC_TBL_NAME); + return tableOp.orElse(null); + } + + public void doSchemaChange(Table table) throws UserException { + List clauses = getModifyColumnClauses(table); + if (!clauses.isEmpty()) { + TableName tableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, + StatisticConstants.DB_NAME, table.getName()); + AlterTableStmt alter = new AlterTableStmt(tableName, clauses); + Env.getCurrentEnv().alterTable(alter); + } + } + + public List getModifyColumnClauses(Table table) { + List clauses = Lists.newArrayList(); + for (Column col : table.fullSchema) { + if (col.isKey() && col.getType().isVarchar() + && col.getType().getLength() < StatisticConstants.MAX_NAME_LEN) { + TypeDef typeDef = new TypeDef( + ScalarType.createVarchar(StatisticConstants.MAX_NAME_LEN), col.isAllowNull()); + ColumnNullableType nullableType = + col.isAllowNull() ? ColumnNullableType.NULLABLE : ColumnNullableType.NOT_NULLABLE; + ColumnDef columnDef = new ColumnDef(col.getName(), typeDef, true, null, + nullableType, -1, new ColumnDef.DefaultValue(false, null), ""); + try { + columnDef.analyze(true); + } catch (AnalysisException e) { + LOG.warn("Failed to analyze column {}", col.getName()); + continue; + } + ModifyColumnClause clause = new ModifyColumnClause(columnDef, null, null, Maps.newHashMap()); + clause.setColumn(columnDef.toColumn()); + clauses.add(clause); + } + } + return clauses; + } + @VisibleForTesting public static void modifyTblReplicaCount(Database database, String tblName) { if (Config.isCloudMode() diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 93d9a8d8dfb1ee..b0d25ad2b252b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -358,7 +358,7 @@ public MTMVRefreshSnapshot getRefreshSnapshot() { * * @return mvPartitionName ==> mvPartitionKeyDesc */ - public Map generateMvPartitionDescs() { + public Map generateMvPartitionDescs() throws AnalysisException { Map mtmvItems = getAndCopyPartitionItems(); Map result = Maps.newHashMap(); for (Entry entry : mtmvItems.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index b0d27ac7b5c7d9..8a9ce4a2a2a545 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -113,6 +113,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -2763,6 +2764,20 @@ public long rowStorePageSize() { return PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; } + public void setStoragePageSize(long storagePageSize) { + TableProperty tableProperty = getOrCreatTableProperty(); + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, + Long.valueOf(storagePageSize).toString()); + tableProperty.buildStoragePageSize(); + } + + public long storagePageSize() { + if (tableProperty != null) { + return tableProperty.storagePageSize(); + } + return PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE; + } + public void setStorageFormat(TStorageFormat storageFormat) { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, storageFormat.name()); @@ -3245,8 +3260,10 @@ public PartitionType getPartitionType() { } @Override - public Map getAndCopyPartitionItems() { - readLock(); + public Map getAndCopyPartitionItems() throws AnalysisException { + if (!tryReadLock(1, TimeUnit.MINUTES)) { + throw new AnalysisException("get table read lock timeout, database=" + getDBName() + ",table=" + getName()); + } try { Map res = Maps.newHashMap(); for (Entry entry : getPartitionInfo().getIdToItem(false).entrySet()) { @@ -3283,11 +3300,6 @@ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) { return new MTMVVersionSnapshot(visibleVersion, id); } - @Override - public boolean needAutoRefresh() { - return true; - } - @Override public boolean isPartitionColumnAllowNull() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java index 67cc351cdb35ee..3cc8ffb3f7e142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; import com.google.gson.annotations.SerializedName; @@ -62,7 +63,8 @@ public class S3StorageVault extends StorageVault { TYPE, S3Properties.ACCESS_KEY, S3Properties.SECRET_KEY, - VAULT_NAME + VAULT_NAME, + PropertyConverter.USE_PATH_STYLE )); @SerializedName(value = "properties") diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 8517c4fa58a5fb..1ac556c6846c05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -107,6 +107,8 @@ public class TableProperty implements Writable, GsonPostProcessable { private long rowStorePageSize = PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE; + private long storagePageSize = PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE; + private String compactionPolicy = PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY; private long timeSeriesCompactionGoalSizeMbytes @@ -324,6 +326,17 @@ public long rowStorePageSize() { return rowStorePageSize; } + public TableProperty buildStoragePageSize() { + storagePageSize = Long.parseLong( + properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, + Long.toString(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE))); + return this; + } + + public long storagePageSize() { + return storagePageSize; + } + public TableProperty buildSkipWriteIndexOnLoad() { skipWriteIndexOnLoad = Boolean.parseBoolean( properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, "false")); @@ -724,6 +737,7 @@ public void gsonPostProcess() throws IOException { buildStoreRowColumn(); buildRowStoreColumns(); buildRowStorePageSize(); + buildStoragePageSize(); buildSkipWriteIndexOnLoad(); buildCompactionPolicy(); buildTimeSeriesCompactionGoalSizeMbytes(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/analysis/UseCloudClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/analysis/UseCloudClusterStmt.java index 40bd2dbebc6be4..e35d337ef3570e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/analysis/UseCloudClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/analysis/UseCloudClusterStmt.java @@ -93,7 +93,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { cluster, PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) { throw new AnalysisException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + "'@'" + ConnectContext.get().getRemoteIP() - + "' for cloud cluster '" + cluster + "'"); + + "' for compute group '" + cluster + "'"); } if (!((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterNames().contains(cluster)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java index 3138bad382a0fb..f4c6005a0d828a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java @@ -281,7 +281,7 @@ public void checkCloudClusterPriv(String clusterName) throws DdlException { if (LOG.isDebugEnabled()) { LOG.debug("current instance does not have a cluster name :{}", clusterName); } - throw new DdlException(String.format("Cluster %s not exist", clusterName), + throw new DdlException(String.format("Compute Group %s not exist", clusterName), ErrorCode.ERR_CLOUD_CLUSTER_ERROR); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index fab1ce58775a02..b5197a60786614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -180,7 +180,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getEnableMowLightDelete(), tbl.getInvertedIndexFileStorageFormat(), tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested(), clusterKeyIndexes); + tbl.variantEnableFlattenNested(), clusterKeyIndexes, + tbl.storagePageSize()); requestBuilder.addTabletMetas(builder); } if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { @@ -230,7 +231,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, List rowStoreColumnUniqueIds, boolean enableMowLightDelete, TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize, - boolean variantEnableFlattenNested, List clusterKeyIdxes) throws DdlException { + boolean variantEnableFlattenNested, List clusterKeyIdxes, + long storagePageSize) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -265,6 +267,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, builder.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); builder.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold); builder.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold); + builder.setStoragePageSize(storagePageSize); OlapFile.TabletSchemaCloudPB.Builder schemaBuilder = OlapFile.TabletSchemaCloudPB.newBuilder(); schemaBuilder.setSchemaVersion(schemaVersion); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index f1aed5b6278960..c76a4185fca6c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -241,6 +241,16 @@ private List, TRow>> getPartitionInfosInrernal() throws An // get info List, TRow>> partitionInfos = new ArrayList, TRow>>(); + Map> partitionsUnSyncTables = null; + String mtmvPartitionSyncErrorMsg = null; + if (olapTable instanceof MTMV) { + try { + partitionsUnSyncTables = MTMVPartitionUtil + .getPartitionsUnSyncTables((MTMV) olapTable); + } catch (AnalysisException e) { + mtmvPartitionSyncErrorMsg = e.getMessage(); + } + } olapTable.readLock(); try { List partitionIds; @@ -258,16 +268,6 @@ private List, TRow>> getPartitionInfosInrernal() throws An } Joiner joiner = Joiner.on(", "); - Map> partitionsUnSyncTables = null; - String mtmvPartitionSyncErrorMsg = null; - if (olapTable instanceof MTMV) { - try { - partitionsUnSyncTables = MTMVPartitionUtil - .getPartitionsUnSyncTables((MTMV) olapTable, partitionIds); - } catch (AnalysisException e) { - mtmvPartitionSyncErrorMsg = e.getMessage(); - } - } for (Long partitionId : partitionIds) { Partition partition = olapTable.getPartition(partitionId); @@ -363,11 +363,16 @@ private List, TRow>> getPartitionInfosInrernal() throws An if (StringUtils.isEmpty(mtmvPartitionSyncErrorMsg)) { List partitionUnSyncTables = partitionsUnSyncTables.getOrDefault(partitionId, Lists.newArrayList()); - boolean isSync = CollectionUtils.isEmpty(partitionUnSyncTables); + boolean isSync = partitionsUnSyncTables.containsKey(partitionId) && CollectionUtils.isEmpty( + partitionUnSyncTables); partitionInfo.add(isSync); trow.addToColumnValue(new TCell().setBoolVal(isSync)); - partitionInfo.add(partitionUnSyncTables.toString()); - trow.addToColumnValue(new TCell().setStringVal(partitionUnSyncTables.toString())); + // The calculation logic of partitionsUnSyncTables is not protected in the current lock, + // so the obtained partition list may not be consistent with here + String unSyncTables = partitionsUnSyncTables.containsKey(partitionId) + ? partitionUnSyncTables.toString() : "not sure, please try again"; + partitionInfo.add(unSyncTables); + trow.addToColumnValue(new TCell().setStringVal(unSyncTables)); } else { partitionInfo.add(false); trow.addToColumnValue(new TCell().setBoolVal(false)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index 323566c5161109..ecc4c908809161 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -140,6 +140,7 @@ public class SummaryProfile { NEREIDS_REWRITE_TIME, NEREIDS_OPTIMIZE_TIME, NEREIDS_TRANSLATE_TIME, + NEREIDS_DISTRIBUTE_TIME, WORKLOAD_GROUP, ANALYSIS_TIME, PLAN_TIME, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index c0a544d825dcce..003dcdcf59867e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -256,8 +256,10 @@ private String genSingleSessionVariableDoc(SessionVariable sv, Field field, Lang } sb.append("\n\n"); } - sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.READ_ONLY).append("`\n\n"); - sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.GLOBAL).append("`\n\n"); + sb.append(VAR_READ_ONLY[lang.idx]).append("`") + .append((varAttr.flag() & VariableMgr.READ_ONLY) != 0).append("`\n\n"); + sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`") + .append((varAttr.flag() & VariableMgr.GLOBAL) != 0).append("`\n\n"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index b4384f8a7fa77a..5721db0c27e856 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -99,6 +99,9 @@ public class PropertyAnalyzer { public static final String PROPERTIES_ROW_STORE_PAGE_SIZE = "row_store_page_size"; public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384L; + public static final String PROPERTIES_STORAGE_PAGE_SIZE = "storage_page_size"; + public static final long STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536L; + public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; public static final String PROPERTIES_DISTRIBUTION_TYPE = "distribution_type"; @@ -1072,6 +1075,24 @@ public static long analyzeRowStorePageSize(Map properties) throw return rowStorePageSize; } + public static long analyzeStoragePageSize(Map properties) throws AnalysisException { + long storagePageSize = STORAGE_PAGE_SIZE_DEFAULT_VALUE; + if (properties != null && properties.containsKey(PROPERTIES_STORAGE_PAGE_SIZE)) { + String storagePageSizeStr = properties.get(PROPERTIES_STORAGE_PAGE_SIZE); + try { + storagePageSize = Long.parseLong(storagePageSizeStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid storage page size: " + storagePageSizeStr); + } + if (storagePageSize < 4096 || storagePageSize > 10485760) { + throw new AnalysisException("Storage page size must be between 4KB and 10MB."); + } + storagePageSize = alignTo4K(storagePageSize); + properties.remove(PROPERTIES_STORAGE_PAGE_SIZE); + } + return storagePageSize; + } + // analyzeStorageFormat will parse the storage format from properties // sql: alter table tablet_name set ("storage_format" = "v2") // Use this sql to convert all tablets(base and rollup index) to a new format segment diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index f8183028c6acd6..0ab2460f7adfa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1641,6 +1641,10 @@ public PartitionPersistInfo addPartition(Database db, String tableName, AddParti properties.put(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE, Long.toString(olapTable.rowStorePageSize())); } + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) { + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, + Long.toString(olapTable.storagePageSize())); + } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, olapTable.skipWriteIndexOnLoad().toString()); @@ -2168,7 +2172,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.storeRowColumn(), binlogConfig, tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, tbl.rowStorePageSize(), - tbl.variantEnableFlattenNested()); + tbl.variantEnableFlattenNested(), + tbl.storagePageSize()); task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2655,9 +2660,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } - olapTable.setRowStorePageSize(rowStorePageSize); + long storagePageSize = PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE; + try { + storagePageSize = PropertyAnalyzer.analyzeStoragePageSize(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + olapTable.setStoragePageSize(storagePageSize); + // check data sort properties int keyColumnSize = CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) ? keysDesc.keysColumnSize() : keysDesc.getClusterKeysColumnNames().size(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index cfde5e794a3aa4..4f05d6d29cd89e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -25,7 +25,9 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -73,13 +75,20 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { EsRestClient restClient = ((EsExternalCatalog) catalog).getEsRestClient(); - return Optional.of(new SchemaCacheValue( - EsUtil.genColumnsFromEs(restClient, name, null, - ((EsExternalCatalog) catalog).enableMappingEsId()))); + Map column2typeMap = new HashMap<>(); + List columns = EsUtil.genColumnsFromEs(restClient, name, null, + ((EsExternalCatalog) catalog).enableMappingEsId(), column2typeMap); + return Optional.of(new EsSchemaCacheValue(columns, column2typeMap)); + } + + public Map getColumn2type() { + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((EsSchemaCacheValue) value).getColumn2typeMap()).orElse(null); } private EsTable toEsTable() { List schema = getFullSchema(); + Map column2typeMap = getColumn2type(); EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; EsTable esTable = new EsTable(this.id, this.name, schema, TableType.ES_EXTERNAL_TABLE); esTable.setIndexName(name); @@ -95,6 +104,7 @@ private EsTable toEsTable() { esTable.setHosts(String.join(",", esCatalog.getNodes())); esTable.syncTableMetaData(); esTable.setIncludeHiddenIndex(esCatalog.enableIncludeHiddenIndex()); + esTable.setColumn2typeMap(column2typeMap); return esTable; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsSchemaCacheValue.java new file mode 100644 index 00000000000000..e53eae676e8107 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsSchemaCacheValue.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.es; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.SchemaCacheValue; + +import java.util.List; +import java.util.Map; + +public class EsSchemaCacheValue extends SchemaCacheValue { + public Map column2typeMap; + + public EsSchemaCacheValue(List columns, Map column2typeMap) { + super(columns); + this.column2typeMap = column2typeMap; + } + + public Map getColumn2typeMap() { + return column2typeMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java index 0ee9bbd1f7725d..55fea86718aa93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java @@ -189,18 +189,18 @@ public static ObjectNode getMappingProps(String sourceIndex, String indexMapping * Add mappingEsId config in es external catalog. **/ public static List genColumnsFromEs(EsRestClient client, String indexName, String mappingType, - boolean mappingEsId) { + boolean mappingEsId, Map column2typeMap) { String mapping = client.getMapping(indexName); ObjectNode mappings = getMapping(mapping); // Get array_fields while removing _meta property. List arrayFields = new ArrayList<>(); ObjectNode rootSchema = getRootSchema(mappings, mappingType, arrayFields); - return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId, arrayFields); + return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId, arrayFields, column2typeMap); } @VisibleForTesting public static List genColumnsFromEs(String indexName, String mappingType, ObjectNode rootSchema, - boolean mappingEsId, List arrayFields) { + boolean mappingEsId, List arrayFields, Map column2typeMap) { List columns = new ArrayList<>(); if (mappingEsId) { Column column = new Column(); @@ -220,7 +220,8 @@ public static List genColumnsFromEs(String indexName, String mappingType while (iterator.hasNext()) { String fieldName = iterator.next(); ObjectNode fieldValue = (ObjectNode) mappingProps.get(fieldName); - Column column = parseEsField(fieldName, replaceFieldAlias(mappingProps, fieldValue), arrayFields); + Column column = parseEsField(fieldName, replaceFieldAlias(mappingProps, fieldValue), arrayFields, + column2typeMap); columns.add(column); } return columns; @@ -245,7 +246,8 @@ private static ObjectNode replaceFieldAlias(ObjectNode mappingProps, ObjectNode return fieldValue; } - private static Column parseEsField(String fieldName, ObjectNode fieldValue, List arrayFields) { + private static Column parseEsField(String fieldName, ObjectNode fieldValue, List arrayFields, + Map column2typeMap) { Column column = new Column(); column.setName(fieldName); column.setIsKey(true); @@ -256,6 +258,7 @@ private static Column parseEsField(String fieldName, ObjectNode fieldValue, List if (fieldValue.has("type")) { String typeStr = fieldValue.get("type").asText(); column.setComment("Elasticsearch type is " + typeStr); + column2typeMap.put(fieldName, typeStr); // reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html switch (typeStr) { case "null": @@ -298,15 +301,17 @@ private static Column parseEsField(String fieldName, ObjectNode fieldValue, List type = ScalarType.createStringType(); break; case "nested": - case "object": type = Type.JSONB; break; default: type = Type.UNSUPPORTED; } } else { + // When there is no explicit type in mapping, it indicates this type is an `object` in Elasticsearch. + // reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html type = Type.JSONB; - column.setComment("Elasticsearch no type"); + column.setComment("Elasticsearch type is object"); + column2typeMap.put(fieldName, "object"); } if (arrayFields.contains(fieldName)) { column.setType(ArrayType.create(type, true)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java index 19930bb2b14a88..40c113626923d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java @@ -70,14 +70,14 @@ public final class QueryBuilders { * Generate dsl from compound expr. **/ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownList, - Map fieldsContext, BuilderOptions builderOptions) { + Map fieldsContext, BuilderOptions builderOptions, Map column2typeMap) { CompoundPredicate compoundPredicate = (CompoundPredicate) expr; switch (compoundPredicate.getOp()) { case AND: { QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); if (left != null && right != null) { return QueryBuilders.boolQuery().must(left).must(right); } @@ -86,9 +86,9 @@ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownLis case OR: { int beforeSize = notPushDownList.size(); QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); int afterSize = notPushDownList.size(); if (left != null && right != null) { return QueryBuilders.boolQuery().should(left).should(right); @@ -101,7 +101,7 @@ private static QueryBuilder toCompoundEsDsl(Expr expr, List notPushDownLis } case NOT: { QueryBuilder child = toEsDsl(compoundPredicate.getChild(0), notPushDownList, fieldsContext, - builderOptions); + builderOptions, column2typeMap); if (child != null) { return QueryBuilders.boolQuery().mustNot(child); } @@ -122,10 +122,10 @@ private static Expr exprWithoutCast(Expr expr) { return expr; } - public static QueryBuilder toEsDsl(Expr expr) { + public static QueryBuilder toEsDsl(Expr expr, Map column2typeMap) { return toEsDsl(expr, new ArrayList<>(), new HashMap<>(), BuilderOptions.builder().likePushDown(Boolean.parseBoolean(EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)) - .build()); + .build(), column2typeMap); } private static TExprOpcode flipOpCode(TExprOpcode opCode) { @@ -185,32 +185,44 @@ private static QueryBuilder parseIsNullPredicate(Expr expr, String column) { return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(column)); } - private static QueryBuilder parseLikePredicate(Expr expr, String column) { - LikePredicate likePredicate = (LikePredicate) expr; - if (likePredicate.getOp().equals(Operator.LIKE)) { - char[] chars = likePredicate.getChild(1).getStringValue().toCharArray(); - // example of translation : - // abc_123 ===> abc?123 - // abc%ykz ===> abc*123 - // %abc123 ===> *abc123 - // _abc123 ===> ?abc123 - // \\_abc1 ===> \\_abc1 - // abc\\_123 ===> abc\\_123 - // abc\\%123 ===> abc\\%123 - // NOTE. user must input sql like 'abc\\_123' or 'abc\\%ykz' - for (int i = 0; i < chars.length; i++) { - if (chars[i] == '_' || chars[i] == '%') { - if (i == 0) { - chars[i] = (chars[i] == '_') ? '?' : '*'; - } else if (chars[i - 1] != '\\') { - chars[i] = (chars[i] == '_') ? '?' : '*'; - } - } + private static QueryBuilder parseLikeExpression(Expr expr, String column) { + String pattern; + if (expr instanceof LikePredicate) { + LikePredicate likePredicate = (LikePredicate) expr; + if (!likePredicate.getOp().equals(Operator.LIKE)) { + return QueryBuilders.wildcardQuery(column, likePredicate.getChild(1).getStringValue()); + } + pattern = likePredicate.getChild(1).getStringValue(); + } else if (expr instanceof FunctionCallExpr) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + String fnName = functionCallExpr.getFnName().getFunction(); + if (!fnName.equalsIgnoreCase("like")) { + return QueryBuilders.wildcardQuery(column, functionCallExpr.getChild(1).getStringValue()); } - return QueryBuilders.wildcardQuery(column, new String(chars)); + pattern = functionCallExpr.getChild(1).getStringValue(); } else { - return QueryBuilders.wildcardQuery(column, likePredicate.getChild(1).getStringValue()); + throw new IllegalArgumentException("Unsupported expression type"); + } + char[] chars = pattern.toCharArray(); + // example of translation : + // abc_123 ===> abc?123 + // abc%ykz ===> abc*123 + // %abc123 ===> *abc123 + // _abc123 ===> ?abc123 + // \\_abc1 ===> \\_abc1 + // abc\\_123 ===> abc\\_123 + // abc\\%123 ===> abc\\%123 + // NOTE. user must input sql like 'abc\\_123' or 'abc\\%ykz' + for (int i = 0; i < chars.length; i++) { + if (chars[i] == '_' || chars[i] == '%') { + if (i == 0) { + chars[i] = (chars[i] == '_') ? '?' : '*'; + } else if (chars[i - 1] != '\\') { + chars[i] = (chars[i] == '_') ? '?' : '*'; + } + } } + return QueryBuilders.wildcardQuery(column, new String(chars)); } private static QueryBuilder parseInPredicate(Expr expr, String column, boolean needDateCompat) { @@ -257,18 +269,18 @@ private static String getColumnFromExpr(Expr expr) { * Doris expr to es dsl. **/ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map fieldsContext, - BuilderOptions builderOptions) { + BuilderOptions builderOptions, Map column2typeMap) { if (expr == null) { return null; } // esquery functionCallExpr will be rewritten to castExpr in where clause rewriter, // so we get the functionCallExpr here. if (expr instanceof CastExpr) { - return toEsDsl(expr.getChild(0), notPushDownList, fieldsContext, builderOptions); + return toEsDsl(expr.getChild(0), notPushDownList, fieldsContext, builderOptions, column2typeMap); } // CompoundPredicate, `between` also converted to CompoundPredicate. if (expr instanceof CompoundPredicate) { - return toCompoundEsDsl(expr, notPushDownList, fieldsContext, builderOptions); + return toCompoundEsDsl(expr, notPushDownList, fieldsContext, builderOptions, column2typeMap); } TExprOpcode opCode = expr.getOpcode(); boolean isFlip = false; @@ -287,6 +299,7 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map needCompatDateFields = builderOptions.getNeedCompatDateFields(); boolean needDateCompat = needCompatDateFields != null && needCompatDateFields.contains(column); @@ -313,24 +326,29 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map notPushDownList = new ArrayList<>(); + if (table.getColumn2typeMap() == null) { + table.genColumnsFromEs(); + } for (Expr expr : conjuncts) { QueryBuilder queryBuilder = QueryBuilders.toEsDsl(expr, notPushDownList, fieldsContext, BuilderOptions.builder().likePushDown(table.isLikePushDown()) - .needCompatDateFields(table.needCompatDateFields()).build()); + .needCompatDateFields(table.needCompatDateFields()).build(), + table.getColumn2typeMap()); if (queryBuilder != null) { hasFilter = true; boolQueryBuilder.must(queryBuilder); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index b48b47acf1378e..445aae09f0de40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; @@ -27,10 +28,12 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.mtmv.MTMVBaseTableIf; import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot; import org.apache.doris.mtmv.MTMVRefreshContext; import org.apache.doris.mtmv.MTMVRelatedTableIf; @@ -87,7 +90,7 @@ /** * Hive metastore external table. */ -public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableIf { +public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableIf, MTMVBaseTableIf { private static final Logger LOG = LogManager.getLogger(HMSExternalTable.class); public static final Set SUPPORTED_HIVE_FILE_FORMATS; @@ -586,9 +589,9 @@ public Optional getColumnStatistic(String colName) { case ICEBERG: if (GlobalVariable.enableFetchIcebergStats) { return StatisticsUtil.getIcebergColumnStats(colName, - Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable( - catalog, dbName, name - )); + Env.getCurrentEnv().getExtMetaCacheMgr().getIcebergMetadataCache().getIcebergTable( + catalog, dbName, name + )); } else { break; } @@ -819,11 +822,6 @@ private HivePartition getHivePartitionByIdOrAnalysisException(Long partitionId, return partition; } - @Override - public boolean needAutoRefresh() { - return true; - } - @Override public boolean isPartitionColumnAllowNull() { return true; @@ -958,4 +956,10 @@ public boolean isPartitionedTable() { makeSureInitialized(); return !isView() && remoteTable.getPartitionKeysSize() > 0; } + + @Override + public void beforeMTMVRefresh(MTMV mtmv) throws DdlException { + Env.getCurrentEnv().getRefreshManager() + .refreshTable(getCatalog().getName(), getDbName(), getName(), true); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index 4ec4319c228c05..e0b84b0860e551 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -81,8 +81,10 @@ public class MaxComputeScanNode extends FileQueryScanNode { private final MaxComputeExternalTable table; - TableBatchReadSession tableBatchReadSession; + private TableBatchReadSession tableBatchReadSession; private Predicate filterPredicate; + private static final LocationPath ROW_OFFSET_PATH = new LocationPath("/row_offset", Maps.newHashMap()); + private static final LocationPath BYTE_SIZE_PATH = new LocationPath("/byte_size", Maps.newHashMap()); public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv); @@ -441,7 +443,7 @@ public List getSplits() throws UserException { for (com.aliyun.odps.table.read.split.InputSplit split : assigner.getAllSplits()) { MaxComputeSplit maxComputeSplit = - new MaxComputeSplit(new LocationPath("/byte_size", Maps.newHashMap()), + new MaxComputeSplit(BYTE_SIZE_PATH, ((IndexedInputSplit) split).getSplitIndex(), -1, mcCatalog.getSplitByteSize(), modificationTime, null, @@ -464,7 +466,7 @@ public List getSplits() throws UserException { assigner.getSplitByRowOffset(offset, recordsPerSplit); MaxComputeSplit maxComputeSplit = - new MaxComputeSplit(new LocationPath("/row_offset", Maps.newHashMap()), + new MaxComputeSplit(ROW_OFFSET_PATH, offset, recordsPerSplit, totalRowCount, modificationTime, null, Collections.emptyList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index cd477cc9b290c4..29e3d0529f2830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -164,6 +164,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) fileDesc.setDbId(((PaimonExternalTable) source.getTargetTable()).getDbId()); fileDesc.setTblId(source.getTargetTable().getId()); fileDesc.setLastUpdateTime(source.getTargetTable().getUpdateTime()); + fileDesc.setPaimonTable(encodeObjectToString(source.getPaimonTable())); // The hadoop conf should be same with PaimonExternalCatalog.createCatalog()#getConfiguration() fileDesc.setHadoopConf(source.getCatalog().getCatalogProperty().getHadoopProperties()); Optional optDeletionFile = paimonSplit.getDeletionFile(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 3ed88c6ceef62d..7e94d81518ec0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -32,6 +32,7 @@ import com.amazonaws.auth.SystemPropertiesCredentialsProvider; import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; @@ -334,6 +335,15 @@ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map { Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(executor); }); - Env.getCurrentEnv().getEditLog().logExportCreate(job); + LOG.info("add export job. {}", job); + } finally { writeUnlock(); } - LOG.info("add export job. {}", job); + } public void cancelExportJob(CancelExportStmt stmt) throws DdlException, AnalysisException { @@ -470,6 +473,7 @@ public void replayCreateExportJob(ExportJob job) { public void replayUpdateJobState(ExportJobStateTransfer stateTransfer) { writeLock(); try { + LOG.info("replay update export job: {}, {}", stateTransfer.getJobId(), stateTransfer.getState()); ExportJob job = exportIdToJob.get(stateTransfer.getJobId()); job.replayExportJobState(stateTransfer.getState()); job.setStartTimeMs(stateTransfer.getStartTimeMs()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index 3544aeda2b867a..ef429a1d564208 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -113,7 +113,7 @@ public LoadingTaskPlanner(Long loadJobId, long txnId, long dbId, OlapTable table PrivPredicate.SELECT)) { this.analyzer.setUDFAllowed(true); } else { - this.analyzer.setUDFAllowed(false); + this.analyzer.setUDFAllowed(Config.enable_udf_in_load); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 1c8f51bd4ebf24..c5c72eae3c5a8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -619,7 +619,7 @@ private static void taskReport(long backendId, Map> running List diffTasks = AgentTaskQueue.getDiffTasks(backendId, runningTasks); - AgentBatchTask batchTask = new AgentBatchTask(); + AgentBatchTask batchTask = new AgentBatchTask(Config.report_resend_batch_task_num_per_rpc); long taskReportTime = System.currentTimeMillis(); for (AgentTask task : diffTasks) { // these tasks no need to do diff @@ -974,7 +974,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.getRowStoreColumnsUniqueIds(rowStoreColumns), objectPool, olapTable.rowStorePageSize(), - olapTable.variantEnableFlattenNested()); + olapTable.variantEnableFlattenNested(), + olapTable.storagePageSize()); createReplicaTask.setIsRecoverTask(true); createReplicaTask.setInvertedIndexFileStorageFormat(olapTable .getInvertedIndexFileStorageFormat()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVBaseTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVBaseTableIf.java new file mode 100644 index 00000000000000..e5086cbb6c9f9f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVBaseTableIf.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.DdlException; + +/** + * The base table of the materialized view should implement this interface and do some things when necessary, + * but it is currently not mandatory + */ +public interface MTMVBaseTableIf { + + /** + * Do something before refreshing the MTMV + * + * @param mtmv + * @throws DdlException + */ + void beforeMTMVRefresh(MTMV mtmv) throws DdlException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java index 8ffcea423d7fab..2c03ad16176fea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java @@ -130,7 +130,7 @@ private void setScheduleJobConfig(JobExecutionConfiguration jobExecutionConfigur public void dropMTMV(MTMV mtmv) throws DdlException { try { Env.getCurrentEnv().getJobManager() - .unregisterJob(mtmv.getJobInfo().getJobName(), false); + .unregisterJob(mtmv.getJobInfo().getJobName(), true); } catch (JobException e) { LOG.warn("drop mtmv job failed, mtmvName: {}", mtmv.getName(), e); throw new DdlException(e.getMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 2a0863a6e61101..1cfb5e021a5309 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -231,12 +231,12 @@ public static boolean isMTMVSync(MTMVRefreshContext context, Set * getPartitionsUnSyncTables * * @param mtmv - * @param partitionIds * @return partitionName ==> UnSyncTableNames * @throws AnalysisException */ - public static Map> getPartitionsUnSyncTables(MTMV mtmv, List partitionIds) + public static Map> getPartitionsUnSyncTables(MTMV mtmv) throws AnalysisException { + List partitionIds = mtmv.getPartitionIds(); Map> res = Maps.newHashMap(); MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv); for (Long partitionId : partitionIds) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java index 516eb904e58463..4a8b14603ce4d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java @@ -35,10 +35,12 @@ public interface MTMVRelatedTableIf extends TableIf { /** * Get all partitions of the table + * Note: This method is called every time there is a refresh and transparent rewrite, + * so if this method is slow, it will significantly reduce query performance * - * @return partitionId->PartitionItem + * @return partitionName->PartitionItem */ - Map getAndCopyPartitionItems(); + Map getAndCopyPartitionItems() throws AnalysisException; /** * getPartitionType LIST/RANGE/UNPARTITIONED @@ -64,8 +66,12 @@ public interface MTMVRelatedTableIf extends TableIf { /** * getPartitionSnapshot + * It is best to use the version. If there is no version, use the last update time + * If snapshots have already been obtained in bulk in the context, + * the results should be obtained directly from the context * * @param partitionName + * @param context * @return partition snapshot at current time * @throws AnalysisException */ @@ -73,7 +79,11 @@ public interface MTMVRelatedTableIf extends TableIf { /** * getTableSnapshot + * It is best to use the version. If there is no version, use the last update time + * If snapshots have already been obtained in bulk in the context, + * the results should be obtained directly from the context * + * @param context * @return table snapshot at current time * @throws AnalysisException */ @@ -85,7 +95,9 @@ public interface MTMVRelatedTableIf extends TableIf { * @return If return false,The method of comparing whether to synchronize will directly return true, * otherwise the snapshot information will be compared */ - boolean needAutoRefresh(); + default boolean needAutoRefresh() { + return true; + } /** * if allow partition column `isAllowNull` diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java index bbb8bf4d378879..8d1304658ff2a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java @@ -159,9 +159,21 @@ List getGroups(String userName) { if (userDn == null) { return groups; } - List groupDns = getDn(org.springframework.ldap.query.LdapQueryBuilder.query() + List groupDns; + + // Support Open Directory implementations + // If no group filter is configured, it defaults to querying groups based on the attribute 'member' + // for standard LDAP implementations + if (!LdapConfig.ldap_group_filter.isEmpty()) { + groupDns = getDn(org.springframework.ldap.query.LdapQueryBuilder.query() + .base(LdapConfig.ldap_group_basedn) + .filter(getGroupFilter(LdapConfig.ldap_group_filter, userName))); + } else { + groupDns = getDn(org.springframework.ldap.query.LdapQueryBuilder.query() .base(LdapConfig.ldap_group_basedn) .where("member").is(userDn)); + } + if (groupDns == null) { return groups; } @@ -209,4 +221,8 @@ protected String doMapFromContext(DirContextOperations ctx) { private String getUserFilter(String userFilter, String userName) { return userFilter.replaceAll("\\{login}", userName); } + + private String getGroupFilter(String groupFilter, String userName) { + return groupFilter.replaceAll("\\{login}", userName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index 12db90761cbd03..cf15dfa2ac9937 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -681,7 +681,9 @@ private void grantInternal(UserIdentity userIdent, String role, TablePattern tbl throws DdlException { writeLock(); try { - checkTablePatternExist(tblPattern); + if (!isReplay) { + checkTablePatternExist(tblPattern); + } if (role == null) { if (!doesUserExist(userIdent)) { throw new DdlException("user " + userIdent + " does not exist"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 86886336d85109..4777a14741e192 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -228,7 +228,7 @@ public Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties, } } - private Plan planWithoutLock( + protected Plan planWithoutLock( LogicalPlan plan, ExplainLevel explainLevel, boolean showPlanProcess, PhysicalProperties requireProperties) { // resolve column, table and function @@ -311,7 +311,7 @@ private Plan planWithoutLock( return physicalPlan; } - private LogicalPlan preprocess(LogicalPlan logicalPlan) { + protected LogicalPlan preprocess(LogicalPlan logicalPlan) { return new PlanPreprocessors(statementContext).process(logicalPlan); } @@ -322,7 +322,7 @@ private void initCascadesContext(LogicalPlan plan, PhysicalProperties requirePro } } - private void analyze(boolean showPlanProcess) { + protected void analyze(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start analyze plan"); } @@ -337,7 +337,7 @@ private void analyze(boolean showPlanProcess) { /** * Logical plan rewrite based on a series of heuristic rules. */ - private void rewrite(boolean showPlanProcess) { + protected void rewrite(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start rewrite plan"); } @@ -349,7 +349,7 @@ private void rewrite(boolean showPlanProcess) { } // DependsRules: EnsureProjectOnTopJoin.class - private void optimize() { + protected void optimize() { if (LOG.isDebugEnabled()) { LOG.debug("Start optimize plan"); } @@ -360,7 +360,7 @@ private void optimize() { } } - private void splitFragments(PhysicalPlan resultPlan) { + protected void splitFragments(PhysicalPlan resultPlan) { if (resultPlan instanceof PhysicalSqlCache) { return; } @@ -455,7 +455,7 @@ private void splitFragments(PhysicalPlan resultPlan) { } } - private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) { + protected void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) { boolean canUseNereidsDistributePlanner = SessionVariable.canUseNereidsDistributePlanner(); if ((!canUseNereidsDistributePlanner && explainLevel.isPlanLevel)) { return; @@ -465,18 +465,21 @@ private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) { } splitFragments(physicalPlan); + doDistribute(canUseNereidsDistributePlanner); + } + protected void doDistribute(boolean canUseNereidsDistributePlanner) { if (!canUseNereidsDistributePlanner) { return; } - distributedPlans = new DistributePlanner(fragments).plan(); + distributedPlans = new DistributePlanner(statementContext, fragments).plan(); if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsDistributeTime(); } } - private PhysicalPlan postProcess(PhysicalPlan physicalPlan) { + protected PhysicalPlan postProcess(PhysicalPlan physicalPlan) { return new PlanPostProcessors(cascadesContext).process(physicalPlan); } @@ -595,11 +598,11 @@ public String getExplainString(ExplainOptions explainOptions) { ExplainLevel explainLevel = getExplainLevel(explainOptions); String plan = ""; String mvSummary = ""; - if (this.getPhysicalPlan() != null && cascadesContext != null) { + if ((this.getPhysicalPlan() != null || this.getOptimizedPlan() != null) && cascadesContext != null) { mvSummary = cascadesContext.getMaterializationContexts().isEmpty() ? "" : "\n\n========== MATERIALIZATIONS ==========\n" + MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(), - this.getPhysicalPlan()); + this.getPhysicalPlan() == null ? this.getOptimizedPlan() : this.getPhysicalPlan()); } switch (explainLevel) { case PARSED_PLAN: @@ -735,6 +738,10 @@ public CascadesContext getCascadesContext() { return cascadesContext; } + public ConnectContext getConnectContext() { + return cascadesContext.getConnectContext(); + } + public static PhysicalProperties buildInitRequireProperties() { return PhysicalProperties.GATHER; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index ed64864da5079f..69848ca8f04da2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -47,18 +47,19 @@ import org.apache.doris.qe.ShortCircuitQueryContext; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.statistics.Statistics; +import org.apache.doris.system.Backend; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.sparkproject.guava.base.Throwables; import java.io.Closeable; import java.util.ArrayList; @@ -171,6 +172,8 @@ public class StatementContext implements Closeable { private String disableJoinReorderReason; + private Backend groupCommitMergeBackend; + public StatementContext() { this(ConnectContext.get(), null, 0); } @@ -568,4 +571,13 @@ public Optional getDisableJoinReorderReason() { public void setDisableJoinReorderReason(String disableJoinReorderReason) { this.disableJoinReorderReason = disableJoinReorderReason; } + + public Backend getGroupCommitMergeBackend() { + return groupCommitMergeBackend; + } + + public void setGroupCommitMergeBackend( + Backend groupCommitMergeBackend) { + this.groupCommitMergeBackend = groupCommitMergeBackend; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index c9e49b7535b45f..627a983c123b8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -78,6 +78,7 @@ import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; @@ -557,7 +558,7 @@ public Expr visitTableGeneratingFunction(TableGeneratingFunction function, @Override public Expr visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, PlanTranslatorContext context) { NullableMode nullableMode = NullableMode.DEPEND_ON_ARGUMENT; - if (binaryArithmetic instanceof AlwaysNullable) { + if (binaryArithmetic instanceof AlwaysNullable || binaryArithmetic instanceof PropagateNullLiteral) { nullableMode = NullableMode.ALWAYS_NULLABLE; } else if (binaryArithmetic instanceof AlwaysNotNullable) { nullableMode = NullableMode.ALWAYS_NOT_NULLABLE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 599511302a72fe..35694169498870 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -225,10 +225,6 @@ public String getTypeName() { @Override public String toString() { - if (!leftRejectEdges.isEmpty() || !rightRejectEdges.isEmpty()) { - return String.format("<%s --%s-- %s>[%s , %s]", LongBitmap.toString(leftExtendedNodes), - this.getTypeName(), LongBitmap.toString(rightExtendedNodes), leftRejectEdges, rightRejectEdges); - } return String.format("<%s --%s-- %s>", LongBitmap.toString(leftExtendedNodes), this.getTypeName(), LongBitmap.toString(rightExtendedNodes)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java index aecb03a4158521..656fde3024e79f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java @@ -116,7 +116,6 @@ private void ensureChildrenRewritten() { // some rule return new plan tree, which the number of new plan node > 1, // we should transform this new plan nodes too. - // NOTICE: this relay on pull up cte anchor if (isTraverseChildren.test(plan)) { pushChildrenJobs(plan); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java index 0f87a745b5e43f..60969cdf6e701d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java @@ -60,8 +60,6 @@ protected final RewriteResult rewrite(Plan plan, List rules, RewriteJobCon } Plan newPlan = newPlans.get(0); if (!newPlan.deepEquals(plan)) { - // don't remove this comment, it can help us to trace some bug when developing. - NereidsTracer.logRewriteEvent(rule.toString(), pattern, plan, newPlan); String traceBefore = null; if (showPlanProcess) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java index 19d92e2f4f361a..806b2cf61ea454 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java @@ -58,7 +58,6 @@ public void execute() { RewriteJobContext newRewriteJobContext = rewriteJobContext.withChildrenVisited(true); pushJob(new PlanTreeRewriteTopDownJob(newRewriteJobContext, context, isTraverseChildren, rules)); - // NOTICE: this relay on pull up cte anchor if (isTraverseChildren.test(rewriteJobContext.plan)) { pushChildrenJobs(newRewriteJobContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 86a849d3639e6c..42c61e376508a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -30,11 +30,13 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BuiltinAggregateFunctions; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.job.common.IntervalUnit; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -195,6 +197,8 @@ import org.apache.doris.nereids.DorisParser.ShowCreateMTMVContext; import org.apache.doris.nereids.DorisParser.ShowCreateProcedureContext; import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext; +import org.apache.doris.nereids.DorisParser.ShowVariablesContext; +import org.apache.doris.nereids.DorisParser.ShowViewContext; import org.apache.doris.nereids.DorisParser.SimpleColumnDefContext; import org.apache.doris.nereids.DorisParser.SimpleColumnDefsContext; import org.apache.doris.nereids.DorisParser.SingleStatementContext; @@ -424,6 +428,8 @@ import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetVariableCommand; import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand; @@ -4003,6 +4009,7 @@ public SetDefaultStorageVaultCommand visitSetDefaultStorageVault(SetDefaultStora } @Override + public Object visitRefreshCatalog(RefreshCatalogContext ctx) { if (ctx.name != null) { @@ -4011,5 +4018,44 @@ public Object visitRefreshCatalog(RefreshCatalogContext ctx) { return new RefreshCatalogCommand(catalogName, properties); } throw new AnalysisException("catalog name can not be null"); + + public LogicalPlan visitShowVariables(ShowVariablesContext ctx) { + SetType type = SetType.DEFAULT; + if (ctx.GLOBAL() != null) { + type = SetType.GLOBAL; + } else if (ctx.LOCAL() != null || ctx.SESSION() != null) { + type = SetType.SESSION; + } + if (ctx.wildWhere() != null) { + if (ctx.wildWhere().LIKE() != null) { + return new ShowVariablesCommand(type, stripQuotes(ctx.wildWhere().STRING_LITERAL().getText())); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT `VARIABLE_NAME` AS `Variable_name`, `VARIABLE_VALUE` AS `Value` FROM "); + sb.append("`").append(InternalCatalog.INTERNAL_CATALOG_NAME).append("`"); + sb.append("."); + sb.append("`").append(InfoSchemaDb.DATABASE_NAME).append("`"); + sb.append("."); + if (type == SetType.GLOBAL) { + sb.append("`global_variables` "); + } else { + sb.append("`session_variables` "); + } + sb.append(getOriginSql(ctx.wildWhere())); + return new NereidsParser().parseSingle(sb.toString()); + } + } else { + return new ShowVariablesCommand(type, null); + } + } + + @Override + public ShowViewCommand visitShowView(ShowViewContext ctx) { + List tableNameParts = visitMultipartIdentifier(ctx.tableName); + String databaseName = null; + if (ctx.database != null) { + databaseName = stripQuotes(ctx.database.getText()); + } + return new ShowViewCommand(databaseName, new TableNameInfo(tableNameParts)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 4f4a8bebfdf562..1a95fb5f9e3807 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -505,12 +505,15 @@ private PhysicalProperties computeShuffleJoinOutputProperties( switch (hashJoin.getJoinType()) { case INNER_JOIN: + case CROSS_JOIN: if (shuffleSide == ShuffleSide.LEFT) { - return new PhysicalProperties(DistributionSpecHash.merge( - rightHashSpec, leftHashSpec, outputShuffleType)); + return new PhysicalProperties( + DistributionSpecHash.merge(rightHashSpec, leftHashSpec, outputShuffleType) + ); } else { - return new PhysicalProperties(DistributionSpecHash.merge( - leftHashSpec, rightHashSpec, outputShuffleType)); + return new PhysicalProperties( + DistributionSpecHash.merge(leftHashSpec, rightHashSpec, outputShuffleType) + ); } case LEFT_SEMI_JOIN: case LEFT_ANTI_JOIN: @@ -526,12 +529,13 @@ private PhysicalProperties computeShuffleJoinOutputProperties( case RIGHT_SEMI_JOIN: case RIGHT_ANTI_JOIN: case RIGHT_OUTER_JOIN: - if (shuffleSide == ShuffleSide.RIGHT) { - return new PhysicalProperties( - rightHashSpec.withShuffleTypeAndForbidColocateJoin(outputShuffleType) - ); - } else { + if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec, hashJoin.getHashJoinConjuncts())) { return new PhysicalProperties(rightHashSpec); + } else { + // retain left shuffle type, since coordinator use left most node to schedule fragment + // forbid colocate join, since right table already shuffle + return new PhysicalProperties(rightHashSpec.withShuffleTypeAndForbidColocateJoin( + leftHashSpec.getShuffleType())); } case FULL_OUTER_JOIN: return PhysicalProperties.createAnyFromHash(leftHashSpec, rightHashSpec); @@ -563,6 +567,9 @@ private ShuffleSide computeShuffleSide(DistributionSpecHash leftHashSpec, Distri case STORAGE_BUCKETED: // use storage hash to shuffle right to left to do bucket shuffle join return ShuffleSide.RIGHT; + case EXECUTION_BUCKETED: + // compatible old ut + return ShuffleSide.RIGHT; default: } break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 42dd4b59da1956..868f97949c0705 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -455,7 +455,9 @@ private boolean compareEdgeWithNode(Edge query, Edge view) { if (query instanceof FilterEdge && view instanceof FilterEdge) { return compareFilterEdgeWithNode((FilterEdge) query, viewFilterEdgesAfterInferring.get(view.getIndex())); } else if (query instanceof JoinEdge && view instanceof JoinEdge) { - return compareJoinEdgeWithNode((JoinEdge) query, viewJoinEdgesAfterInferring.get(view.getIndex())); + // compare original or inferred join edge + return compareJoinEdgeWithNode((JoinEdge) query, viewJoinEdgesAfterInferring.get(view.getIndex())) + || compareJoinEdgeWithNode((JoinEdge) query, (JoinEdge) view); } return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 609125280ded4b..df535d59d87399 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -35,7 +35,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.statistics.ColumnStatistic; @@ -348,7 +347,7 @@ public String toString() { * ToSummaryString, this contains only summary info. */ public static String toSummaryString(List materializationContexts, - PhysicalPlan physicalPlan) { + Plan physicalPlan) { if (materializationContexts.isEmpty()) { return ""; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index fa29d4d0e123be..526ec7030d2db5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -210,28 +210,25 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, }); // Collect expression from join condition in hyper graph for (JoinEdge edge : hyperGraph.getJoinEdges()) { - List hashJoinConjuncts = edge.getHashJoinConjuncts(); + List joinConjunctExpressions = edge.getExpressions(); // shuttle expression in edge for the build of LogicalCompatibilityContext later. // Record the exprId to expr map in the processing to strut info // TODO get exprId to expr map when complex project is ready in join dege - hashJoinConjuncts.forEach(conjunctExpr -> { - ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = - new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(conjunctExpr), ImmutableSet.of(), - ImmutableSet.of(), new BitSet()); - topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); - // Replace expressions by expression map - List replacedExpressions = replaceContext.getReplacedExpressions(); + ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = + new ExpressionLineageReplacer.ExpressionReplaceContext( + joinConjunctExpressions.stream().map(expr -> (Expression) expr) + .collect(Collectors.toList()), + ImmutableSet.of(), ImmutableSet.of(), new BitSet()); + topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); + // Replace expressions by expression map + List replacedExpressions = replaceContext.getReplacedExpressions(); + for (int i = 0; i < replacedExpressions.size(); i++) { putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, - ExpressionPosition.JOIN_EDGE, replacedExpressions.get(0), conjunctExpr); - // Record this, will be used in top level expression shuttle later, see the method - // ExpressionLineageReplacer#visitGroupPlan - namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); - }); - List otherJoinConjuncts = edge.getOtherJoinConjuncts(); - if (!otherJoinConjuncts.isEmpty()) { - return false; + ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i)); } + // Record this, will be used in top level expression shuttle later, see the method + // ExpressionLineageReplacer#visitGroupPlan + namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); } // Collect expression from where in hyper graph hyperGraph.getFilterEdges().forEach(filterEdge -> { @@ -621,9 +618,6 @@ public Boolean visitLogicalJoin(LogicalJoin join if (!checkContext.getSupportJoinTypes().contains(join.getJoinType())) { return false; } - if (!join.getOtherJoinConjuncts().isEmpty()) { - return false; - } return visit(join, checkContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index 42d4cd59b0d6cd..d71e128e9f166d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -29,9 +29,9 @@ import com.google.common.collect.ImmutableBiMap.Builder; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Set; @@ -92,39 +92,25 @@ public static List generate(List sources, List } // relation appear more than once, should cartesian them and power set to correct combination // if query is select * from tableA0, tableA1, materialized view is select * from tableA2, tableA3, - // tableA is the same table used by both query and materialized view - // relationMapping will be - // tableA0 tableA2 - // tableA0 tableA3 - // tableA1 tableA2 - // tableA1 tableA3 - ImmutableList> relationMapping = Sets.cartesianProduct( - sourceMappedRelations, targetMappedRelations) - .stream() - .map(listPair -> Pair.of(listPair.get(0), listPair.get(1))) - .collect(ImmutableList.toImmutableList()); - - // the mapping in relationMappingPowerList should be bi-direction + // the relationMappingPowerList in relationMappingPowerList should be bi-direction // [ - // {tableA0 tableA2, tableA1 tableA3} - // {tableA0 tableA3, tableA1 tableA2} + // {tableA0 -> tableA2, tableA1 -> tableA3} + // {tableA0 -> tableA3, tableA1 -> tableA2} // ] + // query is select * from tableA0, tableA1, tableA4 List> relationMappingPowerList = new ArrayList<>(); - int relationMappingSize = relationMapping.size(); - int relationMappingMinSize = Math.min(sourceMappedRelations.size(), targetMappedRelations.size()); - for (int i = 0; i < relationMappingSize; i++) { - HashBiMap relationBiMap = HashBiMap.create(); - relationBiMap.put(relationMapping.get(i).key(), relationMapping.get(i).value()); - for (int j = i + 1; j < relationMappingSize; j++) { - if (!relationBiMap.containsKey(relationMapping.get(j).key()) - && !relationBiMap.containsValue(relationMapping.get(j).value())) { - relationBiMap.put(relationMapping.get(j).key(), relationMapping.get(j).value()); - } - } - // mapping should contain min num of relation in source or target at least - if (relationBiMap.size() >= relationMappingMinSize) { - relationMappingPowerList.add(relationBiMap); + List> combinations = getUniquePermutation( + sourceMappedRelations.toArray(sourceMappedRelations.toArray(new MappedRelation[0])), + targetMappedRelations.toArray(new MappedRelation[0])); + for (Pair combination : combinations) { + BiMap combinationBiMap = HashBiMap.create(); + MappedRelation[] key = combination.key(); + MappedRelation[] value = combination.value(); + int length = Math.min(key.length, value.length); + for (int i = 0; i < length; i++) { + combinationBiMap.put(key[i], value[i]); } + relationMappingPowerList.add(combinationBiMap); } mappedRelations.add(relationMappingPowerList); } @@ -167,4 +153,58 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(mappedRelationMap); } + + /** + * Permutation and remove duplicated element + * For example: + * Given [1, 4, 5] and [191, 194, 195] + * This would return + * [ + * [(1, 191) (4, 194) (5, 195)], + * [(1, 191) (4, 195) (5, 194)], + * [(1, 194) (4, 191) (5, 195)], + * [(1, 194) (4, 195) (5, 191)], + * [(1, 195) (4, 191) (5, 194)], + * [(1, 195) (4, 194) (5, 191)] + * ] + * */ + private static List> getUniquePermutation( + MappedRelation[] left, MappedRelation[] right) { + boolean needSwap = left.length > right.length; + if (needSwap) { + MappedRelation[] temp = left; + left = right; + right = temp; + } + + boolean[] used = new boolean[right.length]; + MappedRelation[] current = new MappedRelation[left.length]; + List> results = new ArrayList<>(); + backtrack(left, right, 0, used, current, results); + if (needSwap) { + List> tmpResults = results; + results = new ArrayList<>(); + for (Pair relation : tmpResults) { + results.add(Pair.of(relation.value(), relation.key())); + } + } + return results; + } + + private static void backtrack(MappedRelation[] left, MappedRelation[] right, int index, + boolean[] used, MappedRelation[] current, List> results) { + if (index == left.length) { + results.add(Pair.of(Arrays.copyOf(left, left.length), Arrays.copyOf(current, current.length))); + return; + } + + for (int i = 0; i < right.length; i++) { + if (!used[i]) { + used[i] = true; + current[index] = right[i]; + backtrack(left, right, index + 1, used, current, results); + used[i] = false; + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 4ba4d97b95ea9d..1e01e544a2510d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -55,6 +55,7 @@ import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -715,7 +716,8 @@ private Optional preProcess(Expression expression) { if (expression instanceof AggregateFunction || expression instanceof TableGeneratingFunction) { return Optional.of(expression); } - if (expression instanceof PropagateNullable && ExpressionUtils.hasNullLiteral(expression.getArguments())) { + if (ExpressionUtils.hasNullLiteral(expression.getArguments()) + && (expression instanceof PropagateNullLiteral || expression instanceof PropagateNullable)) { return Optional.of(new NullLiteral(expression.getDataType())); } if (!ExpressionUtils.isAllLiteral(expression.getArguments())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java index ef0d86b579cd9d..bf1ce95d1d04d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java @@ -21,6 +21,7 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV3Type; @@ -35,7 +36,7 @@ /** * Divide Expression. */ -public class Divide extends BinaryArithmetic implements AlwaysNullable { +public class Divide extends BinaryArithmetic implements AlwaysNullable, PropagateNullLiteral { public Divide(Expression left, Expression right) { super(ImmutableList.of(left, right), Operator.DIVIDE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionEvaluator.java index 0233b9882663da..0b77118eb1835f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExpressionEvaluator.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.expressions; -import org.apache.doris.catalog.Env; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; @@ -30,8 +29,6 @@ import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; -import org.apache.doris.nereids.types.DataType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; @@ -59,13 +56,11 @@ public enum ExpressionEvaluator { * Evaluate the value of the expression. */ public Expression eval(Expression expression) { - if (!(expression.isConstant() || expression.foldable()) || expression instanceof AggregateFunction) { return expression; } String fnName = null; - DataType ret = expression.getDataType(); if (expression instanceof BinaryArithmetic) { BinaryArithmetic arithmetic = (BinaryArithmetic) expression; fnName = arithmetic.getLegacyOperator().getName(); @@ -77,14 +72,6 @@ public Expression eval(Expression expression) { fnName = function.getName(); } - if ((Env.getCurrentEnv().isNullResultWithOneNullParamFunction(fnName))) { - for (Expression e : expression.children()) { - if (e instanceof NullLiteral) { - return new NullLiteral(ret); - } - } - } - return invoke(expression, fnName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IntegralDivide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IntegralDivide.java index 38a4d89231c57f..a7783dcfb1e0e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IntegralDivide.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IntegralDivide.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import com.google.common.base.Preconditions; @@ -30,7 +31,7 @@ /** * A DIV B */ -public class IntegralDivide extends BinaryArithmetic implements AlwaysNullable { +public class IntegralDivide extends BinaryArithmetic implements AlwaysNullable, PropagateNullLiteral { public IntegralDivide(Expression left, Expression right) { super(ImmutableList.of(left, right), Operator.INT_DIVIDE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java index 569e7cabb3b14e..d152cfe1217d59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DecimalV3Type; @@ -31,7 +32,7 @@ /** * Mod Expression. */ -public class Mod extends BinaryArithmetic implements AlwaysNullable { +public class Mod extends BinaryArithmetic implements AlwaysNullable, PropagateNullLiteral { public Mod(Expression left, Expression right) { super(ImmutableList.of(left, right), Operator.MOD); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/PropagateNullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/PropagateNullLiteral.java new file mode 100644 index 00000000000000..0a844fb81d4866 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/PropagateNullLiteral.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions; + +/** + * If input has NullLiteral, then output would be NullLiteral when folding constant + * + * e.g. Acos(NULL) ==> NULL. + */ +public interface PropagateNullLiteral { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java new file mode 100644 index 00000000000000..16f8232606ff5f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplode.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * PosExplode(array('a','b','c')) generate two columns and three rows with: + * pose column: 0, 1, 2 + * value column: 'a', 'b', 'c' + */ +public class PosExplode extends TableGeneratingFunction implements UnaryExpression, PropagateNullable { + + /** + * constructor with 1 argument. + */ + public PosExplode(Expression arg) { + super("posexplode", arg); + } + + /** + * withChildren. + */ + @Override + public PosExplode withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new PosExplode(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof ArrayType)) { + throw new AnalysisException("only support array type for posexplode function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(new StructType(ImmutableList.of( + new StructField("pos", IntegerType.INSTANCE, true, ""), + new StructField("col", ((ArrayType) child().getDataType()).getItemType(), true, "")))) + .args(child().getDataType())); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPosExplode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java new file mode 100644 index 00000000000000..6d181354f414bc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/PosExplodeOuter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.IntegerType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * PosExplode(array('a','b','c')) generate two columns and three rows with: + * pose column: 0, 1, 2 + * value column: 'a', 'b', 'c' + */ +public class PosExplodeOuter extends TableGeneratingFunction implements UnaryExpression, AlwaysNullable { + + /** + * constructor with 1 argument. + */ + public PosExplodeOuter(Expression arg) { + super("posexplode_outer", arg); + } + + /** + * withChildren. + */ + @Override + public PosExplodeOuter withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new PosExplodeOuter(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof ArrayType)) { + throw new AnalysisException("only support array type for posexplode_outer function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(StructLiteral.constructStructType( + Lists.newArrayList(IntegerType.INSTANCE, + ((ArrayType) child().getDataType()).getItemType()))) + .args(child().getDataType())); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPosExplodeOuter(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java index 2193221c326363..02527c6901e327 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Acos.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'acos'. This class is generated by GenerateFunction. */ public class Acos extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AppendTrailingCharIfAbsent.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AppendTrailingCharIfAbsent.java index 3bfbbca80c7ed5..607f7a4c4d5534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AppendTrailingCharIfAbsent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/AppendTrailingCharIfAbsent.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'append_trailing_char_if_absent'. This class is generated by GenerateFunction. */ public class AppendTrailingCharIfAbsent extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java index 22e1ff59b7df28..fb8d857ff84056 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Asin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'asin'. This class is generated by GenerateFunction. */ public class Asin extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromArray.java index 8304e0684a5241..1e325c80f13bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromArray.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromArray.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; @@ -39,7 +40,7 @@ * ScalarFunction 'bitmap_from_array'. This class is generated by GenerateFunction. */ public class BitmapFromArray extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE).args(ArrayType.of(BigIntType.INSTANCE)), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBase64.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBase64.java index 0994a120ece693..b21ffc9655eb45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBase64.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromBase64.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BitmapType; @@ -36,7 +37,7 @@ * ScalarFunction 'bitmap_from_string'. This class is generated by GenerateFunction. */ public class BitmapFromBase64 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromString.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromString.java index 1e537b084b13c0..748d2ca8a431e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromString.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapFromString.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BitmapType; @@ -36,7 +37,7 @@ * ScalarFunction 'bitmap_from_string'. This class is generated by GenerateFunction. */ public class BitmapFromString extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMax.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMax.java index 7a4cdec7d1f36e..39bb95f943a520 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMax.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMax.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -35,7 +36,7 @@ * ScalarFunction 'bitmap_max'. This class is generated by GenerateFunction. */ public class BitmapMax extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMin.java index 0d1a394b8f9178..03db67e515346f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapMin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -35,7 +36,7 @@ * ScalarFunction 'bitmap_min'. This class is generated by GenerateFunction. */ public class BitmapMin extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetInRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetInRange.java index cbf54a79703237..bff56d499d4dba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetInRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetInRange.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -35,7 +36,7 @@ * ScalarFunction 'bitmap_subset_in_range'. This class is generated by GenerateFunction. */ public class BitmapSubsetInRange extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetLimit.java index a8de375af8d2d8..768e63ef9ad3e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapSubsetLimit.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -35,7 +36,7 @@ * ScalarFunction 'bitmap_subset_limit'. This class is generated by GenerateFunction. */ public class BitmapSubsetLimit extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index 11b44d8acee13a..faee04cacd82b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; @@ -41,7 +42,7 @@ * ScalarFunction 'convert_tz'. This class is generated by GenerateFunction. */ public class ConvertTz extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, Monotonic { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CryptoFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CryptoFunction.java index b3f253f65df80b..151f7ffc7732be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CryptoFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CryptoFunction.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.qe.ConnectContext; @@ -30,7 +31,7 @@ /** CryptoFunction */ public abstract class CryptoFunction extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public CryptoFunction(String name, Expression... arguments) { super(name, arguments); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java index 3f0cdcc6b8abea..3f7ae84975031a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Date.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -39,7 +40,7 @@ * ScalarFunction 'date'. This class is generated by GenerateFunction. */ public class Date extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, Monotonic { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, Monotonic { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java index 43f85542b0eff7..2cc97e7c3afecc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateFormat.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; @@ -38,7 +39,7 @@ * ScalarFunction 'date_format'. This class is generated by GenerateFunction. */ public class DateFormat extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayCeil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayCeil.java index 7f559389f6c9fa..e77c307b523869 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayCeil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayCeil.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -36,7 +37,7 @@ * ScalarFunction 'day_ceil'. This class is generated by GenerateFunction. */ public class DayCeil extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayFloor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayFloor.java index f42371f5f068ed..b7e04e3a374629 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayFloor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayFloor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -36,7 +37,7 @@ * ScalarFunction 'day_floor'. This class is generated by GenerateFunction. */ public class DayFloor extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DigitalMasking.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DigitalMasking.java index 19789fba08fe0b..8d13b4ae5be90d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DigitalMasking.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DigitalMasking.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.VarcharType; @@ -34,7 +35,7 @@ * ScalarFunction 'digital_masking'. */ public class DigitalMasking extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dlog10.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dlog10.java index a967ccacebf9f6..2dfd3e05510d5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dlog10.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dlog10.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'dlog10'. This class is generated by GenerateFunction. */ public class Dlog10 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java index 3caef79776b3bb..3127ab3c3d234f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dsqrt.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'dsqrt'. This class is generated by GenerateFunction. */ public class Dsqrt extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Fmod.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Fmod.java index 08ee2af55f8dc2..28440d3e3d0937 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Fmod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Fmod.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -35,7 +36,7 @@ * ScalarFunction 'fmod'. This class is generated by GenerateFunction. */ public class Fmod extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE, DoubleType.INSTANCE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromBase64.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromBase64.java index 65d4ac0b9c6d10..d5dde4e5ca7c65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromBase64.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromBase64.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'from_base64'. This class is generated by GenerateFunction. */ public class FromBase64 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromDays.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromDays.java index a2b5a420c346c1..c3d19588ce5f2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromDays.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromDays.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'from_days'. This class is generated by GenerateFunction. */ public class FromDays extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(IntegerType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromUnixtime.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromUnixtime.java index 773942ba7b0738..05d5d73e0de1a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromUnixtime.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FromUnixtime.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'from_unixtime'. This class is generated by GenerateFunction. */ public class FromUnixtime extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(BigIntType.INSTANCE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonDouble.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonDouble.java index a0d774033d6dfa..e7698ea5f2e54c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonDouble.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonDouble.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ * ScalarFunction 'get_json_double'. This class is generated by GenerateFunction. */ public class GetJsonDouble extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonString.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonString.java index d9c70f511a6bf3..2ad3f25a095326 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonString.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/GetJsonString.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'get_json_string'. This class is generated by GenerateFunction. */ public class GetJsonString extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourCeil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourCeil.java index 08cef59ce2c753..e76151ef9d682c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourCeil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourCeil.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'hour_ceil'. This class is generated by GenerateFunction. */ public class HourCeil extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourFloor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourFloor.java index 2918b288967ac7..567f0a2dd188da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourFloor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourFloor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'hour_floor'. This class is generated by GenerateFunction. */ public class HourFloor extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ln.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ln.java index 6c8a11bd790d5e..cbc86c0f2dc902 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ln.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ln.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'ln'. This class is generated by GenerateFunction. */ public class Ln extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log10.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log10.java index 671315df0ab1ca..cc129cdee0b73a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log10.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log10.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'log10'. This class is generated by GenerateFunction. */ public class Log10 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log2.java index e98d6a1a0dd624..b7ea3a83f7cdc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Log2.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'log2'. This class is generated by GenerateFunction. */ public class Log2 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lpad.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lpad.java index bda37be50be6f0..c0cf798a8f88af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lpad.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lpad.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IntegerType; @@ -36,7 +37,7 @@ * ScalarFunction 'lpad'. This class is generated by GenerateFunction. */ public class Lpad extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthCeil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthCeil.java index e952f56a48228c..627568cf28a145 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthCeil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthCeil.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -36,7 +37,7 @@ * ScalarFunction 'month_ceil'. This class is generated by GenerateFunction. */ public class MonthCeil extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthFloor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthFloor.java index 8703f4c1b87775..f3b0b66c1396d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthFloor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthFloor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -36,7 +37,7 @@ * ScalarFunction 'month_floor'. This class is generated by GenerateFunction. */ public class MonthFloor extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ParseUrl.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ParseUrl.java index 5c0a432b195758..979563b21cc9b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ParseUrl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ParseUrl.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; @@ -34,7 +35,7 @@ * ScalarFunction 'parse_url'. This class is generated by GenerateFunction. */ public class ParseUrl extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Pmod.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Pmod.java index 808e84ea02600e..cbd685dce37573 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Pmod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Pmod.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -37,7 +38,7 @@ * ScalarFunction 'pmod'. This class is generated by GenerateFunction. */ public class Pmod extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final FunctionSignature BIGINT_SIGNATURE = FunctionSignature.ret(BigIntType.INSTANCE) .args(BigIntType.INSTANCE, BigIntType.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtract.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtract.java index e2c04c89e5c19a..63463f88c8dd42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtract.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -36,7 +37,7 @@ * ScalarFunction 'regexp_extract'. This class is generated by GenerateFunction. */ public class RegexpExtract extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtractAll.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtractAll.java index 4276d0ffa8f04a..e1852f0ce73447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtractAll.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpExtractAll.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'regexp_extract_all'. This class is generated by GenerateFunction. */ public class RegexpExtractAll extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java index fd6be729686dd3..8a12b8d7205df7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'regexp_replace'. This class is generated by GenerateFunction. */ public class RegexpReplace extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java index 2dc2e563df9276..f31cf84cfa18e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'regexp_replace_one'. This class is generated by GenerateFunction. */ public class RegexpReplaceOne extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Repeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Repeat.java index 5ed3b20ddb465b..01e092be47c126 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Repeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Repeat.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IntegerType; @@ -36,7 +37,7 @@ * ScalarFunction 'repeat'. This class is generated by GenerateFunction. */ public class Repeat extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT, IntegerType.INSTANCE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Rpad.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Rpad.java index 3bf29d770d59dc..29fe671ed77bd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Rpad.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Rpad.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IntegerType; @@ -36,7 +37,7 @@ * ScalarFunction 'rpad'. This class is generated by GenerateFunction. */ public class Rpad extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondCeil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondCeil.java index 3dea172f78c1d7..04cd08f4c8ce7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondCeil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondCeil.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'second_ceil'. This class is generated by GenerateFunction. */ public class SecondCeil extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondFloor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondFloor.java index fef3e16e784611..ae0af1106073ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondFloor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondFloor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'second_floor'. This class is generated by GenerateFunction. */ public class SecondFloor extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitPart.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitPart.java index d324848c5c326a..b24382811ee2df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitPart.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitPart.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IntegerType; @@ -36,7 +37,7 @@ * ScalarFunction 'split_part'. This class is generated by GenerateFunction. */ public class SplitPart extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java index f954eb07a54083..2111aef502cd52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sqrt.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -34,7 +35,7 @@ * ScalarFunction 'sqrt'. This class is generated by GenerateFunction. */ public class Sqrt extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngle.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngle.java index ee2bc588c2b61b..31fc908c93827c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngle.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngle.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ */ public class StAngle extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngleSphere.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngleSphere.java index b00f6ef33657da..798166426b0459 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngleSphere.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAngleSphere.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -33,7 +34,7 @@ * ScalarFunction 'st_Angle_sphere'. This class is generated by GenerateFunction. */ public class StAngleSphere extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareKm.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareKm.java index 0fa75a67cb20f3..9e09a7dcf1ec22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareKm.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareKm.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ */ public class StAreaSquareKm extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareMeters.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareMeters.java index 9edd403ca4798a..fe8ad82d8ab136 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareMeters.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAreaSquareMeters.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ */ public class StAreaSquareMeters extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java index d2a8c373c07716..80f8844f1b553f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_astext'. This class is generated by GenerateFunction. */ public class StAstext extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java index e09ac13d62448c..bcfd95796d952a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_contains'. This class is generated by GenerateFunction. */ public class StContains extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistanceSphere.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistanceSphere.java index ce11418a5b8a45..7b29ca3933a780 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistanceSphere.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistanceSphere.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -33,7 +34,7 @@ * ScalarFunction 'st_distance_sphere'. This class is generated by GenerateFunction. */ public class StDistanceSphere extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKB.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKB.java index b9b7dd1f07e03a..7fb2f2ee969ac5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKB.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKB.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_geometryfromwkb'. This class is generated by GenerateFunction. */ public class StGeometryFromWKB extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryfromtext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryfromtext.java index 9fa70ed30eb05f..e67214d54f59f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryfromtext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryfromtext.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_geometryfromtext'. This class is generated by GenerateFunction. */ public class StGeometryfromtext extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeomfromtext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeomfromtext.java index 2af0ed68382452..0ff1ec1922b16c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeomfromtext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeomfromtext.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_geomfromtext'. This class is generated by GenerateFunction. */ public class StGeomfromtext extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLinefromtext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLinefromtext.java index e7c5ab14db806a..129b3818239e9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLinefromtext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLinefromtext.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_linefromtext'. This class is generated by GenerateFunction. */ public class StLinefromtext extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPoint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPoint.java index 1ae9be1dd39d62..8bc91e71185517 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPoint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPoint.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_point'. This class is generated by GenerateFunction. */ public class StPoint extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(DoubleType.INSTANCE, DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolyfromtext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolyfromtext.java index 86f5696795d49f..89ccab1b6cc030 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolyfromtext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolyfromtext.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_polyfromtext'. This class is generated by GenerateFunction. */ public class StPolyfromtext extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolygon.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolygon.java index 3dff4ee4b8083b..34458b0949aab6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolygon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StPolygon.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'st_polygon'. This class is generated by GenerateFunction. */ public class StPolygon extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java index b28e33f1ad26ba..e692fd663e6789 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ * ScalarFunction 'st_x'. This class is generated by GenerateFunction. */ public class StX extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java index cc623b9f5e2a76..bd0730f3cce4fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +37,7 @@ * ScalarFunction 'st_y'. This class is generated by GenerateFunction. */ public class StY extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java index 1fbf15586d5cdd..ff014db6bca7a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -41,7 +42,7 @@ * ScalarFunction 'str_to_date'. This class is generated by GenerateFunction. */ public class StrToDate extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.MAX).args(VarcharType.SYSTEM_DEFAULT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SubReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SubReplace.java index 188ce330e2a8fc..1f0e0a2c20107d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SubReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SubReplace.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; @@ -35,7 +36,7 @@ * ScalarFunction 'sub_replace' */ public class SubReplace extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { + implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java index a1310d3aa16918..7bda334f522710 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDate.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; @@ -37,7 +38,7 @@ * ScalarFunction 'to_date'. This class is generated by GenerateFunction. */ public class ToDate extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDateV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDateV2.java index cfa6fdf76c8b6e..3751049d442642 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDateV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToDateV2.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -35,7 +36,7 @@ * ScalarFunction 'to_datev2'. This class is generated by GenerateFunction. */ public class ToDateV2 extends ScalarFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index 94839b21fe7e27..9fae7c397cada9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -38,6 +38,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeVariantArray; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; +import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdtf; @@ -134,4 +136,12 @@ default R visitExplodeJsonArrayJsonOuter(ExplodeJsonArrayJsonOuter explodeJsonAr default R visitJavaUdtf(JavaUdtf udtf, C context) { return visitTableGeneratingFunction(udtf, context); } + + default R visitPosExplode(PosExplode posExplode, C context) { + return visitTableGeneratingFunction(posExplode, context); + } + + default R visitPosExplodeOuter(PosExplodeOuter posExplodeOuter, C context) { + return visitTableGeneratingFunction(posExplodeOuter, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 8bebb9f2a0e586..427d410f80e596 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -177,5 +177,7 @@ public enum PlanType { PREPARED_COMMAND, EXECUTE_COMMAND, SHOW_CONFIG_COMMAND, + SHOW_VARIABLES_COMMAND, + SHOW_VIEW_COMMAND, REPLAY_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 4dcf3017097a67..d1b8e5a263c503 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -116,4 +118,16 @@ public String treeString() { public Plan withGroupExpression(Optional groupExpression) { throw new RuntimeException("Command do not implement withGroupExpression"); } + + public void verifyCommandSupported() throws DdlException { + // check command has been supported in cloud mode + if (Config.isCloudMode()) { + checkSupportedInCloudMode(); + } + } + + // check if the command is supported in cloud mode + // see checkStmtSupported() in fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java + // override this method if the command is not supported in cloud mode + protected void checkSupportedInCloudMode() throws DdlException {} } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCommand.java new file mode 100644 index 00000000000000..3ce40cdd04d607 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCommand.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.StmtType; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.StmtExecutor; + +/** + * base class for all show commands + */ +public abstract class ShowCommand extends Command implements NoForward { + public ShowCommand(PlanType type) { + super(type); + } + + @Override + public StmtType stmtType() { + return StmtType.SHOW; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + ShowResultSet resultSet = doRun(ctx, executor); + if (resultSet != null) { + if (executor.isProxy()) { + executor.setProxyShowResultSet(resultSet); + } else { + executor.sendResultSet(resultSet); + } + } + } + + public abstract ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception; + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowVariablesCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowVariablesCommand.java new file mode 100644 index 00000000000000..0a24167a2a8fbd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowVariablesCommand.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.SetType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.CaseSensibility; +import org.apache.doris.common.PatternMatcher; +import org.apache.doris.common.PatternMatcherWrapper; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.qe.VariableMgr; + +import java.util.List; + +/** + * ShowVariablesCommand + */ +public class ShowVariablesCommand extends ShowCommand { + private static final String NAME_COL = "Variable_name"; + private static final String VALUE_COL = "Value"; + private static final String DEFAULT_VALUE_COL = "Default_Value"; + private static final String CHANGED_COL = "Changed"; + private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column(NAME_COL, ScalarType.createVarchar(20))) + .addColumn(new Column(VALUE_COL, ScalarType.createVarchar(20))) + .addColumn(new Column(DEFAULT_VALUE_COL, ScalarType.createVarchar(20))) + .addColumn(new Column(CHANGED_COL, ScalarType.createVarchar(20))) + .build(); + + private final SetType type; + private final String pattern; + + public ShowVariablesCommand(SetType type, String pattern) { + super(PlanType.SHOW_VARIABLES_COMMAND); + this.type = type == null ? SetType.DEFAULT : type; + this.pattern = pattern; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowVariablesCommand(this, context); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + PatternMatcher matcher = null; + if (pattern != null) { + matcher = PatternMatcherWrapper.createMysqlPattern(pattern, CaseSensibility.VARIABLES.getCaseSensibility()); + } + List> rows = VariableMgr.dump(type, ctx.getSessionVariable(), matcher); + return new ShowResultSet(META_DATA, rows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowViewCommand.java new file mode 100644 index 00000000000000..08a2d3baae4992 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowViewCommand.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.View; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.PlanUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * ShowViewCommand + */ +public class ShowViewCommand extends ShowCommand { + private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column("View", ScalarType.createVarchar(30))) + .addColumn(new Column("Create View", ScalarType.createVarchar(65535))) + .build(); + + private final String db; + private final TableNameInfo tbl; + private List matchViews = Lists.newArrayList(); + + public ShowViewCommand(String db, TableNameInfo tbl) { + super(PlanType.SHOW_VIEW_COMMAND); + this.db = db; + this.tbl = tbl; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowViewCommand(this, context); + } + + private void validate(ConnectContext ctx) throws Exception { + if (!Strings.isNullOrEmpty(db)) { + // if user specify the `from db`, overwrite the db in `tbl` with this db. + // for example: + // show view from db1.tbl1 from db2; + // will be rewrote to: + // show view from db2.tbl1; + // this act same as in MySQL + tbl.setDb(db); + } + tbl.analyze(ctx); + // disallow external catalog + Util.prohibitExternalCatalog(tbl.getCtl(), this.getClass().getSimpleName()); + + String dbName = tbl.getDb(); + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv( + ConnectContext.get(), tbl.getCtl(), dbName, tbl.getTbl(), PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW VIEW", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbName + ": " + tbl.getTbl()); + } + + Database database = Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbName); + database.getOlapTableOrAnalysisException(tbl.getTbl()); + for (Table table : database.getViewsOnIdOrder()) { + View view = (View) table; + // get table refs instead of get tables because it don't need to check table's validity + List tableNameInfos = getTableNames(ctx, view.getInlineViewDef()); + for (TableNameInfo tableNameInfo : tableNameInfos) { + tableNameInfo.analyze(ctx); + if (tableNameInfo.equals(tbl)) { + matchViews.add(view); + } + } + } + } + + private List getTableNames(ConnectContext ctx, String sql) { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(sql); + StatementContext statementContext = new StatementContext(ctx, + new OriginStatement(sql, 0)); + NereidsPlanner planner = new NereidsPlanner(statementContext); + if (ctx.getStatementContext() == null) { + ctx.setStatementContext(statementContext); + } + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ANALYZED_PLAN); + LogicalPlan logicalPlan = (LogicalPlan) planner.getCascadesContext().getRewritePlan(); + + return PlanUtils.getLogicalScanFromRootPlan(logicalPlan).stream() + .map(plan -> new TableNameInfo(plan.getTable().getFullQualifiers())).collect(Collectors.toList()); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + validate(ctx); + List> rows = Lists.newArrayList(); + for (View view : matchViews) { + view.readLock(); + try { + List createViewStmt = Lists.newArrayList(); + Env.getDdlStmt(view, createViewStmt, null, null, false, true /* hide password */, -1L); + if (!createViewStmt.isEmpty()) { + rows.add(Lists.newArrayList(view.getName(), createViewStmt.get(0))); + } + } finally { + view.readUnlock(); + } + } + return new ShowResultSet(META_DATA, rows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableNameInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableNameInfo.java index 79afb2d1fba476..5d0a42e9807191 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableNameInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableNameInfo.java @@ -129,6 +129,14 @@ public String getDb() { return db; } + /** + * set a new database name + * @param db new database name + */ + public void setDb(String db) { + this.db = db; + } + /** * get table name * @return tableName @@ -162,4 +170,27 @@ public void readFields(DataInput in) throws IOException { db = fromJson.db; tbl = fromJson.tbl; } + + /** + * equals + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TableNameInfo that = (TableNameInfo) o; + return tbl.equals(that.tbl) && db.equals(that.db) && ctl.equals(that.ctl); + } + + /** + * hashCode + */ + @Override + public int hashCode() { + return Objects.hash(tbl, db, ctl); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index b4303bea4a6302..0999c4baa79e3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -31,6 +31,7 @@ import org.apache.doris.load.loadv2.LoadStatistic; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -53,15 +54,22 @@ import org.apache.doris.planner.DataSink; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.qe.Coordinator; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.system.Backend; import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Supplier; /** * insert into select command implementation @@ -135,7 +143,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor * For external uses such as creating a job, only basic analysis is needed without starting a transaction, * in which case this can be set to false. */ - public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor, + public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExecutor, boolean needBeginTransaction) throws Exception { TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); // check auth @@ -159,10 +167,50 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor } OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, targetTableIf, this.logicalQuery, this.insertCtx); LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); - executor.setPlanner(planner); - executor.checkBlockRules(); + + BuildInsertExecutorResult buildResult = planInsertExecutor( + ctx, stmtExecutor, logicalPlanAdapter, targetTableIf + ); + + insertExecutor = buildResult.executor; + + if (!needBeginTransaction) { + targetTableIf.readUnlock(); + return insertExecutor; + } + if (!insertExecutor.isEmptyInsert()) { + insertExecutor.beginTransaction(); + insertExecutor.finalizeSink( + buildResult.planner.getFragments().get(0), buildResult.dataSink, buildResult.physicalSink + ); + } + targetTableIf.readUnlock(); + } catch (Throwable e) { + targetTableIf.readUnlock(); + // the abortTxn in onFail need to acquire table write lock + if (insertExecutor != null) { + insertExecutor.onFail(e); + } + Throwables.propagateIfInstanceOf(e, RuntimeException.class); + throw new IllegalStateException(e.getMessage(), e); + } + + stmtExecutor.setProfileType(ProfileType.LOAD); + // We exposed @StmtExecutor#cancel as a unified entry point for statement interruption, + // so we need to set this here + insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId()); + stmtExecutor.setCoord(insertExecutor.getCoordinator()); + return insertExecutor; + } + + // we should select the factory type first, but we can not initial InsertExecutor at this time, + // because Nereids's DistributePlan are not gernerated, so we return factory and after the + // DistributePlan have been generated, we can create InsertExecutor + private ExecutorFactory selectInsertExecutorFactory( + NereidsPlanner planner, ConnectContext ctx, StmtExecutor stmtExecutor, TableIf targetTableIf) { + try { + stmtExecutor.setPlanner(planner); + stmtExecutor.checkBlockRules(); if (ctx.getConnectType() == ConnectType.MYSQL && ctx.getMysqlChannel() != null) { ctx.getMysqlChannel().reset(); } @@ -170,8 +218,8 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor .>collect(PhysicalSink.class::isInstance)).stream() .findAny(); Preconditions.checkArgument(plan.isPresent(), "insert into command must contain target table"); - PhysicalSink physicalSink = plan.get(); - DataSink sink = planner.getFragments().get(0).getSink(); + PhysicalSink physicalSink = plan.get(); + DataSink dataSink = planner.getFragments().get(0).getSink(); // Transaction insert should reuse the label in the transaction. String label = this.labelName.orElse( ctx.isTxnModel() ? null : String.format("label_%x_%x", ctx.queryId().hi, ctx.queryId().lo)); @@ -179,37 +227,72 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor if (physicalSink instanceof PhysicalOlapTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); OlapTable olapTable = (OlapTable) targetTableIf; + + ExecutorFactory executorFactory; // the insertCtx contains some variables to adjust SinkNode if (ctx.isTxnModel()) { - insertExecutor = new OlapTxnInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert); + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapTxnInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert) + ); } else if (ctx.isGroupCommit()) { - insertExecutor = new OlapGroupCommitInsertExecutor(ctx, olapTable, label, planner, insertCtx, - emptyInsert); + Backend groupCommitBackend = Env.getCurrentEnv() + .getGroupCommitManager() + .selectBackendForGroupCommit(targetTableIf.getId(), ctx); + // set groupCommitBackend for Nereids's DistributePlanner + planner.getCascadesContext().getStatementContext().setGroupCommitMergeBackend(groupCommitBackend); + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapGroupCommitInsertExecutor( + ctx, olapTable, label, planner, insertCtx, emptyInsert, groupCommitBackend + ) + ); } else { - insertExecutor = new OlapInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert); + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert) + ); } - boolean isEnableMemtableOnSinkNode = - olapTable.getTableProperty().getUseSchemaLightChange() - ? insertExecutor.getCoordinator().getQueryOptions().isEnableMemtableOnSinkNode() - : false; - insertExecutor.getCoordinator().getQueryOptions() - .setEnableMemtableOnSinkNode(isEnableMemtableOnSinkNode); + return executorFactory.onCreate(executor -> { + Coordinator coordinator = executor.getCoordinator(); + boolean isEnableMemtableOnSinkNode = olapTable.getTableProperty().getUseSchemaLightChange() + && coordinator.getQueryOptions().isEnableMemtableOnSinkNode(); + coordinator.getQueryOptions().setEnableMemtableOnSinkNode(isEnableMemtableOnSinkNode); + }); } else if (physicalSink instanceof PhysicalHiveTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); HMSExternalTable hiveExternalTable = (HMSExternalTable) targetTableIf; - insertExecutor = new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, - Optional.of(insertCtx.orElse((new HiveInsertCommandContext()))), emptyInsert); + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, + Optional.of(insertCtx.orElse((new HiveInsertCommandContext()))), emptyInsert) + ); // set hive query options } else if (physicalSink instanceof PhysicalIcebergTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); IcebergExternalTable icebergExternalTable = (IcebergExternalTable) targetTableIf; - insertExecutor = new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, - Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext()))), emptyInsert); + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, + Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext()))), + emptyInsert + ) + ); } else if (physicalSink instanceof PhysicalJdbcTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); List cols = ((PhysicalJdbcTableSink) physicalSink).getCols(); - List slots = ((PhysicalJdbcTableSink) physicalSink).getOutput(); + List slots = physicalSink.getOutput(); if (physicalSink.children().size() == 1) { if (physicalSink.child(0) instanceof PhysicalOneRowRelation || physicalSink.child(0) instanceof PhysicalUnion) { @@ -222,36 +305,58 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor } } JdbcExternalTable jdbcExternalTable = (JdbcExternalTable) targetTableIf; - insertExecutor = new JdbcInsertExecutor(ctx, jdbcExternalTable, label, planner, - Optional.of(insertCtx.orElse((new JdbcInsertCommandContext()))), emptyInsert); + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new JdbcInsertExecutor(ctx, jdbcExternalTable, label, planner, + Optional.of(insertCtx.orElse((new JdbcInsertCommandContext()))), emptyInsert) + ); } else { // TODO: support other table types throw new AnalysisException("insert into command only support [olap, hive, iceberg, jdbc] table"); } - if (!needBeginTransaction) { - targetTableIf.readUnlock(); - return insertExecutor; - } - if (!insertExecutor.isEmptyInsert()) { - insertExecutor.beginTransaction(); - insertExecutor.finalizeSink(planner.getFragments().get(0), sink, physicalSink); - } - targetTableIf.readUnlock(); - } catch (Throwable e) { - targetTableIf.readUnlock(); - // the abortTxn in onFail need to acquire table write lock - if (insertExecutor != null) { - insertExecutor.onFail(e); - } - throw e; + } catch (Throwable t) { + Throwables.propagateIfInstanceOf(t, RuntimeException.class); + throw new IllegalStateException(t.getMessage(), t); } + } - executor.setProfileType(ProfileType.LOAD); - // We exposed @StmtExecutor#cancel as a unified entry point for statement interruption, - // so we need to set this here - insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId()); - executor.setCoord(insertExecutor.getCoordinator()); - return insertExecutor; + private BuildInsertExecutorResult planInsertExecutor( + ConnectContext ctx, StmtExecutor stmtExecutor, + LogicalPlanAdapter logicalPlanAdapter, TableIf targetTableIf) throws Throwable { + // the key logical when use new coordinator: + // 1. use NereidsPlanner to generate PhysicalPlan + // 2. use PhysicalPlan to select InsertExecutorFactory, some InsertExecutors want to control + // which backend should be used, for example, OlapGroupCommitInsertExecutor need select + // a backend to do group commit. + // Note: we can not initialize InsertExecutor at this time, because the DistributePlans + // have not been generated, so the NereidsSqlCoordinator can not initial too, + // 3. NereidsPlanner use PhysicalPlan and the provided backend to generate DistributePlan + // 4. ExecutorFactory use the DistributePlan to generate the NereidsSqlCoordinator and InsertExecutor + + StatementContext statementContext = ctx.getStatementContext(); + + AtomicReference executorFactoryRef = new AtomicReference<>(); + NereidsPlanner planner = new NereidsPlanner(statementContext) { + @Override + protected void doDistribute(boolean canUseNereidsDistributePlanner) { + // when enter this method, the step 1 already executed + + // step 2 + executorFactoryRef.set( + selectInsertExecutorFactory(this, ctx, stmtExecutor, targetTableIf) + ); + // step 3 + super.doDistribute(canUseNereidsDistributePlanner); + } + }; + + // step 1, 2, 3 + planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); + + // step 4 + return executorFactoryRef.get().build(); } private void runInternal(ConnectContext ctx, StmtExecutor executor) throws Exception { @@ -289,4 +394,59 @@ private boolean childIsEmptyRelation(PhysicalSink sink) { public StmtType stmtType() { return StmtType.INSERT; } + + /** + * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated + * by NereidsPlanner + */ + private static class ExecutorFactory { + public final NereidsPlanner planner; + public final DataSink dataSink; + public final PhysicalSink physicalSink; + public final Supplier executorSupplier; + private List> createCallback; + + private ExecutorFactory(NereidsPlanner planner, DataSink dataSink, PhysicalSink physicalSink, + Supplier executorSupplier) { + this.planner = planner; + this.dataSink = dataSink; + this.physicalSink = physicalSink; + this.executorSupplier = executorSupplier; + this.createCallback = Lists.newArrayList(); + } + + public static ExecutorFactory from( + NereidsPlanner planner, DataSink dataSink, PhysicalSink physicalSink, + Supplier executorSupplier) { + return new ExecutorFactory(planner, dataSink, physicalSink, executorSupplier); + } + + public ExecutorFactory onCreate(Consumer onCreate) { + this.createCallback.add(onCreate); + return this; + } + + public BuildInsertExecutorResult build() { + AbstractInsertExecutor executor = executorSupplier.get(); + for (Consumer callback : createCallback) { + callback.accept(executor); + } + return new BuildInsertExecutorResult(planner, executor, dataSink, physicalSink); + } + } + + private static class BuildInsertExecutorResult { + private final NereidsPlanner planner; + private final AbstractInsertExecutor executor; + private final DataSink dataSink; + private final PhysicalSink physicalSink; + + public BuildInsertExecutorResult(NereidsPlanner planner, AbstractInsertExecutor executor, DataSink dataSink, + PhysicalSink physicalSink) { + this.planner = planner; + this.executor = executor; + this.dataSink = dataSink; + this.physicalSink = physicalSink; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java index 239328ce93d813..e7b1f4d581892c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java @@ -22,10 +22,8 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; -import org.apache.doris.common.LoadException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.mtmv.MTMVUtil; @@ -39,6 +37,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.system.Backend; import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Strings; @@ -57,10 +56,13 @@ public class OlapGroupCommitInsertExecutor extends OlapInsertExecutor { private static final Logger LOG = LogManager.getLogger(OlapGroupCommitInsertExecutor.class); + private Backend groupCommitBackend; + public OlapGroupCommitInsertExecutor(ConnectContext ctx, Table table, String labelName, NereidsPlanner planner, Optional insertCtx, - boolean emptyInsert) { + boolean emptyInsert, Backend backend) { super(ctx, table, labelName, planner, insertCtx, emptyInsert); + this.groupCommitBackend = backend; } protected static void analyzeGroupCommit(ConnectContext ctx, TableIf table, LogicalPlan logicalQuery, @@ -118,12 +120,8 @@ protected void beforeExec() { LOG.info(msg); throw new AnalysisException(msg); } - try { - this.coordinator.setGroupCommitBe(Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(table.getId(), ctx)); - } catch (LoadException | DdlException e) { - throw new RuntimeException(e); - } + // this is used for old coordinator + this.coordinator.setGroupCommitBe(groupCommitBackend); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java index d6b2d6a360396a..e70629438ac3ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java @@ -74,12 +74,15 @@ public class OlapInsertExecutor extends AbstractInsertExecutor { private static final Logger LOG = LogManager.getLogger(OlapInsertExecutor.class); protected TransactionStatus txnStatus = TransactionStatus.ABORTED; + protected OlapTable olapTable; + /** * constructor */ public OlapInsertExecutor(ConnectContext ctx, Table table, String labelName, NereidsPlanner planner, Optional insertCtx, boolean emptyInsert) { super(ctx, table, labelName, planner, insertCtx, emptyInsert); + this.olapTable = (OlapTable) table; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributeContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributeContext.java new file mode 100644 index 00000000000000..749417f792a111 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributeContext.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.distribute; + +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; + +/** DistributeContext */ +public class DistributeContext { + public final DistributedPlanWorkerManager workerManager; + public final SelectedWorkers selectedWorkers; + + public DistributeContext() { + this.workerManager = new BackendDistributedPlanWorkerManager(); + this.selectedWorkers = new SelectedWorkers(workerManager); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java index ceef281c0fc534..12ab8b42eaab61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java @@ -17,34 +17,74 @@ package org.apache.doris.nereids.trees.plans.distribute; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.DummyWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJobBuilder; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.StaticAssignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJobBuilder; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedScanBucketOlapTableJob; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.DataStreamSink; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.MultiCastDataSink; +import org.apache.doris.planner.MultiCastPlanFragment; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TPartitionType; +import org.apache.doris.thrift.TUniqueId; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.SetMultimap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; /** DistributePlanner */ public class DistributePlanner { - private final List fragments; + private static final Logger LOG = LogManager.getLogger(DistributePlanner.class); + private final StatementContext statementContext; private final FragmentIdMapping idToFragments; - public DistributePlanner(List fragments) { - this.fragments = Objects.requireNonNull(fragments, "fragments can not be null"); + public DistributePlanner(StatementContext statementContext, List fragments) { + this.statementContext = Objects.requireNonNull(statementContext, "statementContext can not be null"); this.idToFragments = FragmentIdMapping.buildFragmentMapping(fragments); } + /** plan */ public FragmentIdMapping plan() { - FragmentIdMapping fragmentJobs = UnassignedJobBuilder.buildJobs(idToFragments); - ListMultimap instanceJobs = AssignedJobBuilder.buildJobs(fragmentJobs); - return buildDistributePlans(fragmentJobs, instanceJobs); + try { + FragmentIdMapping fragmentJobs + = UnassignedJobBuilder.buildJobs(statementContext, idToFragments); + ListMultimap instanceJobs = AssignedJobBuilder.buildJobs(fragmentJobs); + FragmentIdMapping distributedPlans = buildDistributePlans(fragmentJobs, instanceJobs); + FragmentIdMapping linkedPlans = linkPlans(distributedPlans); + updateProfileIfPresent(SummaryProfile::setAssignFragmentTime); + return linkedPlans; + } catch (Throwable t) { + LOG.error("Failed to build distribute plans", t); + throw t; + } } private FragmentIdMapping buildDistributePlans( @@ -58,9 +98,151 @@ private FragmentIdMapping buildDistributePlans( UnassignedJob fragmentJob = idToUnassignedJobs.get(fragmentId); List instanceJobs = idToAssignedJobs.get(fragmentId); - List childrenPlans = idToDistributedPlans.getByChildrenFragments(fragment); - idToDistributedPlans.put(fragmentId, new PipelineDistributedPlan(fragmentJob, instanceJobs, childrenPlans)); + SetMultimap exchangeNodeToChildren = LinkedHashMultimap.create(); + for (PlanFragment childFragment : fragment.getChildren()) { + if (childFragment instanceof MultiCastPlanFragment) { + for (ExchangeNode exchangeNode : ((MultiCastPlanFragment) childFragment).getDestNodeList()) { + if (exchangeNode.getFragment() == fragment) { + exchangeNodeToChildren.put( + exchangeNode, idToDistributedPlans.get(childFragment.getFragmentId()) + ); + } + } + } else { + exchangeNodeToChildren.put( + childFragment.getDestNode(), + idToDistributedPlans.get(childFragment.getFragmentId()) + ); + } + } + + idToDistributedPlans.put(fragmentId, + new PipelineDistributedPlan(fragmentJob, instanceJobs, exchangeNodeToChildren) + ); } return (FragmentIdMapping) idToDistributedPlans; } + + private FragmentIdMapping linkPlans(FragmentIdMapping plans) { + boolean enableShareHashTableForBroadcastJoin = statementContext.getConnectContext() + .getSessionVariable() + .enableShareHashTableForBroadcastJoin; + for (DistributedPlan receiverPlan : plans.values()) { + for (Entry link : receiverPlan.getInputs().entries()) { + linkPipelinePlan( + (PipelineDistributedPlan) receiverPlan, + (PipelineDistributedPlan) link.getValue(), + link.getKey(), + enableShareHashTableForBroadcastJoin + ); + } + } + return plans; + } + + // set shuffle destinations + private void linkPipelinePlan( + PipelineDistributedPlan receiverPlan, + PipelineDistributedPlan senderPlan, + ExchangeNode linkNode, + boolean enableShareHashTableForBroadcastJoin) { + + List receiverInstances = filterInstancesWhichCanReceiveDataFromRemote( + receiverPlan, enableShareHashTableForBroadcastJoin, linkNode); + if (linkNode.getPartitionType() == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED) { + receiverInstances = getDestinationsByBuckets(receiverPlan, receiverInstances); + } + + DataSink sink = senderPlan.getFragmentJob().getFragment().getSink(); + if (sink instanceof MultiCastDataSink) { + MultiCastDataSink multiCastDataSink = (MultiCastDataSink) sink; + receiverPlan.getFragmentJob().getFragment().setOutputPartition(multiCastDataSink.getOutputPartition()); + for (DataStreamSink realSink : multiCastDataSink.getDataStreamSinks()) { + if (realSink.getExchNodeId() == linkNode.getId()) { + senderPlan.addDestinations(realSink, receiverInstances); + break; + } + } + } else { + senderPlan.addDestinations(sink, receiverInstances); + } + } + + private List getDestinationsByBuckets( + PipelineDistributedPlan joinSide, + List receiverInstances) { + UnassignedScanBucketOlapTableJob bucketJob = (UnassignedScanBucketOlapTableJob) joinSide.getFragmentJob(); + int bucketNum = bucketJob.getOlapScanNodes().get(0).getBucketNum(); + return sortDestinationInstancesByBuckets(joinSide, receiverInstances, bucketNum); + } + + private List filterInstancesWhichCanReceiveDataFromRemote( + PipelineDistributedPlan receiverPlan, + boolean enableShareHashTableForBroadcastJoin, + ExchangeNode linkNode) { + boolean useLocalShuffle = receiverPlan.getInstanceJobs().stream() + .anyMatch(LocalShuffleAssignedJob.class::isInstance); + if (useLocalShuffle) { + return getFirstInstancePerShareScan(receiverPlan); + } else if (enableShareHashTableForBroadcastJoin && linkNode.isRightChildOfBroadcastHashJoin()) { + return getFirstInstancePerWorker(receiverPlan.getInstanceJobs()); + } else { + return receiverPlan.getInstanceJobs(); + } + } + + private List sortDestinationInstancesByBuckets( + PipelineDistributedPlan plan, List unsorted, int bucketNum) { + AssignedJob[] instances = new AssignedJob[bucketNum]; + for (AssignedJob instanceJob : unsorted) { + BucketScanSource bucketScanSource = (BucketScanSource) instanceJob.getScanSource(); + for (Integer bucketIndex : bucketScanSource.bucketIndexToScanNodeToTablets.keySet()) { + if (instances[bucketIndex] != null) { + throw new IllegalStateException( + "Multi instances scan same buckets: " + instances[bucketIndex] + " and " + instanceJob + ); + } + instances[bucketIndex] = instanceJob; + } + } + + for (int i = 0; i < instances.length; i++) { + if (instances[i] == null) { + instances[i] = new StaticAssignedJob( + i, + new TUniqueId(-1, -1), + plan.getFragmentJob(), + DummyWorker.INSTANCE, + new DefaultScanSource(ImmutableMap.of()) + ); + } + } + return Arrays.asList(instances); + } + + private List getFirstInstancePerShareScan(PipelineDistributedPlan plan) { + List canReceiveDataFromRemote = Lists.newArrayListWithCapacity(plan.getInstanceJobs().size()); + for (AssignedJob instanceJob : plan.getInstanceJobs()) { + LocalShuffleAssignedJob localShuffleJob = (LocalShuffleAssignedJob) instanceJob; + if (!localShuffleJob.receiveDataFromLocal) { + canReceiveDataFromRemote.add(localShuffleJob); + } + } + return canReceiveDataFromRemote; + } + + private List getFirstInstancePerWorker(List instances) { + Map firstInstancePerWorker = Maps.newLinkedHashMap(); + for (AssignedJob instance : instances) { + firstInstancePerWorker.putIfAbsent(instance.getAssignedWorker(), instance); + } + return Utils.fastToImmutableList(firstInstancePerWorker.values()); + } + + private void updateProfileIfPresent(Consumer profileAction) { + Optional.ofNullable(ConnectContext.get()) + .map(ConnectContext::getExecutor) + .map(StmtExecutor::getSummaryProfile) + .ifPresent(profileAction); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java index 8f176e3caccc7c..5b934bf7bfe534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java @@ -19,7 +19,9 @@ import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; -import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.ExchangeNode; + +import com.google.common.collect.SetMultimap; import java.util.List; import java.util.Objects; @@ -28,11 +30,19 @@ @lombok.Getter public abstract class DistributedPlan extends AbstractTreeNode { protected final UnassignedJob fragmentJob; - protected final List inputs; + protected final SetMultimap inputs; - public DistributedPlan(UnassignedJob fragmentJob, List inputs) { + public DistributedPlan(UnassignedJob fragmentJob, SetMultimap inputs) { this.fragmentJob = Objects.requireNonNull(fragmentJob, "fragmentJob can not be null"); - this.inputs = Utils.fastToImmutableList(Objects.requireNonNull(inputs, "inputs can not be null")); + this.inputs = Objects.requireNonNull(inputs, "inputs can not be null"); + } + + public UnassignedJob getFragmentJob() { + return fragmentJob; + } + + public SetMultimap getInputs() { + return inputs; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java index 95bf36051d2033..12845fbea480aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.distribute; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanFragmentId; @@ -68,4 +69,13 @@ public static FragmentIdMapping buildFragmentMapping(List List valueList() { + return (List) Utils.fastToImmutableList(values()); + } + + public T last() { + List valueList = valueList(); + return valueList.get(valueList.size() - 1); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java index 13f903d8a2cf05..37a00bd0052607 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java @@ -17,32 +17,69 @@ package org.apache.doris.nereids.trees.plans.distribute; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.ExchangeNode; import org.apache.doris.thrift.TExplainLevel; +import com.google.common.collect.Maps; +import com.google.common.collect.SetMultimap; + import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; /** PipelineDistributedPlan */ public class PipelineDistributedPlan extends DistributedPlan { protected final List instanceJobs; + // current, we only support all instances of the same fragment reuse the same destination + private Map> destinations; + /** constructor */ public PipelineDistributedPlan( UnassignedJob fragmentJob, List instanceJobs, - List inputs) { + SetMultimap inputs) { super(fragmentJob, inputs); + + long localShuffleInstanceNum = instanceJobs.stream() + .filter(LocalShuffleAssignedJob.class::isInstance) + .count(); + if (localShuffleInstanceNum != 0 && localShuffleInstanceNum != instanceJobs.size()) { + throw new IllegalStateException("LocalShuffleAssignedJob num is " + localShuffleInstanceNum + + ", should be 0 or " + instanceJobs.size() + ", fragmentJob: " + fragmentJob + + ", instances: " + instanceJobs); + } + this.instanceJobs = Utils.fastToImmutableList( Objects.requireNonNull(instanceJobs, "instanceJobs can not be null") ); + this.destinations = Maps.newLinkedHashMap(); } public List getInstanceJobs() { return instanceJobs; } + public Map> getDestinations() { + return destinations; + } + + public void addDestinations(DataSink sink, List destinations) { + this.destinations.put(sink, destinations); + } + + @Override + public int hashCode() { + return fragmentJob.getFragment().getFragmentId().asInt(); + } + @Override public String toString(int displayFragmentId) { StringBuilder instancesStr = new StringBuilder(); @@ -58,10 +95,25 @@ public String toString(int displayFragmentId) { fragmentJob.getFragment().getExplainString(TExplainLevel.VERBOSE).trim(), " " ); + String destinationStr = destinations.entrySet() + .stream() + .map(kv -> { + AtomicInteger bucketNum = new AtomicInteger(0); + String str = kv.getValue() + .stream() + .map(destination -> " " + + "#" + bucketNum.getAndIncrement() + ": " + + DebugUtil.printId(destination.instanceId())) + .collect(Collectors.joining(",\n")); + return " Exchange " + kv.getKey().getExchNodeId().asInt() + + ": [" + (str.isEmpty() ? "" : "\n" + str + "\n ") + "]"; + }) + .collect(Collectors.joining(",\n")); return "PipelineDistributedPlan(\n" + " id: " + displayFragmentId + ",\n" + " parallel: " + instanceJobs.size() + ",\n" + " fragmentJob: " + fragmentJob + ",\n" + + " destinations: [" + (destinationStr.isEmpty() ? "" : "\n" + destinationStr + "\n ") + "],\n" + " fragment: {\n" + " " + explainString + "\n" + " },\n" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/SelectedWorkers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/SelectedWorkers.java new file mode 100644 index 00000000000000..c4375f51beb51d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/SelectedWorkers.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.distribute; + +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.util.Map; +import java.util.Set; + +/** SelectedWorkers */ +public class SelectedWorkers { + private final DistributedPlanWorkerManager workerManager; + private final Map usedWorkersAddressToBackendID; + private final Set usedWorkers; + + public SelectedWorkers(DistributedPlanWorkerManager workerManager) { + this.workerManager = workerManager; + this.usedWorkersAddressToBackendID = Maps.newLinkedHashMap(); + this.usedWorkers = Sets.newLinkedHashSet(); + } + + /** onCreateAssignedJob */ + public void onCreateAssignedJob(AssignedJob assignedJob) { + BackendWorker worker = (BackendWorker) assignedJob.getAssignedWorker(); + if (usedWorkers.add(worker)) { + Backend backend = worker.getBackend(); + usedWorkersAddressToBackendID.put( + new TNetworkAddress(backend.getHost(), backend.getBePort()), backend.getId() + ); + } + } + + /** tryToSelectRandomUsedWorker */ + public DistributedPlanWorker tryToSelectRandomUsedWorker() { + if (usedWorkers.isEmpty()) { + return workerManager.randomAvailableWorker(); + } else { + long id = workerManager.randomAvailableWorker(usedWorkersAddressToBackendID); + return workerManager.getWorker(id); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java index 7acbe653e983d5..5a2d9bb1e82779 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java @@ -22,15 +22,17 @@ import org.apache.doris.common.Reference; import org.apache.doris.qe.SimpleScheduler; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; +import java.util.Map; import java.util.function.Supplier; /** BackendWorkerManager */ public class BackendDistributedPlanWorkerManager implements DistributedPlanWorkerManager { - private final Supplier> backends = Suppliers.memoize(() -> { + private final Supplier> allClusterBackends = Suppliers.memoize(() -> { try { return Env.getCurrentSystemInfo().getAllBackendsByAllCluster(); } catch (Exception t) { @@ -38,9 +40,21 @@ public class BackendDistributedPlanWorkerManager implements DistributedPlanWorke } }); + private final Supplier> currentClusterBackends = Suppliers.memoize(() -> { + try { + return Env.getCurrentSystemInfo().getBackendsByCurrentCluster(); + } catch (Exception t) { + throw new NereidsException("Can not get backends: " + t, t); + } + }); + + public boolean isCurrentClusterBackend(long backendId) { + return currentClusterBackends.get().containsKey(backendId); + } + @Override public DistributedPlanWorker getWorker(long backendId) { - ImmutableMap backends = this.backends.get(); + ImmutableMap backends = this.allClusterBackends.get(); Backend backend = backends.get(backendId); if (backend == null) { throw new IllegalStateException("Backend " + backendId + " is not exist"); @@ -52,7 +66,7 @@ public DistributedPlanWorker getWorker(long backendId) { public DistributedPlanWorker randomAvailableWorker() { try { Reference selectedBackendId = new Reference<>(); - ImmutableMap backends = this.backends.get(); + ImmutableMap backends = this.currentClusterBackends.get(); SimpleScheduler.getHost(backends, selectedBackendId); Backend selctedBackend = backends.get(selectedBackendId.getRef()); return new BackendWorker(selctedBackend); @@ -60,4 +74,10 @@ public DistributedPlanWorker randomAvailableWorker() { throw new NereidsException("Can not get backends: " + t, t); } } + + @Override + public long randomAvailableWorker(Map addressToBackendID) { + TNetworkAddress backend = SimpleScheduler.getHostByCurrentBackend(addressToBackendID); + return addressToBackendID.get(backend); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java index 702a00dd358d29..63c73b50edcd07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java @@ -26,7 +26,11 @@ public class BackendWorker implements DistributedPlanWorker { private final Backend backend; public BackendWorker(Backend backend) { - this.backend = backend; + this.backend = Objects.requireNonNull(backend, "backend can not be null"); + } + + public Backend getBackend() { + return backend; } @Override @@ -39,6 +43,16 @@ public String address() { return backend.getAddress(); } + @Override + public String brpcAddress() { + return backend.getHost() + brpcPort(); + } + + @Override + public int brpcPort() { + return backend.getBrpcPort(); + } + @Override public String host() { return backend.getHost(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java index c86675a6dab27c..79f8b482d88c2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java @@ -30,6 +30,10 @@ public interface DistributedPlanWorker extends Comparable int port(); + String brpcAddress(); + + int brpcPort(); + // whether is this worker alive? boolean available(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java index f0604d13e346fb..f2ea4ebd0c699c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java @@ -17,9 +17,15 @@ package org.apache.doris.nereids.trees.plans.distribute.worker; +import org.apache.doris.thrift.TNetworkAddress; + +import java.util.Map; + /** DistributedPlanWorkerManager */ public interface DistributedPlanWorkerManager { DistributedPlanWorker getWorker(long backendId); DistributedPlanWorker randomAvailableWorker(); + + long randomAvailableWorker(Map addressToBackendID); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DummyWorker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DummyWorker.java new file mode 100644 index 00000000000000..9a7d2f42476233 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DummyWorker.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.distribute.worker; + +/** DummyWorker */ +public class DummyWorker implements DistributedPlanWorker { + public static final DummyWorker INSTANCE = new DummyWorker(); + + private DummyWorker() {} + + @Override + public long id() { + return 0; + } + + @Override + public String address() { + return "0.0.0.0:0"; + } + + @Override + public String host() { + return "0.0.0.0"; + } + + @Override + public int port() { + return 0; + } + + @Override + public String brpcAddress() { + return "0.0.0.0:0"; + } + + @Override + public int brpcPort() { + return 0; + } + + @Override + public boolean available() { + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java index 4cec3af18da182..89931daefe8253 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java @@ -29,6 +29,7 @@ import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExternalScanRange; import org.apache.doris.thrift.TFileRangeDesc; @@ -40,10 +41,13 @@ import org.apache.doris.thrift.TScanRangeParams; import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -75,16 +79,24 @@ public DistributedPlanWorker selectMinWorkloadWorker(List } @Override - public Map selectReplicaAndWorkerWithoutBucket(ScanNode scanNode) { + public Map selectReplicaAndWorkerWithoutBucket( + ScanNode scanNode, ConnectContext context) { + Map workerScanRanges = Maps.newLinkedHashMap(); // allScanRangesLocations is all scan ranges in all partition which need to scan List allScanRangesLocations = scanNode.getScanRangeLocations(0); + + boolean orderedScanRangeLocations = shouldSortTablets(ImmutableList.of(scanNode), context); + if (orderedScanRangeLocations) { + allScanRangesLocations = sortScanRanges(allScanRangesLocations); + } + for (TScanRangeLocations onePartitionOneScanRangeLocation : allScanRangesLocations) { // usually, the onePartitionOneScanRangeLocation is a tablet in one partition long bytes = getScanRangeSize(scanNode, onePartitionOneScanRangeLocation); WorkerScanRanges assigned = selectScanReplicaAndMinWorkloadWorker( - onePartitionOneScanRangeLocation, bytes); + onePartitionOneScanRangeLocation, bytes, orderedScanRangeLocations); UninstancedScanSource scanRanges = workerScanRanges.computeIfAbsent( assigned.worker, w -> new UninstancedScanSource( @@ -99,22 +111,37 @@ public Map selectReplicaAndWorkerW @Override public Map selectReplicaAndWorkerWithBucket( - UnassignedScanBucketOlapTableJob unassignedJob) { + UnassignedScanBucketOlapTableJob unassignedJob, ConnectContext context) { PlanFragment fragment = unassignedJob.getFragment(); - List scanNodes = unassignedJob.getScanNodes(); + List scanNodes = unassignedJob.getScanNodes(); List olapScanNodes = unassignedJob.getOlapScanNodes(); - BiFunction> bucketScanRangeSupplier = bucketScanRangeSupplier(); + boolean orderedScanRangeLocations = shouldSortTablets(scanNodes, context); + if (orderedScanRangeLocations) { + List sortedOlapScanNodes = Lists.newArrayList(olapScanNodes); + sortedOlapScanNodes.sort(Comparator.comparing(node -> node.getId().asInt())); + scanNodes = olapScanNodes = sortedOlapScanNodes; + } + + BiFunction> bucketScanRangeSupplier + = bucketScanRangeSupplier(orderedScanRangeLocations); Function> bucketBytesSupplier = bucketBytesSupplier(); // all are olap scan nodes if (!scanNodes.isEmpty() && scanNodes.size() == olapScanNodes.size()) { if (olapScanNodes.size() == 1 && fragment.hasBucketShuffleJoin()) { - return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier); + return selectForBucket( + unassignedJob, scanNodes, bucketScanRangeSupplier, + bucketBytesSupplier, orderedScanRangeLocations + ); } else if (fragment.hasColocatePlanNode()) { - return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier); + return selectForBucket( + unassignedJob, scanNodes, bucketScanRangeSupplier, + bucketBytesSupplier, orderedScanRangeLocations + ); } } else if (olapScanNodes.isEmpty() && fragment.getDataPartition() == DataPartition.UNPARTITIONED) { - return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier); + return selectForBucket( + unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier, orderedScanRangeLocations); } throw new IllegalStateException( "Illegal bucket shuffle join or colocate join in fragment:\n" @@ -122,10 +149,16 @@ public Map selectReplicaAndWorkerW ); } - private BiFunction> bucketScanRangeSupplier() { + private BiFunction> bucketScanRangeSupplier( + boolean shouldSortTablets) { return (scanNode, bucketIndex) -> { if (scanNode instanceof OlapScanNode) { - return (List) ((OlapScanNode) scanNode).bucketSeq2locations.get(bucketIndex); + List scanRangeLocations + = ((OlapScanNode) scanNode).bucketSeq2locations.get(bucketIndex); + if (shouldSortTablets) { + scanRangeLocations = sortScanRanges(scanRangeLocations); + } + return scanRangeLocations; } else { // the backend is selected by XxxScanNode.createScanRangeLocations() return scanNode.getScanRangeLocations(0); @@ -145,9 +178,10 @@ private Function> bucketBytesSupplier() { } private Map selectForBucket( - UnassignedJob unassignedJob, List scanNodes, + UnassignedJob unassignedJob, List scanNodes, BiFunction> bucketScanRangeSupplier, - Function> bucketBytesSupplier) { + Function> bucketBytesSupplier, + boolean orderedScanRangeLocations) { Map assignment = Maps.newLinkedHashMap(); Map bucketIndexToBytes = computeEachBucketScanBytes(scanNodes, bucketBytesSupplier); @@ -162,7 +196,10 @@ private Map selectForBucket( = bucketScanRangeSupplier.apply(scanNode, bucketIndex); if (!allPartitionTabletsInOneBucketInOneTable.isEmpty()) { WorkerScanRanges replicaAndWorker = selectScanReplicaAndMinWorkloadWorker( - allPartitionTabletsInOneBucketInOneTable.get(0), allScanNodeScanBytesInOneBucket); + allPartitionTabletsInOneBucketInOneTable.get(0), + allScanNodeScanBytesInOneBucket, + orderedScanRangeLocations + ); selectedWorker = replicaAndWorker.worker; break; } @@ -199,8 +236,13 @@ private Map selectForBucket( } private WorkerScanRanges selectScanReplicaAndMinWorkloadWorker( - TScanRangeLocations tabletLocation, long tabletBytes) { + TScanRangeLocations tabletLocation, long tabletBytes, boolean orderedScanRangeLocations) { List replicaLocations = tabletLocation.getLocations(); + if (orderedScanRangeLocations) { + replicaLocations = Lists.newArrayList(replicaLocations); + Collections.sort(replicaLocations); + } + int replicaNum = replicaLocations.size(); WorkerWorkload minWorkload = new WorkerWorkload(Integer.MAX_VALUE, Long.MAX_VALUE); DistributedPlanWorker minWorkLoadWorker = null; @@ -269,7 +311,7 @@ private List> filterReplicaByWorkerInBucket( } private Map computeEachBucketScanBytes( - List scanNodes, Function> bucketBytesSupplier) { + List scanNodes, Function> bucketBytesSupplier) { Map bucketIndexToBytes = Maps.newLinkedHashMap(); for (ScanNode scanNode : scanNodes) { Map bucketSeq2Bytes = bucketBytesSupplier.apply(scanNode); @@ -299,8 +341,11 @@ private long getScanRangeSize(ScanNode scanNode, TScanRangeLocations scanRangeLo if (extScanRange != null) { TFileScanRange fileScanRange = extScanRange.getFileScanRange(); long size = 0; - for (TFileRangeDesc range : fileScanRange.getRanges()) { - size += range.getSize(); + List ranges = fileScanRange.getRanges(); + if (ranges != null) { + for (TFileRangeDesc range : ranges) { + size += range.getSize(); + } } return size; } @@ -308,6 +353,22 @@ private long getScanRangeSize(ScanNode scanNode, TScanRangeLocations scanRangeLo return 0L; } + private boolean shouldSortTablets(List scanNodes, ConnectContext context) { + return scanNodes.stream().allMatch(OlapScanNode.class::isInstance) + && context.getSessionVariable().enableOrderedScanRangeLocations; + } + + private List sortScanRanges(List tablets) { + tablets = Lists.newArrayList(tablets); + tablets.sort( + (p1, p2) -> org.apache.thrift.TBaseHelper.compareTo( + p1.getScanRange().getPaloScanRange().tablet_id, + p2.getScanRange().getPaloScanRange().tablet_id + ) + ); + return tablets; + } + private static class WorkerWorkload implements Comparable { private int taskNum; private long scanBytes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java index 40876a09e44301..34e524dc8536f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedScanBucketOlapTableJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UninstancedScanSource; import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TScanRangeParams; @@ -35,7 +36,8 @@ public interface ScanWorkerSelector { // for a scan node, select replica for each scan range(denote tablet if the ScanNode is OlapScanNode), // use the replica location to build a worker execute the instance - Map selectReplicaAndWorkerWithoutBucket(ScanNode scanNode); + Map selectReplicaAndWorkerWithoutBucket( + ScanNode scanNode, ConnectContext context); // return // key: Worker, the backend which will process this fragment @@ -49,14 +51,12 @@ public interface ScanWorkerSelector { // and distributed by hash(id) buckets 10. And, so, there has 10 buckets from bucket 0 to // bucket 9, and every bucket contains two tablets, because there are two partitions. Map selectReplicaAndWorkerWithBucket( - UnassignedScanBucketOlapTableJob unassignedJob); + UnassignedScanBucketOlapTableJob unassignedJob, ConnectContext context); static TScanRangeParams buildScanReplicaParams( TScanRangeLocations tabletLocation, TScanRangeLocation replicaLocation) { TScanRangeParams replicaParam = new TScanRangeParams(); replicaParam.scan_range = tabletLocation.scan_range; - // Volume is optional, so we need to set the value and the is-set bit - replicaParam.setVolumeId(replicaLocation.volume_id); return replicaParam; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java index f53ee614523379..8003abf0e7dae2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.util.Utils; import org.apache.doris.planner.ExchangeNode; @@ -31,14 +32,16 @@ /** AbstractUnassignedJob */ public abstract class AbstractUnassignedJob extends AbstractTreeNode implements UnassignedJob { + protected final StatementContext statementContext; protected final PlanFragment fragment; protected final List scanNodes; protected final ListMultimap exchangeToChildJob; /** AbstractUnassignedJob */ - public AbstractUnassignedJob(PlanFragment fragment, List scanNodes, - ListMultimap exchangeToChildJob) { + public AbstractUnassignedJob(StatementContext statementContext, PlanFragment fragment, + List scanNodes, ListMultimap exchangeToChildJob) { super(Utils.fastToImmutableList(exchangeToChildJob.values())); + this.statementContext = Objects.requireNonNull(statementContext, "statementContext can not be null"); this.fragment = Objects.requireNonNull(fragment, "fragment can not be null"); this.scanNodes = Utils.fastToImmutableList( Objects.requireNonNull(scanNodes, "scanNodes can not be null") @@ -47,6 +50,11 @@ public AbstractUnassignedJob(PlanFragment fragment, List scanNodes, = Objects.requireNonNull(exchangeToChildJob, "exchangeToChildJob can not be null"); } + @Override + public StatementContext getStatementContext() { + return statementContext; + } + @Override public PlanFragment getFragment() { return fragment; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java index 2557966b6e84a8..ef3236690f1b34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.planner.ExchangeNode; @@ -39,21 +41,21 @@ public abstract class AbstractUnassignedScanJob extends AbstractUnassignedJob { protected final AtomicInteger shareScanIdGenerator = new AtomicInteger(); - public AbstractUnassignedScanJob(PlanFragment fragment, + public AbstractUnassignedScanJob(StatementContext statementContext, PlanFragment fragment, List scanNodes, ListMultimap exchangeToChildJob) { - super(fragment, scanNodes, exchangeToChildJob); + super(statementContext, fragment, scanNodes, exchangeToChildJob); } @Override - public List computeAssignedJobs(DistributedPlanWorkerManager workerManager, - ListMultimap inputJobs) { + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { - Map workerToScanSource = multipleMachinesParallelization( - workerManager, inputJobs); + Map workerToScanSource + = multipleMachinesParallelization(distributeContext, inputJobs); - List assignedJobs = insideMachineParallelization(workerToScanSource, inputJobs, workerManager); + List assignedJobs = insideMachineParallelization(workerToScanSource, inputJobs, distributeContext); - return fillUpAssignedJobs(assignedJobs, workerManager, inputJobs); + return fillUpAssignedJobs(assignedJobs, distributeContext.workerManager, inputJobs); } protected List fillUpAssignedJobs( @@ -64,14 +66,15 @@ protected List fillUpAssignedJobs( } protected abstract Map multipleMachinesParallelization( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs); + DistributeContext distributeContext, ListMultimap inputJobs); protected List insideMachineParallelization( Map workerToScanRanges, - ListMultimap inputJobs, DistributedPlanWorkerManager workerManager) { + ListMultimap inputJobs, + DistributeContext distributeContext) { - ConnectContext context = ConnectContext.get(); - boolean useLocalShuffleToAddParallel = useLocalShuffleToAddParallel(workerToScanRanges); + ConnectContext context = statementContext.getConnectContext(); + boolean useLocalShuffleToAddParallel = fragment.useSerialSource(ConnectContext.get()); int instanceIndexInFragment = 0; List instances = Lists.newArrayList(); for (Entry entry : workerToScanRanges.entrySet()) { @@ -98,27 +101,31 @@ protected List insideMachineParallelization( scanNodes, 1 ); - // Some tablets too big, we need add parallel to process these tablets after scan, - // for example, use one OlapScanNode to scan data, and use some local instances - // to process Aggregation parallel. We call it `share scan`. Backend will know this - // instances share the same ScanSource, and will not scan same data multiple times. + // when data not big, but aggregation too slow, we will use 1 instance to scan data, + // and use more instances (to ***add parallel***) to process aggregate. + // We call it `ignore data distribution` of `share scan`. Backend will know this instances + // share the same ScanSource, and will not scan same data multiple times. // // +-------------------------------- same fragment in one host -------------------------------------+ // | instance1 instance2 instance3 instance4 | // | \ \ / / | // | | // | OlapScanNode | - // |(share scan node, and local shuffle data to other local instances to parallel compute this data)| + // |(share scan node, instance1 will scan all data and local shuffle to other local instances | + // | to parallel compute this data) | // +------------------------------------------------------------------------------------------------+ ScanSource shareScanSource = instanceToScanRanges.get(0); // one scan range generate multiple instances, // different instances reference the same scan source int shareScanId = shareScanIdGenerator.getAndIncrement(); + ScanSource emptyShareScanSource = shareScanSource.newEmpty(); for (int i = 0; i < instanceNum; i++) { LocalShuffleAssignedJob instance = new LocalShuffleAssignedJob( - instanceIndexInFragment++, shareScanId, context.nextInstanceId(), - this, worker, shareScanSource); + instanceIndexInFragment++, shareScanId, i > 0, + context.nextInstanceId(), this, worker, + i == 0 ? shareScanSource : emptyShareScanSource + ); instances.add(instance); } } else { @@ -145,42 +152,15 @@ protected List insideMachineParallelization( return instances; } - protected boolean useLocalShuffleToAddParallel( - Map workerToScanRanges) { - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isForceToLocalShuffle()) { - return true; - } - return parallelTooLittle(workerToScanRanges); - } - - protected boolean parallelTooLittle(Map workerToScanRanges) { - if (this instanceof UnassignedScanBucketOlapTableJob) { - return scanRangesToLittle(workerToScanRanges) && bucketsTooLittle(workerToScanRanges); - } else if (this instanceof UnassignedScanSingleOlapTableJob - || this instanceof UnassignedScanSingleRemoteTableJob) { - return scanRangesToLittle(workerToScanRanges); - } else { - return false; - } - } - - protected boolean scanRangesToLittle( - Map workerToScanRanges) { - ConnectContext context = ConnectContext.get(); - int backendNum = workerToScanRanges.size(); - for (ScanNode scanNode : scanNodes) { - if (!scanNode.ignoreStorageDataDistribution(context, backendNum)) { - return false; - } - } - return true; - } - protected int degreeOfParallelism(int maxParallel) { Preconditions.checkArgument(maxParallel > 0, "maxParallel must be positive"); if (!fragment.getDataPartition().isPartitioned()) { return 1; } + if (fragment.queryCacheParam != null) { + // backend need use one instance for one tablet to look up tablet query cache + return maxParallel; + } if (scanNodes.size() == 1 && scanNodes.get(0) instanceof OlapScanNode) { OlapScanNode olapScanNode = (OlapScanNode) scanNodes.get(0); // if the scan node have limit and no conjuncts, only need 1 instance to save cpu and mem resource, @@ -195,21 +175,6 @@ protected int degreeOfParallelism(int maxParallel) { return Math.min(maxParallel, Math.max(fragment.getParallelExecNum(), 1)); } - protected boolean bucketsTooLittle(Map workerToScanRanges) { - int parallelExecNum = fragment.getParallelExecNum(); - for (UninstancedScanSource uninstancedScanSource : workerToScanRanges.values()) { - ScanSource scanSource = uninstancedScanSource.scanSource; - if (scanSource instanceof BucketScanSource) { - BucketScanSource bucketScanSource = (BucketScanSource) scanSource; - int bucketNum = bucketScanSource.bucketIndexToScanNodeToTablets.size(); - if (bucketNum >= parallelExecNum) { - return false; - } - } - } - return true; - } - protected List fillUpSingleEmptyInstance(DistributedPlanWorkerManager workerManager) { return ImmutableList.of( assignWorkerAndDataSources(0, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java index f9f6b9dea1451b..d74626b4889468 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java @@ -36,4 +36,5 @@ public interface AssignedJob { ScanSource getScanSource(); String toString(boolean showUnassignedJob); + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java index 2a59f2ad8e8632..102a4c789fef42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; -import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.PlanFragmentId; import org.apache.doris.thrift.TExplainLevel; @@ -35,7 +35,7 @@ public class AssignedJobBuilder { /** buildJobs */ public static ListMultimap buildJobs( Map unassignedJobs) { - BackendDistributedPlanWorkerManager workerManager = new BackendDistributedPlanWorkerManager(); + DistributeContext distributeContext = new DistributeContext(); ListMultimap allAssignedJobs = ArrayListMultimap.create(); for (Entry kv : unassignedJobs.entrySet()) { PlanFragmentId fragmentId = kv.getKey(); @@ -43,7 +43,11 @@ public static ListMultimap buildJobs( ListMultimap inputAssignedJobs = getInputAssignedJobs(unassignedJob, allAssignedJobs); List fragmentAssignedJobs = - unassignedJob.computeAssignedJobs(workerManager, inputAssignedJobs); + unassignedJob.computeAssignedJobs(distributeContext, inputAssignedJobs); + for (AssignedJob assignedJob : fragmentAssignedJobs) { + distributeContext.selectedWorkers.onCreateAssignedJob(assignedJob); + } + if (fragmentAssignedJobs.isEmpty()) { throw new IllegalStateException("Fragment has no instance, unassignedJob: " + unassignedJob + ", fragment: " + unassignedJob.getFragment().getExplainString(TExplainLevel.VERBOSE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java index 33d066a02b9fcb..c9fb78db8255c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java @@ -36,7 +36,7 @@ public class BucketScanSource extends ScanSource { public final Map> bucketIndexToScanNodeToTablets; public BucketScanSource(Map> bucketIndexToScanNodeToTablets) { - this.bucketIndexToScanNodeToTablets = bucketIndexToScanNodeToTablets; + this.bucketIndexToScanNodeToTablets = Maps.newLinkedHashMap(bucketIndexToScanNodeToTablets); } @Override @@ -141,6 +141,11 @@ public void toString(StringBuilder str, String prefix) { str.append("\n").append(prefix).append("]"); } + @Override + public ScanSource newEmpty() { + return new BucketScanSource(Maps.newLinkedHashMap()); + } + @Override public boolean isEmpty() { return bucketIndexToScanNodeToTablets.isEmpty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java index 929f70e73a7e4d..1aa3c3361bee6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.List; import java.util.Map; @@ -35,7 +36,7 @@ public class DefaultScanSource extends ScanSource { public final Map scanNodeToScanRanges; public DefaultScanSource(Map scanNodeToScanRanges) { - this.scanNodeToScanRanges = scanNodeToScanRanges; + this.scanNodeToScanRanges = Maps.newLinkedHashMap(scanNodeToScanRanges); } public static DefaultScanSource empty() { @@ -82,11 +83,6 @@ public boolean isEmpty() { return scanNodeToScanRanges.isEmpty(); } - @Override - public void toString(StringBuilder str, String prefix) { - toString(scanNodeToScanRanges, str, prefix); - } - /** toString */ public static void toString(Map scanNodeToScanRanges, StringBuilder str, String prefix) { if (scanNodeToScanRanges.isEmpty()) { @@ -112,4 +108,14 @@ public static void toString(Map scanNodeToScanRanges, Stri } str.append("\n").append(prefix).append("]"); } + + @Override + public void toString(StringBuilder str, String prefix) { + toString(scanNodeToScanRanges, str, prefix); + } + + @Override + public ScanSource newEmpty() { + return empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java index 50e43fc0282755..2ba269a5a7b89f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java @@ -24,20 +24,35 @@ import java.util.Map; -/** LocalShuffleAssignedJob */ +/** + * LocalShuffleAssignedJob: + * this instance will use ignore_data_distribution function of local shuffle to add parallel + * after scan data + */ public class LocalShuffleAssignedJob extends StaticAssignedJob { public final int shareScanId; + public final boolean receiveDataFromLocal; public LocalShuffleAssignedJob( - int indexInUnassignedJob, int shareScanId, TUniqueId instanceId, + int indexInUnassignedJob, int shareScanId, boolean receiveDataFromLocal, TUniqueId instanceId, UnassignedJob unassignedJob, DistributedPlanWorker worker, ScanSource scanSource) { super(indexInUnassignedJob, instanceId, unassignedJob, worker, scanSource); this.shareScanId = shareScanId; + this.receiveDataFromLocal = receiveDataFromLocal; } @Override protected Map extraInfo() { return ImmutableMap.of("shareScanIndex", String.valueOf(shareScanId)); } + + @Override + protected String formatScanSourceString() { + if (receiveDataFromLocal) { + return "read data from first instance of " + getAssignedWorker(); + } else { + return super.formatScanSourceString(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java index b124e14bd73c77..5802f8ab7efd85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java @@ -30,6 +30,8 @@ public abstract class ScanSource { public abstract boolean isEmpty(); + public abstract ScanSource newEmpty(); + public abstract void toString(StringBuilder str, String prefix); @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java index 1a92cf71019e66..75849ad8146737 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java @@ -77,12 +77,6 @@ public String toString() { @Override public String toString(boolean showUnassignedJob) { - StringBuilder scanSourceString = new StringBuilder(); - if (!scanSource.isEmpty()) { - scanSource.toString(scanSourceString, " "); - } else { - scanSourceString = new StringBuilder("[]"); - } StringBuilder str = new StringBuilder(getClass().getSimpleName()).append("("); if (showUnassignedJob) { str.append("\n unassignedJob: ").append(unassignedJob).append(","); @@ -95,7 +89,7 @@ public String toString(boolean showUnassignedJob) { } return str - .append(",\n scanSource: " + scanSourceString) + .append(",\n scanSource: " + formatScanSourceString()) .append("\n)") .toString(); } @@ -103,4 +97,19 @@ public String toString(boolean showUnassignedJob) { protected Map extraInfo() { return ImmutableMap.of(); } + + protected String formatScanSourceString() { + StringBuilder scanSourceString = new StringBuilder(); + if (!scanSource.isEmpty()) { + scanSource.toString(scanSourceString, " "); + } else { + scanSourceString = new StringBuilder("[]"); + } + return scanSourceString.toString(); + } + + @Override + public int hashCode() { + return indexInUnassignedJob; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java new file mode 100644 index 00000000000000..830342514bd387 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.distribute.worker.job; + +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; + +import java.util.List; + +/** UnassignedGatherJob */ +public class UnassignedGatherJob extends AbstractUnassignedJob { + private boolean useLocalShuffleToAddParallel; + + public UnassignedGatherJob( + StatementContext statementContext, PlanFragment fragment, + ListMultimap exchangeToChildJob) { + super(statementContext, fragment, ImmutableList.of(), exchangeToChildJob); + } + + @Override + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { + ConnectContext connectContext = statementContext.getConnectContext(); + useLocalShuffleToAddParallel = fragment.useSerialSource(connectContext); + + int expectInstanceNum = degreeOfParallelism(); + + DistributedPlanWorker selectedWorker = distributeContext.selectedWorkers.tryToSelectRandomUsedWorker(); + if (useLocalShuffleToAddParallel) { + ImmutableList.Builder instances = ImmutableList.builder(); + + DefaultScanSource shareScan = new DefaultScanSource(ImmutableMap.of()); + LocalShuffleAssignedJob receiveDataFromRemote = new LocalShuffleAssignedJob( + 0, 0, false, + connectContext.nextInstanceId(), this, selectedWorker, shareScan); + + instances.add(receiveDataFromRemote); + for (int i = 1; i < expectInstanceNum; ++i) { + LocalShuffleAssignedJob receiveDataFromLocal = new LocalShuffleAssignedJob( + i, 0, true, + connectContext.nextInstanceId(), this, selectedWorker, shareScan); + instances.add(receiveDataFromLocal); + } + return instances.build(); + } else { + return ImmutableList.of( + assignWorkerAndDataSources( + 0, connectContext.nextInstanceId(), + selectedWorker, new DefaultScanSource(ImmutableMap.of()) + ) + ); + } + } + + protected int degreeOfParallelism() { + return useLocalShuffleToAddParallel ? fragment.getParallelExecNum() : 1; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java index 558af8844974fc..f3d260e289d792 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java @@ -17,8 +17,9 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; -import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; import org.apache.doris.planner.DataGenScanNode; import org.apache.doris.planner.ExchangeNode; @@ -38,9 +39,9 @@ /** UnassignedGatherScanMultiRemoteTablesJob */ public class UnassignedGatherScanMultiRemoteTablesJob extends AbstractUnassignedJob { - public UnassignedGatherScanMultiRemoteTablesJob(PlanFragment fragment, + public UnassignedGatherScanMultiRemoteTablesJob(StatementContext statementContext, PlanFragment fragment, List scanNodes, ListMultimap exchangeToChildJob) { - super(fragment, scanNodes, exchangeToChildJob); + super(statementContext, fragment, scanNodes, exchangeToChildJob); } /** canApply */ @@ -61,9 +62,9 @@ public static boolean canApply(List scanNodes) { } @Override - public List computeAssignedJobs(DistributedPlanWorkerManager workerManager, - ListMultimap inputJobs) { - ConnectContext context = ConnectContext.get(); + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { + ConnectContext connectContext = statementContext.getConnectContext(); Map scanNodeToScanRanges = Maps.newLinkedHashMap(); for (ScanNode scanNode : scanNodes) { List scanRangeLocations = scanNode.getScanRangeLocations(0); @@ -77,9 +78,9 @@ public List computeAssignedJobs(DistributedPlanWorkerManager worker scanNodeToScanRanges.put(scanNode, scanRanges); } - DistributedPlanWorker randomWorker = workerManager.randomAvailableWorker(); + DistributedPlanWorker randomWorker = distributeContext.selectedWorkers.tryToSelectRandomUsedWorker(); return ImmutableList.of( - assignWorkerAndDataSources(0, context.nextInstanceId(), + assignWorkerAndDataSources(0, connectContext.nextInstanceId(), randomWorker, new DefaultScanSource(scanNodeToScanRanges) ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGroupCommitJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGroupCommitJob.java new file mode 100644 index 00000000000000..bcd7218f9f4355 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGroupCommitJob.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.distribute.worker.job; + +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; + +import java.util.List; + +/** UnassignedGroupCommitJob */ +public class UnassignedGroupCommitJob extends AbstractUnassignedJob { + public UnassignedGroupCommitJob(StatementContext statementContext, + PlanFragment fragment, List scanNodes, + ListMultimap exchangeToChildJob) { + super(statementContext, fragment, scanNodes, exchangeToChildJob); + } + + @Override + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { + TUniqueId instanceId = statementContext.getConnectContext().nextInstanceId(); + BackendWorker selectBackend = new BackendWorker(statementContext.getGroupCommitMergeBackend()); + return ImmutableList.of( + new StaticAssignedJob( + 0, instanceId, this, selectBackend, DefaultScanSource.empty() + ) + ); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java index d142460ea2db02..a5d6331440acab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java @@ -17,9 +17,10 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.TreeNode; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; -import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.ScanNode; @@ -34,6 +35,8 @@ * for example: a fragment job, which doesn't parallelization to some instance jobs and also no worker to invoke it */ public interface UnassignedJob extends TreeNode { + StatementContext getStatementContext(); + PlanFragment getFragment(); List getScanNodes(); @@ -41,7 +44,7 @@ public interface UnassignedJob extends TreeNode { ListMultimap getExchangeToChildJob(); List computeAssignedJobs( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs); + DistributeContext distributeContext, ListMultimap inputJobs); // generate an instance job // e.g. build an instance job by a backends and the replica ids it contains diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java index 396ba51e01b4ee..64ff43da8c837a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping; import org.apache.doris.nereids.trees.plans.distribute.worker.LoadBalanceScanWorkerSelector; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; @@ -36,6 +37,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -50,39 +52,49 @@ public class UnassignedJobBuilder { /** * build job from fragment. */ - public static FragmentIdMapping buildJobs(FragmentIdMapping fragments) { + public static FragmentIdMapping buildJobs( + StatementContext statementContext, FragmentIdMapping fragments) { UnassignedJobBuilder builder = new UnassignedJobBuilder(); FragmentLineage fragmentLineage = buildFragmentLineage(fragments); FragmentIdMapping unassignedJobs = new FragmentIdMapping<>(); // build from leaf to parent - for (Entry kv : fragments.entrySet()) { + Iterator> iterator = fragments.entrySet().iterator(); + while (iterator.hasNext()) { + Entry kv = iterator.next(); + boolean isTopFragment = !iterator.hasNext(); + PlanFragmentId fragmentId = kv.getKey(); PlanFragment fragment = kv.getValue(); ListMultimap inputJobs = findInputJobs( fragmentLineage, fragmentId, unassignedJobs); - UnassignedJob unassignedJob = builder.buildJob(fragment, inputJobs); + UnassignedJob unassignedJob = builder.buildJob(statementContext, fragment, inputJobs, isTopFragment); unassignedJobs.put(fragmentId, unassignedJob); } + return unassignedJobs; } private UnassignedJob buildJob( - PlanFragment planFragment, ListMultimap inputJobs) { + StatementContext statementContext, PlanFragment planFragment, + ListMultimap inputJobs, boolean isTopFragment) { List scanNodes = collectScanNodesInThisFragment(planFragment); if (planFragment.specifyInstances.isPresent()) { - return buildSpecifyInstancesJob(planFragment, scanNodes, inputJobs); + return buildSpecifyInstancesJob(statementContext, planFragment, scanNodes, inputJobs); + } else if (scanNodes.isEmpty() && isTopFragment + && statementContext.getGroupCommitMergeBackend() != null) { + return new UnassignedGroupCommitJob(statementContext, planFragment, scanNodes, inputJobs); } else if (!scanNodes.isEmpty() || isLeafFragment(planFragment)) { - return buildLeafOrScanJob(planFragment, scanNodes, inputJobs); + return buildLeafOrScanJob(statementContext, planFragment, scanNodes, inputJobs); } else { - return buildShuffleJob(planFragment, inputJobs); + return buildShuffleJob(statementContext, planFragment, inputJobs); } } private UnassignedJob buildLeafOrScanJob( - PlanFragment planFragment, List scanNodes, + StatementContext statementContext, PlanFragment planFragment, List scanNodes, ListMultimap inputJobs) { int olapScanNodeNum = olapScanNodeNum(scanNodes); @@ -91,20 +103,26 @@ private UnassignedJob buildLeafOrScanJob( // we need assign a backend which contains the data, // so that the OlapScanNode can find the data in the backend // e.g. select * from olap_table - unassignedJob = buildScanOlapTableJob(planFragment, (List) scanNodes, inputJobs, scanWorkerSelector); + unassignedJob = buildScanOlapTableJob( + statementContext, planFragment, (List) scanNodes, inputJobs, scanWorkerSelector + ); } else if (scanNodes.isEmpty()) { // select constant without table, // e.g. select 100 union select 200 - unassignedJob = buildQueryConstantJob(planFragment); + unassignedJob = buildQueryConstantJob(statementContext, planFragment); } else if (olapScanNodeNum == 0) { ScanNode scanNode = scanNodes.get(0); if (scanNode instanceof SchemaScanNode) { // select * from information_schema.tables - unassignedJob = buildScanMetadataJob(planFragment, (SchemaScanNode) scanNode, scanWorkerSelector); + unassignedJob = buildScanMetadataJob( + statementContext, planFragment, (SchemaScanNode) scanNode, scanWorkerSelector + ); } else { // only scan external tables or cloud tables or table valued functions // e,g. select * from numbers('number'='100') - unassignedJob = buildScanRemoteTableJob(planFragment, scanNodes, inputJobs, scanWorkerSelector); + unassignedJob = buildScanRemoteTableJob( + statementContext, planFragment, scanNodes, inputJobs, scanWorkerSelector + ); } } @@ -117,20 +135,21 @@ private UnassignedJob buildLeafOrScanJob( } private UnassignedJob buildSpecifyInstancesJob( - PlanFragment planFragment, List scanNodes, ListMultimap inputJobs) { - return new UnassignedSpecifyInstancesJob(planFragment, scanNodes, inputJobs); + StatementContext statementContext, PlanFragment planFragment, + List scanNodes, ListMultimap inputJobs) { + return new UnassignedSpecifyInstancesJob(statementContext, planFragment, scanNodes, inputJobs); } private UnassignedJob buildScanOlapTableJob( - PlanFragment planFragment, List olapScanNodes, + StatementContext statementContext, PlanFragment planFragment, List olapScanNodes, ListMultimap inputJobs, ScanWorkerSelector scanWorkerSelector) { if (shouldAssignByBucket(planFragment)) { return new UnassignedScanBucketOlapTableJob( - planFragment, olapScanNodes, inputJobs, scanWorkerSelector); + statementContext, planFragment, olapScanNodes, inputJobs, scanWorkerSelector); } else if (olapScanNodes.size() == 1) { return new UnassignedScanSingleOlapTableJob( - planFragment, olapScanNodes.get(0), inputJobs, scanWorkerSelector); + statementContext, planFragment, olapScanNodes.get(0), inputJobs, scanWorkerSelector); } else { throw new IllegalStateException("Not supported multiple scan multiple " + "OlapTable but not contains colocate join or bucket shuffle join: " @@ -156,34 +175,41 @@ private boolean isLeafFragment(PlanFragment planFragment) { return planFragment.getChildren().isEmpty(); } - private UnassignedQueryConstantJob buildQueryConstantJob(PlanFragment planFragment) { - return new UnassignedQueryConstantJob(planFragment); + private UnassignedQueryConstantJob buildQueryConstantJob( + StatementContext statementContext, PlanFragment planFragment) { + return new UnassignedQueryConstantJob(statementContext, planFragment); } private UnassignedJob buildScanMetadataJob( - PlanFragment fragment, SchemaScanNode schemaScanNode, ScanWorkerSelector scanWorkerSelector) { - return new UnassignedScanMetadataJob(fragment, schemaScanNode, scanWorkerSelector); + StatementContext statementContext, PlanFragment fragment, + SchemaScanNode schemaScanNode, ScanWorkerSelector scanWorkerSelector) { + return new UnassignedScanMetadataJob(statementContext, fragment, schemaScanNode, scanWorkerSelector); } private UnassignedJob buildScanRemoteTableJob( - PlanFragment planFragment, List scanNodes, + StatementContext statementContext, PlanFragment planFragment, List scanNodes, ListMultimap inputJobs, ScanWorkerSelector scanWorkerSelector) { if (scanNodes.size() == 1) { return new UnassignedScanSingleRemoteTableJob( - planFragment, scanNodes.get(0), inputJobs, scanWorkerSelector); + statementContext, planFragment, scanNodes.get(0), inputJobs, scanWorkerSelector); } else if (UnassignedGatherScanMultiRemoteTablesJob.canApply(scanNodes)) { // select * from numbers("number" = "10") a union all select * from numbers("number" = "20") b; // use an instance to scan table a and table b - return new UnassignedGatherScanMultiRemoteTablesJob(planFragment, scanNodes, inputJobs); + return new UnassignedGatherScanMultiRemoteTablesJob(statementContext, planFragment, scanNodes, inputJobs); } else { return null; } } - private UnassignedShuffleJob buildShuffleJob( - PlanFragment planFragment, ListMultimap inputJobs) { - return new UnassignedShuffleJob(planFragment, inputJobs); + private UnassignedJob buildShuffleJob( + StatementContext statementContext, PlanFragment planFragment, + ListMultimap inputJobs) { + if (planFragment.isPartitioned()) { + return new UnassignedShuffleJob(statementContext, planFragment, inputJobs); + } else { + return new UnassignedGatherJob(statementContext, planFragment, inputJobs); + } } private static ListMultimap findInputJobs( @@ -246,16 +272,12 @@ private static boolean shouldAssignByBucket(PlanFragment fragment) { if (fragment.hasColocatePlanNode()) { return true; } - if (enableBucketShuffleJoin() && fragment.hasBucketShuffleJoin()) { + if (fragment.hasBucketShuffleJoin()) { return true; } return false; } - private static boolean enableBucketShuffleJoin() { - return true; - } - // the class support find exchange nodes in the fragment, and find child fragment by exchange node id private static class FragmentLineage { private final FragmentIdMapping> parentFragmentToExchangeNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java index bfbafd0739065c..4c9fb15a2b7cef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java @@ -17,8 +17,9 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; -import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.qe.ConnectContext; @@ -32,17 +33,17 @@ /** UnassignedQueryConstantJob */ public class UnassignedQueryConstantJob extends AbstractUnassignedJob { - public UnassignedQueryConstantJob(PlanFragment fragment) { - super(fragment, ImmutableList.of(), ArrayListMultimap.create()); + public UnassignedQueryConstantJob(StatementContext statementContext, PlanFragment fragment) { + super(statementContext, fragment, ImmutableList.of(), ArrayListMultimap.create()); } @Override - public List computeAssignedJobs(DistributedPlanWorkerManager workerManager, - ListMultimap inputJobs) { - DistributedPlanWorker randomWorker = workerManager.randomAvailableWorker(); - ConnectContext context = ConnectContext.get(); + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { + DistributedPlanWorker randomWorker = distributeContext.selectedWorkers.tryToSelectRandomUsedWorker(); + ConnectContext connectContext = statementContext.getConnectContext(); return ImmutableList.of( - new StaticAssignedJob(0, context.nextInstanceId(), this, + new StaticAssignedJob(0, connectContext.nextInstanceId(), this, randomWorker, new DefaultScanSource(ImmutableMap.of()) ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java index 2003815e8d7c14..70a91ca2b30f32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java @@ -22,6 +22,8 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Tablet; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; @@ -58,10 +60,10 @@ public class UnassignedScanBucketOlapTableJob extends AbstractUnassignedScanJob /** UnassignedScanNativeTableJob */ public UnassignedScanBucketOlapTableJob( - PlanFragment fragment, List olapScanNodes, + StatementContext statementContext, PlanFragment fragment, List olapScanNodes, ListMultimap exchangeToChildJob, ScanWorkerSelector scanWorkerSelector) { - super(fragment, (List) olapScanNodes, exchangeToChildJob); + super(statementContext, fragment, (List) olapScanNodes, exchangeToChildJob); this.scanWorkerSelector = Objects.requireNonNull( scanWorkerSelector, "scanWorkerSelector cat not be null"); @@ -75,7 +77,7 @@ public List getOlapScanNodes() { @Override protected Map multipleMachinesParallelization( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs) { + DistributeContext distributeContext, ListMultimap inputJobs) { // for every bucket tablet, select its replica and worker. // for example, colocate join: // { @@ -94,13 +96,15 @@ protected Map multipleMachinesPara // ... // } // } - return scanWorkerSelector.selectReplicaAndWorkerWithBucket(this); + return scanWorkerSelector.selectReplicaAndWorkerWithBucket( + this, statementContext.getConnectContext() + ); } @Override protected List insideMachineParallelization( Map workerToScanRanges, - ListMultimap inputJobs, DistributedPlanWorkerManager workerManager) { + ListMultimap inputJobs, DistributeContext distributeContext) { // separate buckets to instanceNum groups, let one instance process some buckets. // for example, colocate join: // { @@ -129,7 +133,7 @@ protected List insideMachineParallelization( // ... // } List assignedJobs = super.insideMachineParallelization( - workerToScanRanges, inputJobs, workerManager); + workerToScanRanges, inputJobs, distributeContext); // the case: // ```sql @@ -219,7 +223,7 @@ private List fillUpInstances(List instances) { if (!mergedBucketsInSameWorkerInstance) { fillUpInstance = new LocalShuffleAssignedJob( newInstances.size(), shareScanIdGenerator.getAndIncrement(), - context.nextInstanceId(), this, worker, scanSource + false, context.nextInstanceId(), this, worker, scanSource ); } } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanMetadataJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanMetadataJob.java index d15bed010619b1..aab0f20895d049 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanMetadataJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanMetadataJob.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; @@ -37,9 +39,10 @@ public class UnassignedScanMetadataJob extends AbstractUnassignedScanJob { private final SchemaScanNode schemaScanNode; private final ScanWorkerSelector scanWorkerSelector; - public UnassignedScanMetadataJob(PlanFragment fragment, SchemaScanNode schemaScanNode, - ScanWorkerSelector scanWorkerSelector) { - super(fragment, ImmutableList.of(schemaScanNode), ArrayListMultimap.create()); + public UnassignedScanMetadataJob( + StatementContext statementContext, PlanFragment fragment, + SchemaScanNode schemaScanNode, ScanWorkerSelector scanWorkerSelector) { + super(statementContext, fragment, ImmutableList.of(schemaScanNode), ArrayListMultimap.create()); this.scanWorkerSelector = Objects.requireNonNull( scanWorkerSelector, "scanWorkerSelector cat not be null"); this.schemaScanNode = schemaScanNode; @@ -47,8 +50,10 @@ public UnassignedScanMetadataJob(PlanFragment fragment, SchemaScanNode schemaSca @Override protected Map multipleMachinesParallelization( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs) { - return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(schemaScanNode); + DistributeContext distributeContext, ListMultimap inputJobs) { + return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket( + schemaScanNode, statementContext.getConnectContext() + ); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java index ecbadab8af3a93..649e2fa9bb28ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; @@ -37,10 +39,10 @@ public class UnassignedScanSingleOlapTableJob extends AbstractUnassignedScanJob private final ScanWorkerSelector scanWorkerSelector; public UnassignedScanSingleOlapTableJob( - PlanFragment fragment, OlapScanNode olapScanNode, + StatementContext statementContext, PlanFragment fragment, OlapScanNode olapScanNode, ListMultimap exchangeToChildJob, ScanWorkerSelector scanWorkerSelector) { - super(fragment, ImmutableList.of(olapScanNode), exchangeToChildJob); + super(statementContext, fragment, ImmutableList.of(olapScanNode), exchangeToChildJob); this.scanWorkerSelector = Objects.requireNonNull( scanWorkerSelector, "scanWorkerSelector cat not be null"); this.olapScanNode = olapScanNode; @@ -48,7 +50,7 @@ public UnassignedScanSingleOlapTableJob( @Override protected Map multipleMachinesParallelization( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs) { + DistributeContext distributeContext, ListMultimap inputJobs) { // for every tablet, select its replica and worker. // for example: // { @@ -57,13 +59,15 @@ protected Map multipleMachinesPara // BackendWorker("172.0.0.2"): // olapScanNode1: ScanRanges([tablet_10005, tablet_10006, tablet_10007, tablet_10008, tablet_10009]) // } - return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(olapScanNode); + return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket( + olapScanNode, statementContext.getConnectContext() + ); } @Override protected List insideMachineParallelization( Map workerToScanRanges, - ListMultimap inputJobs, DistributedPlanWorkerManager workerManager) { + ListMultimap inputJobs, DistributeContext distributeContext) { // for each worker, compute how many instances should be generated, and which data should be scanned. // for example: // { @@ -77,7 +81,7 @@ protected List insideMachineParallelization( // instance 5: olapScanNode1: ScanRanges([tablet_10007]) // ], // } - return super.insideMachineParallelization(workerToScanRanges, inputJobs, workerManager); + return super.insideMachineParallelization(workerToScanRanges, inputJobs, distributeContext); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java index 08e48202fcf90f..bc98119d939aaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; @@ -40,16 +42,18 @@ public class UnassignedScanSingleRemoteTableJob extends AbstractUnassignedScanJo private final ScanWorkerSelector scanWorkerSelector; public UnassignedScanSingleRemoteTableJob( - PlanFragment fragment, ScanNode scanNode, ListMultimap exchangeToChildJob, - ScanWorkerSelector scanWorkerSelector) { - super(fragment, ImmutableList.of(scanNode), exchangeToChildJob); + StatementContext statementContext, PlanFragment fragment, ScanNode scanNode, + ListMultimap exchangeToChildJob, ScanWorkerSelector scanWorkerSelector) { + super(statementContext, fragment, ImmutableList.of(scanNode), exchangeToChildJob); this.scanWorkerSelector = Objects.requireNonNull(scanWorkerSelector, "scanWorkerSelector is not null"); } @Override protected Map multipleMachinesParallelization( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs) { - return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(scanNodes.get(0)); + DistributeContext distributeContext, ListMultimap inputJobs) { + return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket( + scanNodes.get(0), statementContext.getConnectContext() + ); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java index 5b1cb1f9878fb1..3d937bfb35dbf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java @@ -17,16 +17,21 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; -import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.PlanNode; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.Collection; @@ -38,13 +43,19 @@ /** UnassignedShuffleJob */ public class UnassignedShuffleJob extends AbstractUnassignedJob { - public UnassignedShuffleJob(PlanFragment fragment, ListMultimap exchangeToChildJob) { - super(fragment, ImmutableList.of(), exchangeToChildJob); + private boolean useLocalShuffleToAddParallel; + + public UnassignedShuffleJob( + StatementContext statementContext, PlanFragment fragment, + ListMultimap exchangeToChildJob) { + super(statementContext, fragment, ImmutableList.of(), exchangeToChildJob); } @Override public List computeAssignedJobs( - DistributedPlanWorkerManager workerManager, ListMultimap inputJobs) { + DistributeContext distributeContext, ListMultimap inputJobs) { + useLocalShuffleToAddParallel = fragment.useSerialSource(statementContext.getConnectContext()); + int expectInstanceNum = degreeOfParallelism(); List biggestParallelChildFragment = getInstancesOfBiggestParallelChildFragment(inputJobs); @@ -70,10 +81,6 @@ public List computeAssignedJobs( } protected int degreeOfParallelism() { - if (!fragment.getDataPartition().isPartitioned()) { - return 1; - } - // TODO: check we use nested loop join do right outer / semi / anti join, // we should add an exchange node with gather distribute under the nested loop join @@ -81,6 +88,13 @@ protected int degreeOfParallelism() { if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) { expectInstanceNum = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel(); } + + // TODO: check nested loop join do right outer / semi / anti join + PlanNode leftMostNode = findLeftmostNode(fragment.getPlanRoot()).second; + // when we use nested loop join do right outer / semi / anti join, the instance must be 1. + if (leftMostNode.getNumInstances() == 1) { + expectInstanceNum = 1; + } return expectInstanceNum; } @@ -99,19 +113,60 @@ private List getInstancesOfBiggestParallelChildFragment( return biggestParallelChildFragment; } - private List buildInstances(int instanceNum, Function workerSelector) { + private List buildInstances( + int instanceNum, Function workerSelector) { + ConnectContext connectContext = statementContext.getConnectContext(); + if (useLocalShuffleToAddParallel) { + return buildInstancesWithLocalShuffle(instanceNum, workerSelector, connectContext); + } else { + return buildInstancesWithoutLocalShuffle(instanceNum, workerSelector, connectContext); + } + } + + private List buildInstancesWithoutLocalShuffle( + int instanceNum, Function workerSelector, ConnectContext connectContext) { ImmutableList.Builder instances = ImmutableList.builderWithExpectedSize(instanceNum); - ConnectContext context = ConnectContext.get(); for (int i = 0; i < instanceNum; i++) { DistributedPlanWorker selectedWorker = workerSelector.apply(i); AssignedJob assignedJob = assignWorkerAndDataSources( - i, context.nextInstanceId(), selectedWorker, new DefaultScanSource(ImmutableMap.of()) + i, connectContext.nextInstanceId(), + selectedWorker, new DefaultScanSource(ImmutableMap.of()) ); instances.add(assignedJob); } return instances.build(); } + private List buildInstancesWithLocalShuffle( + int instanceNum, Function workerSelector, ConnectContext connectContext) { + ImmutableList.Builder instances = ImmutableList.builderWithExpectedSize(instanceNum); + Multimap workerToInstanceIds = ArrayListMultimap.create(); + for (int i = 0; i < instanceNum; i++) { + DistributedPlanWorker selectedWorker = workerSelector.apply(i); + workerToInstanceIds.put(selectedWorker, i); + } + + int shareScanId = 0; + for (Entry> kv : workerToInstanceIds.asMap().entrySet()) { + DistributedPlanWorker worker = kv.getKey(); + Collection indexesInFragment = kv.getValue(); + + DefaultScanSource shareScanSource = new DefaultScanSource(ImmutableMap.of()); + + boolean receiveDataFromLocal = false; + for (Integer indexInFragment : indexesInFragment) { + LocalShuffleAssignedJob instance = new LocalShuffleAssignedJob( + indexInFragment, shareScanId, receiveDataFromLocal, connectContext.nextInstanceId(), + this, worker, shareScanSource + ); + instances.add(instance); + receiveDataFromLocal = true; + } + shareScanId++; + } + return instances.build(); + } + private List distinctShuffleWorkers(List instances) { Set candidateWorkerSet = Sets.newLinkedHashSet(); for (AssignedJob instance : instances) { @@ -121,4 +176,16 @@ private List distinctShuffleWorkers(List ins Collections.shuffle(candidateWorkers); return candidateWorkers; } + + // Returns the id of the leftmost node of any of the gives types in 'plan_root', + // or INVALID_PLAN_NODE_ID if no such node present. + private Pair findLeftmostNode(PlanNode plan) { + PlanNode childPlan = plan; + PlanNode fatherPlan = null; + while (childPlan.getChildren().size() != 0 && !(childPlan instanceof ExchangeNode)) { + fatherPlan = childPlan; + childPlan = childPlan.getChild(0); + } + return Pair.of(fatherPlan, childPlan); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java index c450161ae41905..6ded32e0cd98c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java @@ -17,8 +17,9 @@ package org.apache.doris.nereids.trees.plans.distribute.worker.job; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.NereidsSpecifyInstances; -import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.ScanNode; @@ -33,17 +34,17 @@ public class UnassignedSpecifyInstancesJob extends AbstractUnassignedJob { private final NereidsSpecifyInstances specifyInstances; public UnassignedSpecifyInstancesJob( - PlanFragment fragment, List scanNodes, + StatementContext statementContext, PlanFragment fragment, List scanNodes, ListMultimap exchangeToChildJob) { - super(fragment, scanNodes, exchangeToChildJob); + super(statementContext, fragment, scanNodes, exchangeToChildJob); Preconditions.checkArgument(fragment.specifyInstances.isPresent(), "Missing fragment specifyInstances"); this.specifyInstances = fragment.specifyInstances.get(); } @Override - public List computeAssignedJobs(DistributedPlanWorkerManager workerManager, - ListMultimap inputJobs) { + public List computeAssignedJobs( + DistributeContext distributeContext, ListMultimap inputJobs) { return specifyInstances.buildAssignedJobs(this); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 7d5c579068aa4d..67d4defe4b03fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -52,6 +52,8 @@ import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowVariablesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowViewCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetVariableCommand; import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand; @@ -234,7 +236,16 @@ default R visitSetDefaultStorageVault(SetDefaultStorageVaultCommand setDefaultSt return visitCommand(setDefaultStorageVaultCommand, context); } + default R visitRefreshCatalogCommand(RefreshCatalogCommand refreshCatalogCommand, C context) { return visitCommand(refreshCatalogCommand, context); + + default R visitShowVariablesCommand(ShowVariablesCommand showVariablesCommand, C context) { + return visitCommand(showVariablesCommand, context); + } + + default R visitShowViewCommand(ShowViewCommand showViewCommand, C context) { + return visitCommand(showViewCommand, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index 97d46b109b700f..cb6628b01c556b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -25,6 +25,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.UserException; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsRecursiveDerive; import org.apache.doris.thrift.TExchangeNode; @@ -79,6 +80,10 @@ public ExchangeNode(PlanNodeId id, PlanNode inputNode) { computeTupleIds(); } + public TPartitionType getPartitionType() { + return partitionType; + } + public void setPartitionType(TPartitionType partitionType) { this.partitionType = partitionType; } @@ -165,6 +170,10 @@ public void setMergeInfo(SortInfo info) { @Override protected void toThrift(TPlanNode msg) { + // If this fragment has another scan node, this exchange node is serial or not should be decided by the scan + // node. + msg.setIsSerialOperator((isSerialOperator() || fragment.hasSerialScanNode()) + && fragment.useSerialSource(ConnectContext.get())); msg.node_type = TPlanNodeType.EXCHANGE_NODE; msg.exchange_node = new TExchangeNode(); for (TupleId tid : tupleIds) { @@ -224,11 +233,17 @@ public void setRightChildOfBroadcastHashJoin(boolean value) { */ @Override public boolean isSerialOperator() { - return partitionType == TPartitionType.UNPARTITIONED && mergeInfo != null; + return (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isUseSerialExchange() + || partitionType == TPartitionType.UNPARTITIONED) && mergeInfo != null; } @Override public boolean hasSerialChildren() { return isSerialOperator(); } + + @Override + public boolean hasSerialScanChildren() { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java index 9c54bdb406cb5e..02633dc519ca78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java @@ -40,6 +40,9 @@ public void addToDest(ExchangeNode exchangeNode) { destNodeList.add(exchangeNode); } + public List getDestNodeList() { + return destNodeList; + } public List getDestFragmentList() { return destNodeList.stream().map(PlanNode::getFragment).collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 0ebd023ed411ee..ab5307c07e904a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -397,6 +397,10 @@ public PlanFragmentId getId() { return fragmentId; } + public ExchangeNode getDestNode() { + return destNode; + } + public PlanFragment getDestFragment() { if (destNode == null) { return null; @@ -511,15 +515,13 @@ public boolean useSerialSource(ConnectContext context) { && !hasNullAwareLeftAntiJoin() // If planRoot is not a serial operator and has serial children, we can use serial source and improve // parallelism of non-serial operators. - && sink instanceof DataStreamSink && !planRoot.isSerialOperator() - && planRoot.hasSerialChildren(); - } - - public int getNumBackends() { - return numBackends; + // For bucket shuffle / colocate join fragment, always use serial source if the bucket scan nodes are + // serial. + && (hasSerialScanNode() || (sink instanceof DataStreamSink && !planRoot.isSerialOperator() + && planRoot.hasSerialChildren())); } - public void setNumBackends(int numBackends) { - this.numBackends = numBackends; + public boolean hasSerialScanNode() { + return planRoot.hasSerialScanChildren(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 14bd34e93e1f43..73768435154b76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -1388,4 +1388,11 @@ public boolean hasSerialChildren() { } return children.stream().allMatch(PlanNode::hasSerialChildren); } + + public boolean hasSerialScanChildren() { + if (children.isEmpty()) { + return false; + } + return children.stream().anyMatch(PlanNode::hasSerialScanChildren); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index b392075be9b938..85c8de68b8c1a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -855,4 +855,9 @@ public boolean isSerialOperator() { < ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() * numScanBackends() || (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isForceToLocalShuffle()); } + + @Override + public boolean hasSerialScanChildren() { + return isSerialOperator(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index e1a95531989258..3bb1f0018fbdd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -101,7 +101,7 @@ private void resetAnalyzer() { analyzer = new Analyzer(Env.getCurrentEnv(), null); // TODO(cmy): currently we do not support UDF in stream load command. // Because there is no way to check the privilege of accessing UDF.. - analyzer.setUDFAllowed(false); + analyzer.setUDFAllowed(Config.enable_udf_in_load); descTable = analyzer.getDescTbl(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java index 8a4ca5db749156..20c05d982f806e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -60,6 +60,8 @@ public enum EventType { public String ctl = ""; @AuditField(value = "Db") public String db = ""; + @AuditField(value = "CommandType") + public String commandType = ""; @AuditField(value = "State") public String state = ""; @AuditField(value = "ErrorCode") @@ -270,6 +272,11 @@ public AuditEventBuilder setScanBytesFromRemoteStorage(long scanBytesFromRemoteS return this; } + public AuditEventBuilder setCommandType(String commandType) { + auditEvent.commandType = commandType; + return this; + } + public AuditEvent build() { return this.auditEvent; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 5d24452e092d0f..12958d53390c89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -32,6 +32,7 @@ import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.metric.MetricRepo; +import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -211,7 +212,8 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme .setQueryId(ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId())) .setCloudCluster(Strings.isNullOrEmpty(cluster) ? "UNKNOWN" : cluster) .setWorkloadGroup(ctx.getWorkloadGroupName()) - .setFuzzyVariables(!printFuzzyVariables ? "" : ctx.getSessionVariable().printFuzzyVariables()); + .setFuzzyVariables(!printFuzzyVariables ? "" : ctx.getSessionVariable().printFuzzyVariables()) + .setCommandType(ctx.getCommand().toString()); if (ctx.getState().isQuery()) { if (!ctx.getSessionVariable().internalSession) { @@ -292,6 +294,9 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme } } } + if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE && ctx.getState().getErrorCode() == null) { + auditEventBuilder.setState(String.valueOf(MysqlStateType.OK)); + } Env.getCurrentEnv().getWorkloadRuntimeStatusMgr().submitFinishQueryToAudit(auditEventBuilder.build()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 294e9e3405690c..b2124c7b00cabb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -171,20 +171,20 @@ public class Coordinator implements CoordInterface { private static final Logger LOG = LogManager.getLogger(Coordinator.class); - private static final String localIP = FrontendOptions.getLocalHostAddress(); + public static final String localIP = FrontendOptions.getLocalHostAddress(); // Random is used to shuffle instances of partitioned private static final Random instanceRandom = new SecureRandom(); - private static ExecutorService backendRpcCallbackExecutor = ThreadPoolManager.newDaemonProfileThreadPool(32, 100, - "backend-rpc-callback", true); + public static ExecutorService backendRpcCallbackExecutor = ThreadPoolManager.newDaemonProfileThreadPool( + 32, 100, "backend-rpc-callback", true); // Overall status of the entire query; set to the first reported fragment error // status or to CANCELLED, if Cancel() is called. - Status queryStatus = new Status(); + private Status queryStatus = new Status(); // save of related backends of this query - Map addressToBackendID = Maps.newHashMap(); + private Map addressToBackendID = Maps.newHashMap(); protected ImmutableMap idToBackend = ImmutableMap.of(); @@ -193,7 +193,7 @@ public class Coordinator implements CoordInterface { private FragmentIdMapping distributedPlans; // scan node id -> TFileScanRangeParams - protected Map fileScanRangeParamsMap = Maps.newHashMap(); + private Map fileScanRangeParamsMap = Maps.newHashMap(); // Why do we use query global? // When `NOW()` function is in sql, we need only one now(), @@ -217,7 +217,7 @@ public class Coordinator implements CoordInterface { private boolean returnedAllResults = false; // populated in computeFragmentExecParams() - protected final Map fragmentExecParamsMap = Maps.newHashMap(); + private final Map fragmentExecParamsMap = Maps.newHashMap(); private final List fragments; @@ -226,7 +226,7 @@ public class Coordinator implements CoordInterface { private final Map, PipelineExecContext> pipelineExecContexts = new HashMap<>(); private final List needCheckPipelineExecContexts = Lists.newArrayList(); private List receivers = Lists.newArrayList(); - protected final List scanNodes; + private final List scanNodes; private int scanRangeNum = 0; // number of instances of this query, equals to // number of backends executing plan fragments on behalf of this query; @@ -266,16 +266,16 @@ public class Coordinator implements CoordInterface { private Backend groupCommitBackend; // Runtime filter merge instance address and ID - public TNetworkAddress runtimeFilterMergeAddr; - public TUniqueId runtimeFilterMergeInstanceId; + private TNetworkAddress runtimeFilterMergeAddr; + private TUniqueId runtimeFilterMergeInstanceId; // Runtime filter ID to the target instance address of the fragment, // that is expected to use this runtime filter, the instance address is not repeated - public Map> ridToTargetParam = Maps.newHashMap(); + private Map> ridToTargetParam = Maps.newHashMap(); // The runtime filter that expects the instance to be used - public List assignedRuntimeFilters = new ArrayList<>(); - public List topnFilters = new ArrayList<>(); + private List assignedRuntimeFilters = new ArrayList<>(); + private List topnFilters = new ArrayList<>(); // Runtime filter ID to the builder instance number - public Map ridToBuilderNum = Maps.newHashMap(); + private Map ridToBuilderNum = Maps.newHashMap(); private ConnectContext context; private StatsErrorEstimator statsErrorEstimator; @@ -303,7 +303,7 @@ public long getNumReceivedRows() { return numReceivedRows; } - public List gettWorkloadGroups() { + public List getTWorkloadGroups() { return tWorkloadGroups; } @@ -703,7 +703,7 @@ public void close() { } } - private void execInternal() throws Exception { + protected void execInternal() throws Exception { if (LOG.isDebugEnabled() && !scanNodes.isEmpty()) { LOG.debug("debug: in Coordinator::exec. query id: {}, planNode: {}", DebugUtil.printId(queryId), scanNodes.get(0).treeToThrift()); @@ -775,8 +775,7 @@ private void execInternal() throws Exception { deltaUrls = Lists.newArrayList(); loadCounters = Maps.newHashMap(); List relatedBackendIds = Lists.newArrayList(addressToBackendID.values()); - Env.getCurrentEnv().getLoadManager().initJobProgress(jobId, queryId, instanceIds, - relatedBackendIds); + Env.getCurrentEnv().getLoadManager().initJobProgress(jobId, queryId, instanceIds, relatedBackendIds); Env.getCurrentEnv().getProgressManager().addTotalScanNums(String.valueOf(jobId), scanRangeNum); LOG.info("dispatch load job: {} to {}", DebugUtil.printId(queryId), addressToBackendID.keySet()); } @@ -785,7 +784,7 @@ private void execInternal() throws Exception { sendPipelineCtx(); } - private void sendPipelineCtx() throws TException, RpcException, UserException { + protected void sendPipelineCtx() throws TException, RpcException, UserException { lock(); try { Multiset hostCounter = HashMultiset.create(); @@ -831,7 +830,7 @@ private void sendPipelineCtx() throws TException, RpcException, UserException { Long backendId = this.addressToBackendID.get(entry.getKey()); backendFragments.add(Pair.of(fragment.getFragmentId(), backendId)); PipelineExecContext pipelineExecContext = new PipelineExecContext(fragment.getFragmentId(), - entry.getValue(), backendId, executionProfile); + entry.getValue(), idToBackend.get(backendId), executionProfile, jobId); // Each tParam will set the total number of Fragments that need to be executed on the same BE, // and the BE will determine whether all Fragments have been executed based on this information. // Notice. load fragment has a small probability that FragmentNumOnHost is 0, for unknown reasons. @@ -851,7 +850,7 @@ private void sendPipelineCtx() throws TException, RpcException, UserException { PipelineExecContexts ctxs = beToPipelineExecCtxs.get(pipelineExecContext.backend.getId()); if (ctxs == null) { - ctxs = new PipelineExecContexts(pipelineExecContext.backend.getId(), + ctxs = new PipelineExecContexts(queryId, pipelineExecContext.backend, pipelineExecContext.brpcAddress, twoPhaseExecution, entry.getValue().getFragmentNumOnHost()); beToPipelineExecCtxs.putIfAbsent(pipelineExecContext.backend.getId(), ctxs); @@ -946,7 +945,7 @@ private void sendPipelineCtx() throws TException, RpcException, UserException { } } - private Map> waitPipelineRpc(List> waitPipelineRpc(List>>> futures, long leftTimeMs, String operation) throws RpcException, UserException { if (leftTimeMs <= 0) { @@ -1326,7 +1325,7 @@ private void cancelLatch() { } } - private void cancelInternal(Status cancelReason) { + protected void cancelInternal(Status cancelReason) { for (ResultReceiver receiver : receivers) { receiver.cancel(cancelReason); } @@ -1781,7 +1780,7 @@ protected void computeFragmentHosts() throws Exception { params.instanceExecParams.add(instanceParam); // Using serial source means a serial source operator will be used in this fragment (e.g. data will be - // shuffled to only 1 exchange operator) and then splitted by followed local exchanger + // shuffled to only 1 exchange operator) and then split by followed local exchanger int expectedInstanceNum = fragment.getParallelExecNum(); boolean useSerialSource = fragment.useSerialSource(context); if (useSerialSource) { @@ -1888,17 +1887,11 @@ protected void computeFragmentHosts() throws Exception { return scanNode.getId().asInt() == planNodeId; }).findFirst(); - /** - * Ignore storage data distribution iff: - * 1. `parallelExecInstanceNum * numBackends` is larger than scan ranges. - * 2. Use Nereids planner. - */ boolean sharedScan = true; int expectedInstanceNum = Math.min(parallelExecInstanceNum, leftMostNode.getNumInstances()); - boolean ignoreStorageDataDistribution = node.isPresent() - && fragment.useSerialSource(context); - if (node.isPresent() && ignoreStorageDataDistribution) { + boolean ignoreStorageDataDistribution = fragment.useSerialSource(context); + if (ignoreStorageDataDistribution) { expectedInstanceNum = Math.max(expectedInstanceNum, 1); // if have limit and no conjuncts, only need 1 instance to save cpu and // mem resource @@ -2821,7 +2814,7 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc protected final BucketShuffleJoinController bucketShuffleJoinController = new BucketShuffleJoinController(fragmentIdToScanNodeIds); - public class PipelineExecContext { + public static class PipelineExecContext { TPipelineFragmentParams rpcParams; PlanFragmentId fragmentId; boolean initiated; @@ -2832,23 +2825,25 @@ public class PipelineExecContext { Backend backend; long lastMissingHeartbeatTime = -1; long beProcessEpoch = 0; + private long jobId; public PipelineExecContext(PlanFragmentId fragmentId, - TPipelineFragmentParams rpcParams, Long backendId, - ExecutionProfile executionProfile) { + TPipelineFragmentParams rpcParams, Backend backend, + ExecutionProfile executionProfile, long jobId) { this.fragmentId = fragmentId; this.rpcParams = rpcParams; this.initiated = false; this.done = false; - this.backend = idToBackend.get(backendId); + this.backend = backend; this.address = new TNetworkAddress(backend.getHost(), backend.getBePort()); this.brpcAddress = new TNetworkAddress(backend.getHost(), backend.getBrpcPort()); this.beProcessEpoch = backend.getProcessEpoch(); + this.jobId = jobId; this.lastMissingHeartbeatTime = backend.getLastMissingHeartbeatTime(); - executionProfile.addFragmentBackend(fragmentId, backendId); + executionProfile.addFragmentBackend(fragmentId, backend.getId()); } /** @@ -2900,8 +2895,10 @@ public List buildFragmentInstanceInfo( } } - public class PipelineExecContexts { + public static class PipelineExecContexts { + TUniqueId queryId; long beId; + Backend backend; TNetworkAddress brpcAddr; List ctxs = Lists.newArrayList(); boolean twoPhaseExecution = false; @@ -2910,9 +2907,12 @@ public class PipelineExecContexts { boolean hasCancelled = false; boolean cancelInProcess = false; - public PipelineExecContexts(long beId, TNetworkAddress brpcAddr, boolean twoPhaseExecution, + public PipelineExecContexts(TUniqueId queryId, + Backend backend, TNetworkAddress brpcAddr, boolean twoPhaseExecution, int instanceNumber) { - this.beId = beId; + this.queryId = queryId; + this.backend = backend; + this.beId = backend.getId(); this.brpcAddr = brpcAddr; this.twoPhaseExecution = twoPhaseExecution; this.instanceNumber = instanceNumber; @@ -2926,6 +2926,14 @@ public int getInstanceNumber() { return instanceNumber; } + public List getCtxs() { + return ctxs; + } + + public Backend getBackend() { + return backend; + } + /** * The BackendExecState in states are all send to the same BE. * So only the first BackendExecState need to carry some common fields, such as DescriptorTbl, @@ -3001,6 +3009,14 @@ public PExecPlanFragmentResult get(long timeout, TimeUnit unit) { }; } + public void setSerializeFragments(ByteString serializedFragments) { + this.serializedFragments = serializedFragments; + } + + public ByteString getSerializedFragments() { + return serializedFragments; + } + public long serializeFragments() throws TException { TPipelineFragmentParamsList paramsList = new TPipelineFragmentParamsList(); for (PipelineExecContext cts : ctxs) { @@ -3026,7 +3042,7 @@ public String debugInfo() { private synchronized void cancelQuery(Status cancelReason) { if (LOG.isDebugEnabled()) { LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", - idToBackend.get(beId), DebugUtil.printId(queryId), cancelReason.toString()); + backend, DebugUtil.printId(queryId), cancelReason.toString()); } if (this.hasCancelled || this.cancelInProcess) { @@ -3047,7 +3063,7 @@ public void onSuccess(InternalService.PCancelPlanFragmentResult result) { hasCancelled = true; } else { LOG.warn("Failed to cancel query {} backend: {}, reason: {}", - DebugUtil.printId(queryId), idToBackend.get(beId), status.toString()); + DebugUtil.printId(queryId), backend, status.toString()); } } } @@ -3055,7 +3071,7 @@ public void onSuccess(InternalService.PCancelPlanFragmentResult result) { public void onFailure(Throwable t) { cancelInProcess = false; LOG.warn("Failed to cancel query {} backend: {}, reason: {}", - DebugUtil.printId(queryId), idToBackend.get(beId), cancelReason.toString(), t); + DebugUtil.printId(queryId), backend, cancelReason.toString(), t); } }, backendRpcCallbackExecutor); cancelInProcess = true; @@ -3373,15 +3389,11 @@ public List getInvolvedBackends() { return backendAddresses; } - public Map getFragmentExecParamsMap() { - return fragmentExecParamsMap; - } - public List getFragments() { return fragments; } - private void updateProfileIfPresent(Consumer profileAction) { + protected void updateProfileIfPresent(Consumer profileAction) { Optional.ofNullable(context).map(ConnectContext::getExecutor).map(StmtExecutor::getSummaryProfile) .ifPresent(profileAction); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java new file mode 100644 index 00000000000000..2ab94808f27ac4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -0,0 +1,426 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.Status; +import org.apache.doris.common.profile.ExecutionProfile; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.mysql.MysqlCommand; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.nereids.trees.plans.physical.TopnFilter; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.RuntimeFilter; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.qe.runtime.LoadProcessor; +import org.apache.doris.qe.runtime.QueryProcessor; +import org.apache.doris.resource.workloadgroup.QueryQueue; +import org.apache.doris.resource.workloadgroup.QueueToken; +import org.apache.doris.service.ExecuteEnv; +import org.apache.doris.thrift.TDescriptorTable; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPipelineWorkloadGroup; +import org.apache.doris.thrift.TQueryGlobals; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TResourceLimit; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class CoordinatorContext { + private static final Logger LOG = LogManager.getLogger(CoordinatorContext.class); + + // these are some constant parameters + public final NereidsCoordinator coordinator; + public final List fragments; + public final boolean isBlockQuery; + public final DataSink dataSink; + public final ExecutionProfile executionProfile; + public final ConnectContext connectContext; + public final PipelineDistributedPlan topDistributedPlan; + public final List distributedPlans; + public final TUniqueId queryId; + public final TQueryGlobals queryGlobals; + public final TQueryOptions queryOptions; + public final TDescriptorTable descriptorTable; + public final TNetworkAddress coordinatorAddress = new TNetworkAddress(Coordinator.localIP, Config.rpc_port); + // public final TNetworkAddress directConnectFrontendAddress; + public final List runtimeFilters; + public final List topnFilters; + public final List scanNodes; + public final Supplier timeoutDeadline = Suppliers.memoize(this::computeTimeoutDeadline); + public final Supplier instanceNum = Suppliers.memoize(this::computeInstanceNum); + public final Supplier> instanceIds = Suppliers.memoize(this::getInstanceIds); + public final Supplier> backends = Suppliers.memoize(this::getBackends); + public final Supplier scanRangeNum = Suppliers.memoize(this::getScanRangeNum); + public final Supplier directConnectFrontendAddress + = Suppliers.memoize(this::computeDirectConnectCoordinator); + + // these are some mutable states + private volatile Status status = new Status(); + private volatile Optional queryQueue = Optional.empty(); + private volatile Optional queueToken = Optional.empty(); + private volatile List workloadGroups = ImmutableList.of(); + + // query or load processor + private volatile JobProcessor jobProcessor; + + // for sql execution + private CoordinatorContext( + NereidsCoordinator coordinator, + ConnectContext connectContext, + boolean isBlockQuery, + List distributedPlans, + List fragments, + List runtimeFilters, + List topnFilters, + List scanNodes, + ExecutionProfile executionProfile, + TQueryGlobals queryGlobals, + TQueryOptions queryOptions, + TDescriptorTable descriptorTable) { + this.connectContext = connectContext; + this.isBlockQuery = isBlockQuery; + this.fragments = fragments; + this.distributedPlans = distributedPlans; + this.topDistributedPlan = distributedPlans.get(distributedPlans.size() - 1); + this.dataSink = topDistributedPlan.getFragmentJob().getFragment().getSink(); + this.runtimeFilters = runtimeFilters == null ? Lists.newArrayList() : runtimeFilters; + this.topnFilters = topnFilters == null ? Lists.newArrayList() : topnFilters; + this.scanNodes = scanNodes; + this.queryId = connectContext.queryId(); + this.executionProfile = executionProfile; + this.queryGlobals = queryGlobals; + this.queryOptions = queryOptions; + this.descriptorTable = descriptorTable; + + this.coordinator = Objects.requireNonNull(coordinator, "coordinator can not be null"); + } + + // for broker load + private CoordinatorContext( + NereidsCoordinator coordinator, + long jobId, + List fragments, + List distributedPlans, + List scanNodes, + TUniqueId queryId, + TQueryOptions queryOptions, + TQueryGlobals queryGlobals, + TDescriptorTable descriptorTable, + ExecutionProfile executionProfile) { + this.coordinator = coordinator; + this.isBlockQuery = true; + this.fragments = fragments; + this.distributedPlans = distributedPlans; + this.topDistributedPlan = distributedPlans.get(distributedPlans.size() - 1); + this.dataSink = topDistributedPlan.getFragmentJob().getFragment().getSink(); + this.scanNodes = scanNodes; + this.queryId = queryId; + this.queryOptions = queryOptions; + this.queryGlobals = queryGlobals; + this.descriptorTable = descriptorTable; + this.executionProfile = executionProfile; + + this.connectContext = ConnectContext.get(); + this.runtimeFilters = ImmutableList.of(); + this.topnFilters = ImmutableList.of(); + this.jobProcessor = new LoadProcessor(this, jobId); + } + + public void setQueueInfo(QueryQueue queryQueue, QueueToken queueToken) { + this.queryQueue = Optional.ofNullable(queryQueue); + this.queueToken = Optional.ofNullable(queueToken); + } + + public Optional getQueueToken() { + return this.queueToken; + } + + public Optional getQueryQueue() { + return queryQueue; + } + + public void setWorkloadGroups(List workloadGroups) { + this.workloadGroups = workloadGroups; + } + + public List getWorkloadGroups() { + return workloadGroups; + } + + public void updateProfileIfPresent(Consumer profileAction) { + Optional.ofNullable(connectContext) + .map(ConnectContext::getExecutor) + .map(StmtExecutor::getSummaryProfile) + .ifPresent(profileAction); + } + + public boolean twoPhaseExecution() { + // If #fragments >=2, use twoPhaseExecution with exec_plan_fragments_prepare and exec_plan_fragments_start, + // else use exec_plan_fragments directly. + // we choose #fragments > 1 because in some cases + // we need ensure that A fragment is already prepared to receive data before B fragment sends data. + // For example: select * from numbers("number"="10") will generate ExchangeNode and + // TableValuedFunctionScanNode, we should ensure TableValuedFunctionScanNode does not + // send data until ExchangeNode is ready to receive. + return distributedPlans.size() > 1; + } + + public boolean isEos() { + return jobProcessor instanceof QueryProcessor && coordinator.isEos(); + } + + public void cancelSchedule(Status cancelReason) { + coordinator.cancelInternal(cancelReason); + } + + public synchronized void withLock(Runnable callback) { + callback.run(); + } + + public synchronized T withLock(Callable callback) throws Exception { + return callback.call(); + } + + public synchronized Status readCloneStatus() { + return new Status(status.getErrorCode(), status.getErrorMsg()); + } + + public synchronized Status updateStatusIfOk(Status newStatus) { + // If query is done, we will ignore their cancelled updates, and let the remote fragments to clean up async. + Status originStatus = readCloneStatus(); + if (coordinator.getJobProcessor() instanceof QueryProcessor && coordinator.isEos() + && newStatus.isCancelled()) { + return originStatus; + } + // nothing to update + if (newStatus.ok()) { + return originStatus; + } + + // don't override an error status; also, cancellation has already started + if (!this.status.ok()) { + return originStatus; + } + + status = new Status(newStatus.getErrorCode(), newStatus.getErrorMsg()); + coordinator.cancelInternal(readCloneStatus()); + return originStatus; + } + + public void setJobProcessor(JobProcessor jobProcessor) { + this.jobProcessor = jobProcessor; + } + + public JobProcessor getJobProcessor() { + return jobProcessor; + } + + public LoadProcessor asLoadProcessor() { + return (LoadProcessor) jobProcessor; + } + + public QueryProcessor asQueryProcessor() { + return (QueryProcessor) jobProcessor; + } + + public static CoordinatorContext buildForSql(NereidsPlanner planner, NereidsCoordinator coordinator) { + ConnectContext connectContext = planner.getCascadesContext().getConnectContext(); + TQueryOptions queryOptions = initQueryOptions(connectContext); + TQueryGlobals queryGlobals = initQueryGlobals(connectContext); + TDescriptorTable descriptorTable = planner.getDescTable().toThrift(); + + ExecutionProfile executionProfile = new ExecutionProfile( + connectContext.queryId, + planner.getFragments() + .stream() + .map(fragment -> fragment.getFragmentId().asInt()) + .collect(Collectors.toList()) + ); + return new CoordinatorContext( + coordinator, connectContext, planner.isBlockQuery(), + planner.getDistributedPlans().valueList(), + planner.getFragments(), planner.getRuntimeFilters(), planner.getTopnFilters(), + planner.getScanNodes(), executionProfile, queryGlobals, queryOptions, descriptorTable + ); + } + + public static CoordinatorContext buildForLoad( + NereidsCoordinator coordinator, + long jobId, TUniqueId queryId, + List fragments, + List distributedPlans, + List scanNodes, + DescriptorTable descTable, + String timezone, boolean loadZeroTolerance, + boolean enableProfile) { + TQueryOptions queryOptions = new TQueryOptions(); + queryOptions.setEnableProfile(enableProfile); + queryOptions.setBeExecVersion(Config.be_exec_version); + + TQueryGlobals queryGlobals = new TQueryGlobals(); + queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); + queryGlobals.setTimestampMs(System.currentTimeMillis()); + queryGlobals.setTimeZone(timezone); + queryGlobals.setLoadZeroTolerance(loadZeroTolerance); + + ExecutionProfile executionProfile = new ExecutionProfile( + queryId, + fragments.stream() + .map(fragment -> fragment.getFragmentId().asInt()) + .collect(Collectors.toList()) + ); + + return new CoordinatorContext(coordinator, jobId, fragments, distributedPlans, + scanNodes, queryId, queryOptions, queryGlobals, descTable.toThrift(), + executionProfile); + } + + private static TQueryOptions initQueryOptions(ConnectContext context) { + TQueryOptions queryOptions = context.getSessionVariable().toThrift(); + queryOptions.setBeExecVersion(Config.be_exec_version); + queryOptions.setQueryTimeout(context.getExecTimeout()); + queryOptions.setExecutionTimeout(context.getExecTimeout()); + if (queryOptions.getExecutionTimeout() < 1) { + LOG.info("try set timeout less than 1", new RuntimeException("")); + } + queryOptions.setFeProcessUuid(ExecuteEnv.getInstance().getProcessUUID()); + queryOptions.setMysqlRowBinaryFormat(context.getCommand() == MysqlCommand.COM_STMT_EXECUTE); + + setOptionsFromUserProperty(context, queryOptions); + return queryOptions; + } + + private static TQueryGlobals initQueryGlobals(ConnectContext context) { + TQueryGlobals queryGlobals = new TQueryGlobals(); + queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); + queryGlobals.setTimestampMs(System.currentTimeMillis()); + queryGlobals.setNanoSeconds(LocalDateTime.now().getNano()); + queryGlobals.setLoadZeroTolerance(false); + if (context.getSessionVariable().getTimeZone().equals("CST")) { + queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE); + } else { + queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); + } + return queryGlobals; + } + + private static void setOptionsFromUserProperty(ConnectContext connectContext, TQueryOptions queryOptions) { + String qualifiedUser = connectContext.getQualifiedUser(); + // set cpu resource limit + int cpuLimit = Env.getCurrentEnv().getAuth().getCpuResourceLimit(qualifiedUser); + if (cpuLimit > 0) { + // overwrite the cpu resource limit from session variable; + TResourceLimit resourceLimit = new TResourceLimit(); + resourceLimit.setCpuLimit(cpuLimit); + queryOptions.setResourceLimit(resourceLimit); + } + // set exec mem limit + long maxExecMemByte = connectContext.getSessionVariable().getMaxExecMemByte(); + long memLimit = maxExecMemByte > 0 ? maxExecMemByte : + Env.getCurrentEnv().getAuth().getExecMemLimit(qualifiedUser); + if (memLimit > 0) { + // overwrite the exec_mem_limit from session variable; + queryOptions.setMemLimit(memLimit); + queryOptions.setMaxReservation(memLimit); + queryOptions.setInitialReservationTotalClaims(memLimit); + queryOptions.setBufferPoolLimit(memLimit); + } + } + + private Set getInstanceIds() { + Set instanceIds = Sets.newLinkedHashSet(); + for (DistributedPlan distributedPlan : distributedPlans) { + PipelineDistributedPlan pipelinePlan = (PipelineDistributedPlan) distributedPlan; + List instanceJobs = pipelinePlan.getInstanceJobs(); + for (AssignedJob instanceJob : instanceJobs) { + instanceIds.add(instanceJob.instanceId()); + } + } + return instanceIds; + } + + private Integer computeInstanceNum() { + return distributedPlans + .stream() + .map(plan -> plan.getInstanceJobs().size()) + .reduce(Integer::sum) + .get(); + } + + private long computeTimeoutDeadline() { + return System.currentTimeMillis() + queryOptions.getExecutionTimeout() * 1000L; + } + + private Map getBackends() { + Map backends = Maps.newLinkedHashMap(); + for (DistributedPlan distributedPlan : distributedPlans) { + PipelineDistributedPlan pipelinePlan = (PipelineDistributedPlan) distributedPlan; + List instanceJobs = pipelinePlan.getInstanceJobs(); + for (AssignedJob instanceJob : instanceJobs) { + DistributedPlanWorker worker = instanceJob.getAssignedWorker(); + backends.put(new TNetworkAddress(worker.address(), worker.port()), worker.id()); + } + } + return backends; + } + + private TNetworkAddress computeDirectConnectCoordinator() { + if (connectContext != null && connectContext.isProxy() + && !StringUtils.isEmpty(connectContext.getCurrentConnectedFEIp())) { + return new TNetworkAddress(ConnectContext.get().getCurrentConnectedFEIp(), Config.rpc_port); + } else { + return coordinatorAddress; + } + } + + private int getScanRangeNum() { + int scanRangeNum = 0; + for (ScanNode scanNode : scanNodes) { + scanRangeNum += scanNode.getScanRangeNum(); + } + return scanRangeNum; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index b449a4bc8e9b4e..45a6acba6dc38f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -33,6 +33,13 @@ // NOTE: If you want access your variable safe, please hold VariableMgr's lock before access. public final class GlobalVariable { + public static final int VARIABLE_VERSION_0 = 0; + public static final int VARIABLE_VERSION_100 = 100; + public static final int VARIABLE_VERSION_200 = 200; + public static final int VARIABLE_VERSION_300 = 300; + public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300; + public static final String VARIABLE_VERSION = "variable_version"; + public static final String VERSION_COMMENT = "version_comment"; public static final String VERSION = "version"; public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; @@ -69,6 +76,9 @@ public final class GlobalVariable { public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats"; + @VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE + | VariableMgr.READ_ONLY | VariableMgr.GLOBAL) + public static int variableVersion = CURRENT_VARIABLE_VERSION; @VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY) public static String versionComment = "Doris version " diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java new file mode 100644 index 00000000000000..ede218848c7221 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/JobProcessor.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.common.Status; +import org.apache.doris.qe.runtime.PipelineExecutionTask; + +public interface JobProcessor { + void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask); + + void cancel(Status cancelReason); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/LoadContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/LoadContext.java new file mode 100644 index 00000000000000..958aa6d1122d31 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/LoadContext.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; +import org.apache.doris.load.loadv2.LoadJob; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.task.LoadEtlTask; +import org.apache.doris.thrift.TErrorTabletInfo; +import org.apache.doris.thrift.TTabletCommitInfo; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class LoadContext { + + private volatile String trackingUrl; + private volatile long transactionId; + private volatile String label; + private final List exportFiles = Lists.newCopyOnWriteArrayList(); + private final Map loadCounters = Maps.newLinkedHashMap(); + private final List deltaUrls = Lists.newCopyOnWriteArrayList(); + private final List errorTabletInfos = Lists.newCopyOnWriteArrayList(); + + // in pipelinex, the commit info may be duplicate, so we remove the duplicate ones + // key: backendsId + // values: tabletId + private final Map, TTabletCommitInfo> commitInfoMap = Maps.newLinkedHashMap(); + + public synchronized Map getLoadCounters() { + return ImmutableMap.copyOf(loadCounters); + } + + public synchronized void updateLoadCounters(Map newLoadCounters) { + long numRowsNormal = 0L; + String value = this.loadCounters.get(LoadEtlTask.DPP_NORMAL_ALL); + if (value != null) { + numRowsNormal = Long.parseLong(value); + } + long numRowsAbnormal = 0L; + value = this.loadCounters.get(LoadEtlTask.DPP_ABNORMAL_ALL); + if (value != null) { + numRowsAbnormal = Long.parseLong(value); + } + long numRowsUnselected = 0L; + value = this.loadCounters.get(LoadJob.UNSELECTED_ROWS); + if (value != null) { + numRowsUnselected = Long.parseLong(value); + } + + // new load counters + value = newLoadCounters.get(LoadEtlTask.DPP_NORMAL_ALL); + if (value != null) { + numRowsNormal += Long.parseLong(value); + } + value = newLoadCounters.get(LoadEtlTask.DPP_ABNORMAL_ALL); + if (value != null) { + numRowsAbnormal += Long.parseLong(value); + } + value = newLoadCounters.get(LoadJob.UNSELECTED_ROWS); + if (value != null) { + numRowsUnselected += Long.parseLong(value); + } + + this.loadCounters.put(LoadEtlTask.DPP_NORMAL_ALL, Long.toString(numRowsNormal)); + this.loadCounters.put(LoadEtlTask.DPP_ABNORMAL_ALL, Long.toString(numRowsAbnormal)); + this.loadCounters.put(LoadJob.UNSELECTED_ROWS, Long.toString(numRowsUnselected)); + } + + public List getDeltaUrls() { + return Utils.fastToImmutableList(deltaUrls); + } + + public void updateDeltaUrls(List deltaUrls) { + if (!deltaUrls.isEmpty()) { + this.deltaUrls.addAll(deltaUrls); + } + } + + public synchronized void updateCommitInfos(List commitInfos) { + // distinct commit info in the map + for (TTabletCommitInfo commitInfo : commitInfos) { + this.commitInfoMap.put(Pair.of(commitInfo.backendId, commitInfo.tabletId), commitInfo); + } + } + + public synchronized List getCommitInfos() { + return Utils.fastToImmutableList(commitInfoMap.values()); + } + + public void updateTrackingUrl(String trackingUrl) { + this.trackingUrl = trackingUrl; + } + + public String getTrackingUrl() { + return trackingUrl; + } + + public void updateTransactionId(long transactionId) { + this.transactionId = transactionId; + } + + public long getTransactionId() { + return transactionId; + } + + public String getLabel() { + return label; + } + + public void updateLabel(String label) { + this.label = label; + } + + public void addExportFiles(List files) { + this.exportFiles.addAll(files); + } + + public List getExportFiles() { + return exportFiles; + } + + public synchronized void updateErrorTabletInfos(List errorTabletInfos) { + if (this.errorTabletInfos.size() <= Config.max_error_tablet_of_broker_load) { + this.errorTabletInfos.addAll(errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load + - this.errorTabletInfos.size()).collect(Collectors.toList())); + } + } + + public List getErrorTabletInfos() { + return errorTabletInfos; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index d5f4ab7dfcccea..376d4740e632a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -151,6 +151,7 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS executor.execute(); if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { stmtStr = executeStmt.toSql(); + stmtStr = stmtStr + " /*originalSql = " + prepareCommand.getOriginalStmt().originStmt + "*/"; } } catch (Throwable e) { // Catch all throwable. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4f5af3762c51af..a14aea25463ad4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -18,177 +18,490 @@ package org.apache.doris.qe; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.analysis.StorageBackend; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FsBroker; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.Status; +import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.ExecutionProfile; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.stats.StatsErrorEstimator; -import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; -import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping; import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanRanges; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource; -import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; -import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.DataSink; import org.apache.doris.planner.PlanFragment; -import org.apache.doris.planner.Planner; +import org.apache.doris.planner.ResultFileSink; +import org.apache.doris.planner.ResultSink; import org.apache.doris.planner.ScanNode; +import org.apache.doris.planner.SchemaScanNode; +import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.qe.QueryStatisticsItem.FragmentInstanceInfo; +import org.apache.doris.qe.runtime.LoadProcessor; +import org.apache.doris.qe.runtime.MultiFragmentsPipelineTask; +import org.apache.doris.qe.runtime.PipelineExecutionTask; +import org.apache.doris.qe.runtime.PipelineExecutionTaskBuilder; +import org.apache.doris.qe.runtime.QueryProcessor; +import org.apache.doris.qe.runtime.SingleFragmentPipelineTask; +import org.apache.doris.qe.runtime.ThriftPlansBuilder; +import org.apache.doris.resource.workloadgroup.QueryQueue; +import org.apache.doris.resource.workloadgroup.QueueToken; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TErrorTabletInfo; import org.apache.doris.thrift.TNetworkAddress; -import org.apache.doris.thrift.TScanRangeParams; +import org.apache.doris.thrift.TPipelineFragmentParamsList; +import org.apache.doris.thrift.TPipelineWorkloadGroup; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TQueryType; +import org.apache.doris.thrift.TReportExecStatusParams; +import org.apache.doris.thrift.TTabletCommitInfo; +import org.apache.doris.thrift.TUniqueId; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; /** NereidsCoordinator */ public class NereidsCoordinator extends Coordinator { - private NereidsPlanner nereidsPlanner; - private FragmentIdMapping distributedPlans; + private static final Logger LOG = LogManager.getLogger(NereidsCoordinator.class); + protected final CoordinatorContext coordinatorContext; + + protected volatile PipelineExecutionTask executionTask; + + // sql execution public NereidsCoordinator(ConnectContext context, Analyzer analyzer, - Planner planner, StatsErrorEstimator statsErrorEstimator, NereidsPlanner nereidsPlanner) { + NereidsPlanner planner, StatsErrorEstimator statsErrorEstimator) { super(context, analyzer, planner, statsErrorEstimator); - this.nereidsPlanner = Objects.requireNonNull(nereidsPlanner, "nereidsPlanner can not be null"); - this.distributedPlans = Objects.requireNonNull( - nereidsPlanner.getDistributedPlans(), "distributedPlans can not be null" + + this.coordinatorContext = CoordinatorContext.buildForSql(planner, this); + this.coordinatorContext.setJobProcessor(buildJobProcessor(coordinatorContext)); + + Preconditions.checkState(!planner.getFragments().isEmpty() + && coordinatorContext.instanceNum.get() > 0, "Fragment and Instance can not be empty˚"); + } + + // broker load + public NereidsCoordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, + List fragments, List distributedPlans, + List scanNodes, String timezone, boolean loadZeroTolerance, + boolean enableProfile) { + super(jobId, queryId, descTable, fragments, scanNodes, timezone, loadZeroTolerance, enableProfile); + this.coordinatorContext = CoordinatorContext.buildForLoad( + this, jobId, queryId, fragments, distributedPlans, scanNodes, + descTable, timezone, loadZeroTolerance, enableProfile ); + + Preconditions.checkState(!fragments.isEmpty() + && coordinatorContext.instanceNum.get() > 0, "Fragment and Instance can not be empty˚"); } @Override - protected void processFragmentAssignmentAndParams() throws Exception { - // prepare information - prepare(); + public void exec() throws Exception { + enqueue(coordinatorContext.connectContext); + + processTopSink(coordinatorContext, coordinatorContext.topDistributedPlan); + + QeProcessorImpl.INSTANCE.registerInstances(coordinatorContext.queryId, coordinatorContext.instanceNum.get()); + + Map workerToFragments + = ThriftPlansBuilder.plansToThrift(coordinatorContext); + executionTask = PipelineExecutionTaskBuilder.build(coordinatorContext, workerToFragments); + executionTask.execute(); + } - computeFragmentExecParams(); + @Override + public boolean isTimeout() { + return System.currentTimeMillis() > coordinatorContext.timeoutDeadline.get(); } @Override - protected void computeFragmentHosts() { - // translate distributed plan to params - for (DistributedPlan distributedPlan : distributedPlans.values()) { - UnassignedJob fragmentJob = distributedPlan.getFragmentJob(); - PlanFragment fragment = fragmentJob.getFragment(); + public void cancel(Status cancelReason) { + coordinatorContext.getQueueToken().ifPresent(QueueToken::cancel); - bucketShuffleJoinController - .isBucketShuffleJoin(fragment.getFragmentId().asInt(), fragment.getPlanRoot()); + for (ScanNode scanNode : coordinatorContext.scanNodes) { + scanNode.stop(); + } - setFileScanParams(distributedPlan); + if (cancelReason.ok()) { + throw new RuntimeException("Should use correct cancel reason, but it is " + cancelReason); + } - FragmentExecParams fragmentExecParams = fragmentExecParamsMap.computeIfAbsent( - fragment.getFragmentId(), id -> new FragmentExecParams(fragment) - ); - List instanceJobs = ((PipelineDistributedPlan) distributedPlan).getInstanceJobs(); - boolean useLocalShuffle = useLocalShuffle(distributedPlan); - if (useLocalShuffle) { - fragmentExecParams.ignoreDataDistribution = true; - fragmentExecParams.parallelTasksNum = 1; - } else { - fragmentExecParams.parallelTasksNum = instanceJobs.size(); + TUniqueId queryId = coordinatorContext.queryId; + Status originQueryStatus = coordinatorContext.updateStatusIfOk(cancelReason); + if (!originQueryStatus.ok()) { + if (LOG.isDebugEnabled()) { + // Print an error stack here to know why send cancel again. + LOG.warn("Query {} already in abnormal status {}, but received cancel again," + + "so that send cancel to BE again", + DebugUtil.printId(queryId), originQueryStatus.toString(), + new Exception("cancel failed")); } + } else { + LOG.warn("Cancel execution of query {}, this is a outside invoke, cancelReason {}", + DebugUtil.printId(queryId), cancelReason); + } + cancelInternal(cancelReason); + } + + public QueryProcessor asQueryProcessor() { + return coordinatorContext.asQueryProcessor(); + } + + public JobProcessor getJobProcessor() { + return coordinatorContext.getJobProcessor(); + } + + public LoadProcessor asLoadProcessor() { + return coordinatorContext.asLoadProcessor(); + } + + @Override + public void setTWorkloadGroups(List tWorkloadGroups) { + coordinatorContext.setWorkloadGroups(tWorkloadGroups); + } + + @Override + public List getTWorkloadGroups() { + return coordinatorContext.getWorkloadGroups(); + } + + @Override + public boolean isQueryCancelled() { + return coordinatorContext.readCloneStatus().isCancelled(); + } + + @Override + public RowBatch getNext() throws Exception { + return coordinatorContext.asQueryProcessor().getNext(); + } + + public boolean isEos() { + return coordinatorContext.asQueryProcessor().isEos(); + } + + @Override + public long getNumReceivedRows() { + return coordinatorContext.asQueryProcessor().getNumReceivedRows(); + } + + @Override + public long getJobId() { + return coordinatorContext.asLoadProcessor().jobId; + } + + /* + * Waiting the coordinator finish executing. + * return false if waiting timeout. + * return true otherwise. + * NOTICE: return true does not mean that coordinator executed success, + * the caller should check queryStatus for result. + * + * We divide the entire waiting process into multiple rounds, + * with a maximum of 30 seconds per round. And after each round of waiting, + * check the status of the BE. If the BE status is abnormal, the wait is ended + * and the result is returned. Otherwise, continue to the next round of waiting. + * This method mainly avoids the problem that the Coordinator waits for a long time + * after some BE can no long return the result due to some exception, such as BE is down. + */ + @Override + public boolean join(int timeoutS) { + return coordinatorContext.asLoadProcessor().join(timeoutS); + } + + @Override + public boolean isDone() { + return coordinatorContext.asLoadProcessor().isDone(); + } + + @Override + public void updateFragmentExecStatus(TReportExecStatusParams params) { + JobProcessor jobProcessor = coordinatorContext.getJobProcessor(); + if (jobProcessor instanceof LoadProcessor) { + coordinatorContext.asLoadProcessor().updateFragmentExecStatus(params); + } + } + + @Override + public TUniqueId getQueryId() { + return coordinatorContext.queryId; + } + + @Override + public TQueryOptions getQueryOptions() { + return coordinatorContext.queryOptions; + } + + @Override + public Status getExecStatus() { + return coordinatorContext.readCloneStatus(); + } + + @Override + public void setQueryType(TQueryType type) { + coordinatorContext.queryOptions.setQueryType(type); + } + + @Override + public void setLoadZeroTolerance(boolean loadZeroTolerance) { + coordinatorContext.queryGlobals.setLoadZeroTolerance(loadZeroTolerance); + } + + @Override + public int getScanRangeNum() { + return coordinatorContext.scanRangeNum.get(); + } + + @Override + public ConnectContext getConnectContext() { + return coordinatorContext.connectContext; + } + + @Override + public QueueToken getQueueToken() { + return coordinatorContext.getQueueToken().orElse(null); + } + + @Override + public Map getLoadCounters() { + return coordinatorContext.asLoadProcessor().loadContext.getLoadCounters(); + } - for (AssignedJob instanceJob : instanceJobs) { - DistributedPlanWorker worker = instanceJob.getAssignedWorker(); - TNetworkAddress address = new TNetworkAddress(worker.host(), worker.port()); - FInstanceExecParam instanceExecParam = new FInstanceExecParam( - null, address, 0, fragmentExecParams); - instanceExecParam.instanceId = instanceJob.instanceId(); - fragmentExecParams.instanceExecParams.add(instanceExecParam); - addressToBackendID.put(address, worker.id()); - ScanSource scanSource = instanceJob.getScanSource(); - if (scanSource instanceof BucketScanSource) { - setForBucketScanSource(instanceExecParam, (BucketScanSource) scanSource, useLocalShuffle); - } else { - setForDefaultScanSource(instanceExecParam, (DefaultScanSource) scanSource, useLocalShuffle); + @Override + public List getDeltaUrls() { + return coordinatorContext.asLoadProcessor().loadContext.getDeltaUrls(); + } + + @Override + public List getCommitInfos() { + return coordinatorContext.asLoadProcessor().loadContext.getCommitInfos(); + } + + @Override + public List getExportFiles() { + return coordinatorContext.asLoadProcessor().loadContext.getExportFiles(); + } + + @Override + public long getTxnId() { + return coordinatorContext.asLoadProcessor().loadContext.getTransactionId(); + } + + @Override + public void setTxnId(long txnId) { + coordinatorContext.asLoadProcessor().loadContext.updateTransactionId(txnId); + } + + @Override + public String getLabel() { + return coordinatorContext.asLoadProcessor().loadContext.getLabel(); + } + + @Override + public String getTrackingUrl() { + return coordinatorContext.asLoadProcessor().loadContext.getTrackingUrl(); + } + + @Override + public List getErrorTabletInfos() { + return coordinatorContext.asLoadProcessor().loadContext.getErrorTabletInfos(); + } + + @Override + public List getInvolvedBackends() { + return Utils.fastToImmutableList(coordinatorContext.backends.get().keySet()); + } + + @Override + public List getFragmentInstanceInfos() { + List infos = Lists.newArrayList(); + if (executionTask != null) { + for (MultiFragmentsPipelineTask multiFragmentsPipelineTask : executionTask.getChildrenTasks().values()) { + for (SingleFragmentPipelineTask fragmentTask : multiFragmentsPipelineTask.getChildrenTasks().values()) { + infos.addAll(fragmentTask.buildFragmentInstanceInfo()); } } } + infos.sort(Comparator.comparing(FragmentInstanceInfo::getFragmentId)); + return infos; + } + + @Override + public List getFragments() { + return coordinatorContext.fragments; + } + + @Override + public ExecutionProfile getExecutionProfile() { + return coordinatorContext.executionProfile; + } + + @Override + public void setMemTableOnSinkNode(boolean enableMemTableOnSinkNode) { + coordinatorContext.queryOptions.setEnableMemtableOnSinkNode(enableMemTableOnSinkNode); + } + + @Override + public void setBatchSize(int batchSize) { + coordinatorContext.queryOptions.setBatchSize(batchSize); } - private void setFileScanParams(DistributedPlan distributedPlan) { - for (ScanNode scanNode : distributedPlan.getFragmentJob().getScanNodes()) { - if (scanNode instanceof FileQueryScanNode) { - fileScanRangeParamsMap.put( - scanNode.getId().asInt(), - ((FileQueryScanNode) scanNode).getFileScanRangeParams() - ); + @Override + public void setTimeout(int timeout) { + coordinatorContext.queryOptions.setQueryTimeout(timeout); + coordinatorContext.queryOptions.setExecutionTimeout(timeout); + if (coordinatorContext.queryOptions.getExecutionTimeout() < 1) { + LOG.warn("try set timeout less than 1: {}", coordinatorContext.queryOptions.getExecutionTimeout()); + } + } + + @Override + public void setLoadMemLimit(long loadMemLimit) { + coordinatorContext.queryOptions.setLoadMemLimit(loadMemLimit); + } + + @Override + public void setExecMemoryLimit(long execMemoryLimit) { + coordinatorContext.queryOptions.setMemLimit(execMemoryLimit); + } + + // this method is used to provide profile metrics: `Instances Num Per BE` + @Override + public Map getBeToInstancesNum() { + Map result = Maps.newLinkedHashMap(); + if (executionTask != null) { + for (MultiFragmentsPipelineTask beTasks : executionTask.getChildrenTasks().values()) { + TNetworkAddress brpcAddress = beTasks.getBackend().getBrpcAddress(); + String brpcAddrString = brpcAddress.hostname.concat(":").concat("" + brpcAddress.port); + result.put(brpcAddrString, beTasks.getChildrenTasks().size()); } } + return result; } - private boolean useLocalShuffle(DistributedPlan distributedPlan) { - List instanceJobs = ((PipelineDistributedPlan) distributedPlan).getInstanceJobs(); - for (AssignedJob instanceJob : instanceJobs) { - if (instanceJob instanceof LocalShuffleAssignedJob) { - return true; + @Override + public void close() { + // NOTE: all close method should be no exception + if (coordinatorContext.getQueryQueue().isPresent() && coordinatorContext.getQueueToken().isPresent()) { + try { + coordinatorContext.getQueryQueue().get().releaseAndNotify(coordinatorContext.getQueueToken().get()); + } catch (Throwable t) { + LOG.error("error happens when coordinator close ", t); } } - return false; + + try { + for (ScanNode scanNode : coordinatorContext.scanNodes) { + scanNode.stop(); + } + } catch (Throwable t) { + LOG.error("error happens when scannode stop ", t); + } } - private void setForDefaultScanSource( - FInstanceExecParam instanceExecParam, DefaultScanSource scanSource, boolean isShareScan) { - for (Entry scanNodeIdToReplicaIds : scanSource.scanNodeToScanRanges.entrySet()) { - ScanNode scanNode = scanNodeIdToReplicaIds.getKey(); - ScanRanges scanReplicas = scanNodeIdToReplicaIds.getValue(); - instanceExecParam.perNodeScanRanges.put(scanNode.getId().asInt(), scanReplicas.params); - instanceExecParam.perNodeSharedScans.put(scanNode.getId().asInt(), isShareScan); + protected void cancelInternal(Status cancelReason) { + coordinatorContext.withLock(() -> coordinatorContext.getJobProcessor().cancel(cancelReason)); + } + + protected void processTopSink( + CoordinatorContext coordinatorContext, PipelineDistributedPlan topPlan) throws AnalysisException { + setForArrowFlight(coordinatorContext, topPlan); + setForBroker(coordinatorContext, topPlan); + } + + private void setForArrowFlight(CoordinatorContext coordinatorContext, PipelineDistributedPlan topPlan) { + ConnectContext connectContext = coordinatorContext.connectContext; + DataSink dataSink = coordinatorContext.dataSink; + if (dataSink instanceof ResultSink || dataSink instanceof ResultFileSink) { + if (connectContext != null && !connectContext.isReturnResultFromLocal()) { + Preconditions.checkState(connectContext.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)); + + AssignedJob firstInstance = topPlan.getInstanceJobs().get(0); + BackendWorker worker = (BackendWorker) firstInstance.getAssignedWorker(); + Backend backend = worker.getBackend(); + + connectContext.setFinstId(firstInstance.instanceId()); + if (backend.getArrowFlightSqlPort() < 0) { + throw new IllegalStateException("be arrow_flight_sql_port cannot be empty."); + } + connectContext.setResultFlightServerAddr(backend.getArrowFlightAddress()); + connectContext.setResultInternalServiceAddr(backend.getBrpcAddress()); + connectContext.setResultOutputExprs(topPlan.getFragmentJob().getFragment().getOutputExprs()); + } } } - private void setForBucketScanSource(FInstanceExecParam instanceExecParam, - BucketScanSource bucketScanSource, boolean isShareScan) { - for (Entry> bucketIndexToScanTablets : - bucketScanSource.bucketIndexToScanNodeToTablets.entrySet()) { - Integer bucketIndex = bucketIndexToScanTablets.getKey(); - instanceExecParam.addBucketSeq(bucketIndex); - Map scanNodeToRangeMap = bucketIndexToScanTablets.getValue(); - for (Entry scanNodeToRange : scanNodeToRangeMap.entrySet()) { - ScanNode scanNode = scanNodeToRange.getKey(); - ScanRanges scanRanges = scanNodeToRange.getValue(); - List scanBucketTablets = instanceExecParam.perNodeScanRanges.computeIfAbsent( - scanNode.getId().asInt(), id -> Lists.newArrayList()); - scanBucketTablets.addAll(scanRanges.params); - instanceExecParam.perNodeSharedScans.put(scanNode.getId().asInt(), isShareScan); - - if (scanNode instanceof OlapScanNode) { - OlapScanNode olapScanNode = (OlapScanNode) scanNode; - if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) { - int bucketNum = olapScanNode.getBucketNum(); - fragmentIdToSeqToAddressMap.put(olapScanNode.getFragmentId(), new HashMap<>()); - bucketShuffleJoinController.fragmentIdBucketSeqToScanRangeMap - .put(scanNode.getFragmentId(), new BucketSeqToScanRange()); - bucketShuffleJoinController.fragmentIdToBucketNumMap - .put(scanNode.getFragmentId(), bucketNum); - olapScanNode.getFragment().setBucketNum(bucketNum); + private void setForBroker( + CoordinatorContext coordinatorContext, PipelineDistributedPlan topPlan) throws AnalysisException { + DataSink dataSink = coordinatorContext.dataSink; + if (dataSink instanceof ResultFileSink + && ((ResultFileSink) dataSink).getStorageType() == StorageBackend.StorageType.BROKER) { + // set the broker address for OUTFILE sink + ResultFileSink topResultFileSink = (ResultFileSink) dataSink; + DistributedPlanWorker worker = topPlan.getInstanceJobs().get(0).getAssignedWorker(); + FsBroker broker = Env.getCurrentEnv().getBrokerMgr() + .getBroker(topResultFileSink.getBrokerName(), worker.host()); + topResultFileSink.setBrokerAddr(broker.host, broker.port); + } + } + + private void enqueue(ConnectContext context) throws UserException { + // LoadTask does not have context, not controlled by queue now + if (context != null) { + if (Config.enable_workload_group) { + coordinatorContext.setWorkloadGroups(context.getEnv().getWorkloadGroupMgr().getWorkloadGroup(context)); + if (shouldQueue(context)) { + QueryQueue queryQueue = context.getEnv().getWorkloadGroupMgr().getWorkloadGroupQueryQueue(context); + if (queryQueue == null) { + // This logic is actually useless, because when could not find query queue, it will + // throw exception during workload group manager. + throw new UserException("could not find query queue"); } - } else if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) { - int bucketNum = 1; - fragmentIdToSeqToAddressMap.put(scanNode.getFragmentId(), new HashMap<>()); - bucketShuffleJoinController.fragmentIdBucketSeqToScanRangeMap - .put(scanNode.getFragmentId(), new BucketSeqToScanRange()); - bucketShuffleJoinController.fragmentIdToBucketNumMap - .put(scanNode.getFragmentId(), bucketNum); - scanNode.getFragment().setBucketNum(bucketNum); + QueueToken queueToken = queryQueue.getToken(); + int queryTimeout = coordinatorContext.queryOptions.getExecutionTimeout() * 1000; + queueToken.get(DebugUtil.printId(coordinatorContext.queryId), queryTimeout); + coordinatorContext.setQueueInfo(queryQueue, queueToken); } + } else { + context.setWorkloadGroupName(""); + } + } + } - BucketSeqToScanRange bucketSeqToScanRange = bucketShuffleJoinController - .fragmentIdBucketSeqToScanRangeMap.get(scanNode.getFragmentId()); - - Map> scanNodeIdToReplicas - = bucketSeqToScanRange.computeIfAbsent(bucketIndex, set -> Maps.newLinkedHashMap()); - List tablets = scanNodeIdToReplicas.computeIfAbsent( - scanNode.getId().asInt(), id -> new ArrayList<>()); - tablets.addAll(scanRanges.params); + private boolean shouldQueue(ConnectContext context) { + boolean ret = Config.enable_query_queue && !context.getSessionVariable() + .getBypassWorkloadGroup() && !isQueryCancelled(); + if (!ret) { + return false; + } + // a query with ScanNode need not queue only when all its scan node is SchemaScanNode + for (ScanNode scanNode : coordinatorContext.scanNodes) { + if (!(scanNode instanceof SchemaScanNode)) { + return true; } } + return false; + } + + private JobProcessor buildJobProcessor(CoordinatorContext coordinatorContext) { + DataSink dataSink = coordinatorContext.dataSink; + if ((dataSink instanceof ResultSink || dataSink instanceof ResultFileSink)) { + return QueryProcessor.build(coordinatorContext); + } else { + // insert statement has jobId == -1 + return new LoadProcessor(coordinatorContext, -1L); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java index ffbc9744ca4e78..f1304826e39d2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ResultReceiver.java @@ -53,7 +53,7 @@ public class ResultReceiver { private Types.PUniqueId finstId; private Long backendId; private Thread currentThread; - private Future fetchDataAsyncFuture = null; + private volatile Future fetchDataAsyncFuture = null; private Boolean enableParallelResultSink = false; int maxMsgSizeOfResultReceiver; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5f031843025274..1d7bd2e89d15fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -34,6 +34,8 @@ import org.apache.doris.nereids.metrics.EventSwitchParser; import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.planner.GroupCommitBlockSink; import org.apache.doris.qe.VariableMgr.VarAttr; @@ -264,6 +266,8 @@ public class SessionVariable implements Serializable, Writable { public static final String IGNORE_STORAGE_DATA_DISTRIBUTION = "ignore_storage_data_distribution"; + public static final String USE_SERIAL_EXCHANGE = "use_serial_exchange"; + public static final String ENABLE_PARALLEL_SCAN = "enable_parallel_scan"; // Limit the max count of scanners to prevent generate too many scanners. @@ -468,6 +472,12 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_ORC_LAZY_MAT = "enable_orc_lazy_materialization"; + public static final String ORC_TINY_STRIPE_THRESHOLD_BYTES = "orc_tiny_stripe_threshold_bytes"; + + public static final String ORC_ONCE_MAX_READ_BYTES = "orc_once_max_read_bytes"; + + public static final String ORC_MAX_MERGE_DISTANCE_BYTES = "orc_max_merge_distance_bytes"; + public static final String ENABLE_PARQUET_FILTER_BY_MIN_MAX = "enable_parquet_filter_by_min_max"; public static final String ENABLE_ORC_FILTER_BY_MIN_MAX = "enable_orc_filter_by_min_max"; @@ -1094,6 +1104,10 @@ public enum IgnoreSplitType { varType = VariableAnnotation.EXPERIMENTAL, needForward = true) private boolean ignoreStorageDataDistribution = true; + @VariableMgr.VarAttr(name = USE_SERIAL_EXCHANGE, fuzzy = true, + varType = VariableAnnotation.EXPERIMENTAL, needForward = true) + private boolean useSerialExchange = false; + @VariableMgr.VarAttr( name = ENABLE_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, description = {"是否在pipelineX引擎上开启local shuffle优化", @@ -1393,7 +1407,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { + "right side to do bucket shuffle join" } ) - private boolean enableNereidsDistributePlanner = false; + private boolean enableNereidsDistributePlanner = true; @VariableMgr.VarAttr(name = REWRITE_OR_TO_IN_PREDICATE_THRESHOLD, fuzzy = true) private int rewriteOrToInPredicateThreshold = 2; @@ -1512,7 +1526,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true) public boolean enableNereidsTimeout = true; - @VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true) + @VariableMgr.VarAttr(name = NEREIDS_TIMEOUT_SECOND, needForward = true) public int nereidsTimeoutSecond = 30; @VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG) @@ -1713,6 +1727,46 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { public boolean enableOrcLazyMat = true; + @VariableMgr.VarAttr( + name = ORC_TINY_STRIPE_THRESHOLD_BYTES, + description = {"在orc文件中如果一个stripe的字节大小小于`orc_tiny_stripe_threshold`," + + "我们认为该stripe为 tiny stripe。对于多个连续的tiny stripe我们会进行读取优化，即一次性读多个tiny stripe." + + "如果你不想使用该优化，可以将该值设置为0。默认为 8M。", + "In an orc file, if the byte size of a stripe is less than `orc_tiny_stripe_threshold`," + + "we consider the stripe to be a tiny stripe. For multiple consecutive tiny stripes," + + "we will perform read optimization, that is, read multiple tiny stripes at a time." + + "If you do not want to use this optimization, you can set this value to 0." + + "The default is 8M."}, + needForward = true, + setter = "setOrcTinyStripeThresholdBytes") + public long orcTinyStripeThresholdBytes = 8L * 1024L * 1024L; + + + @VariableMgr.VarAttr( + name = ORC_ONCE_MAX_READ_BYTES, + description = {"在使用tiny stripe读取优化的时候，会对多个tiny stripe合并成一次IO，" + + "该参数用来控制每次IO请求的最大字节大小。你不应该将值设置的小于`orc_tiny_stripe_threshold`。默认为 8M。", + "When using tiny stripe read optimization, multiple tiny stripes will be merged into one IO." + + "This parameter is used to control the maximum byte size of each IO request." + + "You should not set the value less than `orc_tiny_stripe_threshold`." + + "The default is 8M."}, + needForward = true, + setter = "setOrcOnceMaxReadBytes") + public long orcOnceMaxReadBytes = 8L * 1024L * 1024L; + + + @VariableMgr.VarAttr( + name = ORC_MAX_MERGE_DISTANCE_BYTES, + description = {"在使用tiny stripe读取优化的时候，由于tiny stripe并不一定连续。" + + "当两个tiny stripe之间距离大于该参数时，我们不会将其合并成一次IO。默认为 1M。", + "When using tiny stripe read optimization, since tiny stripes are not necessarily continuous," + + "when the distance between two tiny stripes is greater than this parameter," + + "we will not merge them into one IO. The default value is 1M."}, + needForward = true, + setter = "setOrcMaxMergeDistanceBytes") + public long orcMaxMergeDistanceBytes = 1024L * 1024L; + + @VariableMgr.VarAttr( name = ENABLE_PARQUET_FILTER_BY_MIN_MAX, description = {"控制 parquet reader 是否启用 min-max 值过滤。默认为 true。", @@ -2279,6 +2333,7 @@ public void initFuzzyModeVariables() { this.parallelPrepareThreshold = random.nextInt(32) + 1; this.enableCommonExprPushdown = random.nextBoolean(); this.enableLocalExchange = random.nextBoolean(); + this.useSerialExchange = random.nextBoolean(); // This will cause be dead loop, disable it first // this.disableJoinReorder = random.nextBoolean(); this.enableCommonExpPushDownForInvertedIndex = random.nextBoolean(); @@ -2792,6 +2847,32 @@ public void setFragmentInstanceNum(String value) throws Exception { this.parallelExecInstanceNum = val; } + public void setOrcTinyStripeThresholdBytes(String value) throws Exception { + long val = checkFieldLongValue(ORC_TINY_STRIPE_THRESHOLD_BYTES, 0, value); + this.orcTinyStripeThresholdBytes = val; + } + + public void setOrcOnceMaxReadBytes(String value) throws Exception { + long val = checkFieldLongValue(ORC_ONCE_MAX_READ_BYTES, 0, value); + this.orcOnceMaxReadBytes = val; + } + + public void setOrcMaxMergeDistanceBytes(String value) throws Exception { + long val = checkFieldLongValue(ORC_MAX_MERGE_DISTANCE_BYTES, 0, value); + this.orcMaxMergeDistanceBytes = val; + } + + private long checkFieldLongValue(String variableName, long minValue, String value) throws Exception { + long val = Long.parseLong(value); + if (val < minValue) { + throw new Exception( + variableName + " value should greater than or equal " + String.valueOf(minValue) + + ", you set value is: " + value); + } + return val; + } + + private int checkFieldValue(String variableName, int minValue, String value) throws Exception { int val = Integer.valueOf(value); if (val < minValue) { @@ -3404,11 +3485,11 @@ public static boolean canUseNereidsDistributePlanner() { } ConnectContext connectContext = ConnectContext.get(); if (connectContext == null) { - return false; + return true; } StatementContext statementContext = connectContext.getStatementContext(); if (statementContext == null) { - return false; + return true; } StatementBase parsedStatement = statementContext.getParsedStatement(); if (!(parsedStatement instanceof LogicalPlanAdapter)) { @@ -3417,7 +3498,9 @@ public static boolean canUseNereidsDistributePlanner() { LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStatement).getLogicalPlan(); SessionVariable sessionVariable = connectContext.getSessionVariable(); // TODO: support other sink - if (logicalPlan instanceof UnboundResultSink && sessionVariable.enableNereidsDistributePlanner) { + if ((logicalPlan instanceof UnboundResultSink + || logicalPlan instanceof LogicalFileSink + || logicalPlan instanceof InsertIntoTableCommand) && sessionVariable.enableNereidsDistributePlanner) { return true; } return false; @@ -3860,6 +3943,11 @@ public TQueryOptions toThrift() { tResult.setInListValueCountThreshold(inListValueCountThreshold); tResult.setEnablePhraseQuerySequentialOpt(enablePhraseQuerySequentialOpt); tResult.setEnableAutoCreateWhenOverwrite(enableAutoCreateWhenOverwrite); + + tResult.setOrcTinyStripeThresholdBytes(orcTinyStripeThresholdBytes); + tResult.setOrcMaxMergeDistanceBytes(orcMaxMergeDistanceBytes); + tResult.setOrcOnceMaxReadBytes(orcOnceMaxReadBytes); + return tResult; } @@ -3921,7 +4009,6 @@ public void readFromJson(String json) throws IOException { if (attr == null) { continue; } - if (!root.containsKey(attr.name())) { continue; } @@ -4459,4 +4546,8 @@ public TSerdeDialect getSerdeDialect() { public boolean isEnableCooldownReplicaAffinity() { return enableCooldownReplicaAffinity; } + + public boolean isUseSerialExchange() { + return useSerialExchange && getEnableLocalExchange(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index f4c33d75cdcc51..9fa50afa702b00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -338,6 +338,10 @@ public StmtExecutor(ConnectContext ctx, StatementBase parsedStmt) { context.getSessionVariable().getAutoProfileThresholdMs()); } + public boolean isProxy() { + return isProxy; + } + public static InternalService.PDataRow getRowStringValue(List cols, FormatOptions options) throws UserException { if (cols.isEmpty()) { @@ -391,7 +395,7 @@ private Map getSummaryInfo(boolean isFinished) { String taskState = "RUNNING"; if (isFinished) { if (coord != null) { - taskState = coord.queryStatus.getErrorCode().name(); + taskState = coord.getExecStatus().getErrorCode().name(); } else { taskState = context.getState().toString(); } @@ -746,6 +750,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception { // t3: observer fe receive editlog creating the table from the master fe syncJournalIfNeeded(); try { + ((Command) logicalPlan).verifyCommandSupported(); ((Command) logicalPlan).run(context, this); } catch (MustFallbackException e) { if (LOG.isDebugEnabled()) { @@ -1915,8 +1920,7 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, context.getState().setIsQuery(true); } else if (planner instanceof NereidsPlanner && ((NereidsPlanner) planner).getDistributedPlans() != null) { coord = new NereidsCoordinator(context, analyzer, - planner, context.getStatsErrorEstimator(), - (NereidsPlanner) planner); + (NereidsPlanner) planner, context.getStatsErrorEstimator()); profile.addExecutionProfile(coord.getExecutionProfile()); QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QueryInfo(context, originStmt.originStmt, coord)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index a69ca0776e24c2..b7a4f8bb710074 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -469,6 +469,11 @@ public static void replayGlobalVariableV2(GlobalVarPersistInfo info) throws DdlE try { String json = info.getPersistJsonString(); JSONObject root = (JSONObject) JSONValue.parse(json); + // if not variable version, we set it to 0 to ensure we could force set global variable. + boolean hasVariableVersion = root.containsKey(GlobalVariable.VARIABLE_VERSION); + if (!hasVariableVersion) { + GlobalVariable.variableVersion = GlobalVariable.VARIABLE_VERSION_0; + } for (Object varName : root.keySet()) { VarContext varContext = ctxByVarName.get((String) varName); if (Env.isCheckpointThread()) { @@ -728,11 +733,15 @@ public static List> dump(SetType type, SessionVariable sessionVar, rlock.lock(); try { for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { - // not show removed variables VarAttr varAttr = entry.getValue().getField().getAnnotation(VarAttr.class); + // not show removed variables if (VariableAnnotation.REMOVED.equals(varAttr.varType())) { continue; } + // not show invisible variables + if ((VariableMgr.INVISIBLE & varAttr.flag()) != 0) { + continue; + } // Filter variable not match to the regex. if (matcher != null && !matcher.match(entry.getKey())) { continue; @@ -947,4 +956,48 @@ private static ImmutableSortedMap.Builder getStringVarContex } return builder; } + + public static void forceUpdateVariables() { + int currentVariableVersion = GlobalVariable.variableVersion; + if (currentVariableVersion == GlobalVariable.VARIABLE_VERSION_0) { + // update from 2.0.15 or below to 2.0.16 or higher + if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) { + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.NEREIDS_TIMEOUT_SECOND, "30"); + } + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_100) { + // update from 2.1.6 or below to 2.1.7 or higher + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_DML, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_DML_WITH_PIPELINE, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_NEREIDS_PLANNER, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, + String.valueOf(true)); + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_PIPELINE_X_ENGINE, + String.valueOf(true)); + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) { + // update from 3.0.2 or below to 3.0.3 or higher + VariableMgr.refreshDefaultSessionVariables("update variable version", + SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, + String.valueOf(false)); + } + if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_300) { + // update to master + // do nothing + } + if (currentVariableVersion < GlobalVariable.CURRENT_VARIABLE_VERSION) { + VariableMgr.refreshDefaultSessionVariables("update variable version", + GlobalVariable.VARIABLE_VERSION, + String.valueOf(GlobalVariable.CURRENT_VARIABLE_VERSION)); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/protocol/TFastSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/protocol/TFastSerializer.java new file mode 100644 index 00000000000000..7a317d23472ab7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/protocol/TFastSerializer.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.protocol; + +import com.google.protobuf.ByteString; +import com.google.protobuf.UnsafeByteOperations; +import org.apache.thrift.TBase; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransportException; + +import java.io.ByteArrayOutputStream; + +/** + * Copy from TSerializer and support change the initial byte array capacity + */ +public class TFastSerializer { + + /** + * This is the byte array that data is actually serialized into + */ + // private final NoResizeByteOutputStream baos; + private final ByteArrayOutputStream baos; + + /** + * This transport wraps that byte array + */ + private final TIOStreamTransport transport; + + /** + * Internal protocol used for serializing objects. + */ + private TProtocol protocol; + + /** + * Create a new TSerializer. It will use the TProtocol specified by the + * factory that is passed in. + * + * @param initCapacity init capacity to the write buffer + * @param protocolFactory Factory to create a protocol + * @throws TTransportException if there an error initializing the underlying transport. + */ + public TFastSerializer(int initCapacity, TProtocolFactory protocolFactory) throws TTransportException { + baos = new ByteArrayOutputStream(initCapacity); + transport = new TIOStreamTransport(new TConfiguration(), baos); + protocol = protocolFactory.getProtocol(transport); + } + + /** + * Serialize the Thrift object into a byte array. The process is simple, + * just clear the byte array output, write the object into it, and grab the + * raw bytes. + * + * @param base The object to serialize + * @return Serialized object in byte[] format + * @throws TException if an error is encountered during serialization. + */ + public ByteString serialize(TBase base) throws TException { + baos.reset(); + base.write(protocol); + return UnsafeByteOperations.unsafeWrap(baos.toByteArray()); + } + + /** + * Serialize the Thrift object into a Java string, using the default JVM + * charset encoding. + * + * @param base The object to serialize + * @return Serialized object as a String + * @throws TException if an error is encountered during serialization. + */ + public String toString(TBase base) throws TException { + return new String(serialize(base).toByteArray()); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/AbstractRuntimeTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/AbstractRuntimeTask.java new file mode 100644 index 00000000000000..1059440792e529 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/AbstractRuntimeTask.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import java.util.Map; +import java.util.Objects; + +public abstract class AbstractRuntimeTask> { + protected final ChildrenRuntimeTasks childrenTasks; + + public AbstractRuntimeTask(ChildrenRuntimeTasks childrenTasks) { + this.childrenTasks = Objects.requireNonNull(childrenTasks, "childrenTasks can not be null"); + } + + public void execute() throws Throwable { + for (Child childrenTask : childrenTasks.allTasks()) { + childrenTask.execute(); + } + } + + public Map getChildrenTasks() { + return childrenTasks.allTaskMap(); + } + + protected Child childTask(ChildId childId) { + return childrenTasks.get(childId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/BackendFragmentId.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/BackendFragmentId.java new file mode 100644 index 00000000000000..26166c54c83c60 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/BackendFragmentId.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class BackendFragmentId { + public final long backendId; + public final int fragmentId; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ChildrenRuntimeTasks.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ChildrenRuntimeTasks.java new file mode 100644 index 00000000000000..b6ac80d1066a0b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ChildrenRuntimeTasks.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.nereids.util.Utils; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +public class ChildrenRuntimeTasks> { + // LinkedHashMap: make sure the key set order is same as the input map + // so that we can initial the runtime filter merge backend first + private final Map childrenTasks = Maps.newLinkedHashMap(); + + public ChildrenRuntimeTasks(Map childrenTasks) { + this.childrenTasks.putAll(childrenTasks); + } + + public C get(Id id) { + return childrenTasks.get(id); + } + + public List allTasks() { + return Utils.fastToImmutableList(childrenTasks.values()); + } + + public Map allTaskMap() { + return ImmutableMap.copyOf(childrenTasks); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LeafRuntimeTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LeafRuntimeTask.java new file mode 100644 index 00000000000000..7a9e523b05eeaa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LeafRuntimeTask.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import com.google.common.collect.ImmutableMap; + +public class LeafRuntimeTask extends AbstractRuntimeTask { + public LeafRuntimeTask() { + super(new ChildrenRuntimeTasks<>(ImmutableMap.of())); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java new file mode 100644 index 00000000000000..3a448521fca0bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.Status; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.datasource.hive.HMSTransaction; +import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.JobProcessor; +import org.apache.doris.qe.LoadContext; +import org.apache.doris.thrift.TFragmentInstanceReport; +import org.apache.doris.thrift.TReportExecStatusParams; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +public class LoadProcessor implements JobProcessor { + private static final Logger LOG = LogManager.getLogger(LoadProcessor.class); + + public final CoordinatorContext coordinatorContext; + public final LoadContext loadContext; + public final long jobId; + + // this latch is used to wait finish for load, for example, insert into statement + // MarkedCountDownLatch: + // key: fragmentId, value: backendId + private volatile Optional executionTask; + private volatile Optional> latch; + private volatile Optional> backendFragmentTasks; + private volatile List topFragmentTasks; + + public LoadProcessor(CoordinatorContext coordinatorContext, long jobId) { + this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null"); + this.loadContext = new LoadContext(); + this.executionTask = Optional.empty(); + this.latch = Optional.empty(); + this.backendFragmentTasks = Optional.empty(); + + // only we set is report success, then the backend would report the fragment status, + // then we can not the fragment is finished, and we can return in the NereidsCoordinator::join + coordinatorContext.queryOptions.setIsReportSuccess(true); + // the insert into statement isn't a job + this.jobId = jobId; + TUniqueId queryId = coordinatorContext.queryId; + Env.getCurrentEnv().getLoadManager().initJobProgress( + jobId, queryId, coordinatorContext.instanceIds.get(), + Utils.fastToImmutableList(coordinatorContext.backends.get().values()) + ); + Env.getCurrentEnv().getProgressManager().addTotalScanNums( + String.valueOf(jobId), coordinatorContext.scanRangeNum.get() + ); + + topFragmentTasks = Lists.newArrayList(); + + LOG.info("dispatch load job: {} to {}", DebugUtil.printId(queryId), coordinatorContext.backends.get().keySet()); + } + + @Override + public void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask) { + Preconditions.checkArgument(pipelineExecutionTask != null, "sqlPipelineTask can not be null"); + + this.executionTask = Optional.of(pipelineExecutionTask); + Map backendFragmentTasks + = buildBackendFragmentTasks(pipelineExecutionTask); + this.backendFragmentTasks = Optional.of(backendFragmentTasks); + + MarkedCountDownLatch latch = new MarkedCountDownLatch<>(backendFragmentTasks.size()); + for (BackendFragmentId backendFragmentId : backendFragmentTasks.keySet()) { + latch.addMark(backendFragmentId.fragmentId, backendFragmentId.backendId); + } + this.latch = Optional.of(latch); + + int topFragmentId = coordinatorContext.topDistributedPlan + .getFragmentJob() + .getFragment() + .getFragmentId() + .asInt(); + List topFragmentTasks = Lists.newArrayList(); + for (MultiFragmentsPipelineTask multiFragmentPipelineTask : pipelineExecutionTask.childrenTasks.allTasks()) { + for (SingleFragmentPipelineTask fragmentTask : multiFragmentPipelineTask.childrenTasks.allTasks()) { + if (fragmentTask.getFragmentId() == topFragmentId) { + topFragmentTasks.add(fragmentTask); + } + } + } + this.topFragmentTasks = topFragmentTasks; + } + + @Override + public void cancel(Status cancelReason) { + if (executionTask.isPresent()) { + for (MultiFragmentsPipelineTask fragmentsTask : executionTask.get().getChildrenTasks().values()) { + fragmentsTask.cancelExecute(cancelReason); + } + latch.get().countDownToZero(new Status()); + } + } + + public boolean isDone() { + return latch.map(l -> l.getCount() == 0).orElse(false); + } + + public boolean join(int timeoutS) { + PipelineExecutionTask pipelineExecutionTask = this.executionTask.orElse(null); + if (pipelineExecutionTask == null) { + return true; + } + + long fixedMaxWaitTime = 30; + + long leftTimeoutS = timeoutS; + while (leftTimeoutS > 0) { + long waitTime = Math.min(leftTimeoutS, fixedMaxWaitTime); + boolean awaitRes = false; + try { + awaitRes = await(waitTime, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Do nothing + } + if (awaitRes) { + return true; + } + + if (!checkHealthy()) { + return true; + } + + leftTimeoutS -= waitTime; + } + return false; + } + + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + if (!latch.isPresent()) { + return false; + } + return latch.get().await(timeout, unit); + } + + public void updateFragmentExecStatus(TReportExecStatusParams params) { + SingleFragmentPipelineTask fragmentTask = backendFragmentTasks.get().get( + new BackendFragmentId(params.getBackendId(), params.getFragmentId())); + if (fragmentTask == null || !fragmentTask.processReportExecStatus(params)) { + return; + } + TUniqueId queryId = coordinatorContext.queryId; + Status status = new Status(params.status); + // for now, abort the query if we see any error except if the error is cancelled + // and returned_all_results_ is true. + // (UpdateStatus() initiates cancellation, if it hasn't already been initiated) + if (!status.ok()) { + if (coordinatorContext.isEos() && status.isCancelled()) { + LOG.warn("Query {} has returned all results, fragment_id={} instance_id={}, be={}" + + " is reporting failed status {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), + status.toString()); + } else { + LOG.warn("one instance report fail, query_id={} fragment_id={} instance_id={}, be={}," + + " error message: {}", + DebugUtil.printId(queryId), params.getFragmentId(), + DebugUtil.printId(params.getFragmentInstanceId()), + params.getBackendId(), status.toString()); + coordinatorContext.updateStatusIfOk(status); + } + } + LoadContext loadContext = coordinatorContext.asLoadProcessor().loadContext; + if (params.isSetDeltaUrls()) { + loadContext.updateDeltaUrls(params.getDeltaUrls()); + } + if (params.isSetLoadCounters()) { + loadContext.updateLoadCounters(params.getLoadCounters()); + } + if (params.isSetTrackingUrl()) { + loadContext.updateTrackingUrl(params.getTrackingUrl()); + } + if (params.isSetTxnId()) { + loadContext.updateTransactionId(params.getTxnId()); + } + if (params.isSetLabel()) { + loadContext.updateLabel(params.getLabel()); + } + if (params.isSetExportFiles()) { + loadContext.addExportFiles(params.getExportFiles()); + } + if (params.isSetCommitInfos()) { + loadContext.updateCommitInfos(params.getCommitInfos()); + } + if (params.isSetErrorTabletInfos()) { + loadContext.updateErrorTabletInfos(params.getErrorTabletInfos()); + } + long txnId = loadContext.getTransactionId(); + if (params.isSetHivePartitionUpdates()) { + ((HMSTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) + .updateHivePartitionUpdates(params.getHivePartitionUpdates()); + } + if (params.isSetIcebergCommitDatas()) { + ((IcebergTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) + .updateIcebergCommitData(params.getIcebergCommitDatas()); + } + + if (fragmentTask.isDone()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Query {} fragment {} is marked done", + DebugUtil.printId(queryId), params.getFragmentId()); + } + latch.get().markedCountDown(params.getFragmentId(), params.getBackendId()); + } + + if (params.isSetLoadedRows() && jobId != -1) { + if (params.isSetFragmentInstanceReports()) { + for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) { + Env.getCurrentEnv().getLoadManager().updateJobProgress( + jobId, params.getBackendId(), params.getQueryId(), report.getFragmentInstanceId(), + report.getLoadedRows(), report.getLoadedBytes(), params.isDone()); + Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), + params.getQueryId(), report.getFragmentInstanceId(), report.getNumFinishedRange()); + } + } else { + Env.getCurrentEnv().getLoadManager().updateJobProgress( + jobId, params.getBackendId(), params.getQueryId(), params.getFragmentInstanceId(), + params.getLoadedRows(), params.getLoadedBytes(), params.isDone()); + Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), + params.getQueryId(), params.getFragmentInstanceId(), params.getFinishedScanRanges()); + } + } + } + + private Map buildBackendFragmentTasks( + PipelineExecutionTask executionTask) { + ImmutableMap.Builder backendFragmentTasks + = ImmutableMap.builder(); + for (Entry backendTask : executionTask.getChildrenTasks().entrySet()) { + Long backendId = backendTask.getKey(); + for (Entry fragmentIdToTask : backendTask.getValue() + .getChildrenTasks().entrySet()) { + Integer fragmentId = fragmentIdToTask.getKey(); + SingleFragmentPipelineTask fragmentTask = fragmentIdToTask.getValue(); + backendFragmentTasks.put(new BackendFragmentId(backendId, fragmentId), fragmentTask); + } + } + return backendFragmentTasks.build(); + } + + /* + * Check the state of backends in needCheckBackendExecStates. + * return true if all of them are OK. Otherwise, return false. + */ + private boolean checkHealthy() { + for (SingleFragmentPipelineTask topFragmentTask : topFragmentTasks) { + if (!topFragmentTask.isBackendHealthy(jobId)) { + long backendId = topFragmentTask.getBackend().getId(); + Status unhealthyStatus = new Status( + TStatusCode.INTERNAL_ERROR, "backend " + backendId + " is down"); + coordinatorContext.updateStatusIfOk(unhealthyStatus); + return false; + } + } + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java new file mode 100644 index 00000000000000..a1a91aab5176b8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/MultiFragmentsPipelineTask.java @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.common.Status; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.InternalService.PCancelPlanFragmentResult; +import org.apache.doris.proto.InternalService.PExecPlanFragmentResult; +import org.apache.doris.proto.InternalService.PExecPlanFragmentStartRequest; +import org.apache.doris.proto.Types; +import org.apache.doris.proto.Types.PUniqueId; +import org.apache.doris.qe.Coordinator; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.SimpleScheduler; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.protobuf.ByteString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class MultiFragmentsPipelineTask extends AbstractRuntimeTask { + private static final Logger LOG = LogManager.getLogger(PipelineExecutionTask.class); + + // immutable parameters + private final CoordinatorContext coordinatorContext; + private final Backend backend; + private final BackendServiceProxy backendClientProxy; + + // mutable states + + // we will set fragmentsParams and serializeFragments to null after send rpc, to save memory + private ByteString serializeFragments; + private final AtomicBoolean hasCancelled; + private final AtomicBoolean cancelInProcess; + + public MultiFragmentsPipelineTask( + CoordinatorContext coordinatorContext, Backend backend, BackendServiceProxy backendClientProxy, + ByteString serializeFragments, + Map fragmentTasks) { + super(new ChildrenRuntimeTasks<>(fragmentTasks)); + this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null"); + this.backend = Objects.requireNonNull(backend, "backend can not be null"); + this.backendClientProxy = Objects.requireNonNull(backendClientProxy, "backendClientProxy can not be null"); + this.serializeFragments = Objects.requireNonNull( + serializeFragments, "serializeFragments can not be null" + ); + this.hasCancelled = new AtomicBoolean(); + this.cancelInProcess = new AtomicBoolean(); + } + + public Future sendPhaseOneRpc(boolean twoPhaseExecution) { + return execRemoteFragmentsAsync( + backendClientProxy, serializeFragments, backend.getBrpcAddress(), twoPhaseExecution + ); + } + + public Future sendPhaseTwoRpc() { + return execPlanFragmentStartAsync(backendClientProxy, backend.getBrpcAddress()); + } + + @Override + public String toString() { + TNetworkAddress brpcAddress = backend.getBrpcAddress(); + return "MultiFragmentsPipelineTask(Backend " + backend.getId() + + "(" + brpcAddress.getHostname() + ":" + brpcAddress.getPort() + "): [" + + childrenTasks.allTasks() + .stream() + .map(singleFragment -> "F" + singleFragment.getFragmentId()) + .collect(Collectors.joining(", ")) + "])"; + } + + public synchronized void cancelExecute(Status cancelReason) { + TUniqueId queryId = coordinatorContext.queryId; + if (LOG.isDebugEnabled()) { + LOG.debug("cancelRemoteFragments backend: {}, query={}, reason: {}", + backend, DebugUtil.printId(queryId), cancelReason.toString()); + } + + if (this.hasCancelled.get() || this.cancelInProcess.get()) { + LOG.info("Frangment has already been cancelled. Query {} backend: {}", + DebugUtil.printId(queryId), backend); + return; + } + try { + TNetworkAddress brpcAddress = backend.getBrpcAddress(); + try { + ListenableFuture cancelResult = + BackendServiceProxy.getInstance().cancelPipelineXPlanFragmentAsync( + brpcAddress, queryId, cancelReason); + Futures.addCallback(cancelResult, new FutureCallback() { + public void onSuccess(InternalService.PCancelPlanFragmentResult result) { + cancelInProcess.set(false); + if (result.hasStatus()) { + Status status = new Status(result.getStatus()); + if (status.getErrorCode() == TStatusCode.OK) { + hasCancelled.set(true); + } else { + LOG.warn("Failed to cancel query {} backend: {}, reason: {}", + DebugUtil.printId(queryId), backend, status.toString()); + } + } + LOG.warn("Failed to cancel query {} backend: {} reason: {}", + DebugUtil.printId(queryId), backend, "without status"); + } + + public void onFailure(Throwable t) { + cancelInProcess.set(false); + LOG.warn("Failed to cancel query {} backend: {}, reason: {}", + DebugUtil.printId(queryId), backend, cancelReason.toString(), t); + } + }, Coordinator.backendRpcCallbackExecutor); + cancelInProcess.set(true); + } catch (RpcException e) { + LOG.warn("cancel plan fragment get a exception, address={}:{}", brpcAddress.getHostname(), + brpcAddress.getPort()); + SimpleScheduler.addToBlacklist(backend.getId(), e.getMessage()); + } + } catch (Exception e) { + LOG.warn("catch a exception", e); + } + } + + public Backend getBackend() { + return backend; + } + + private Future execRemoteFragmentsAsync( + BackendServiceProxy proxy, ByteString serializedFragments, TNetworkAddress brpcAddr, + boolean twoPhaseExecution) { + Preconditions.checkNotNull(serializedFragments); + try { + return proxy.execPlanFragmentsAsync(brpcAddr, serializedFragments, twoPhaseExecution); + } catch (RpcException e) { + // DO NOT throw exception here, return a complete future with error code, + // so that the following logic will cancel the fragment. + return futureWithException(e); + } finally { + // save memory + this.serializeFragments = null; + } + } + + public Future execPlanFragmentStartAsync( + BackendServiceProxy proxy, TNetworkAddress brpcAddr) { + TUniqueId queryId = coordinatorContext.queryId; + try { + PExecPlanFragmentStartRequest.Builder builder = PExecPlanFragmentStartRequest.newBuilder(); + PUniqueId qid = PUniqueId.newBuilder().setHi(queryId.hi).setLo(queryId.lo).build(); + builder.setQueryId(qid); + return proxy.execPlanFragmentStartAsync(brpcAddr, builder.build()); + } catch (RpcException e) { + // DO NOT throw exception here, return a complete future with error code, + // so that the following logic will cancel the fragment. + return futureWithException(e); + } + } + + private Future futureWithException(RpcException e) { + return new Future() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public PExecPlanFragmentResult get() { + PExecPlanFragmentResult result = PExecPlanFragmentResult.newBuilder().setStatus( + Types.PStatus.newBuilder().addErrorMsgs(e.getMessage()) + .setStatusCode(TStatusCode.THRIFT_RPC_ERROR.getValue()).build()).build(); + return result; + } + + @Override + public PExecPlanFragmentResult get(long timeout, TimeUnit unit) { + return get(); + } + }; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java new file mode 100644 index 00000000000000..1a8f2a216cd713 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTask.java @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.common.Config; +import org.apache.doris.common.Status; +import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.proto.InternalService.PExecPlanFragmentResult; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.SimpleScheduler; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.joda.time.DateTime; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +/** + * SqlPipelineTask. + * + * This class is used to describe which backend process which fragments + */ +public class PipelineExecutionTask extends AbstractRuntimeTask { + private static final Logger LOG = LogManager.getLogger(PipelineExecutionTask.class); + + // immutable parameters + private final long timeoutDeadline; + private final CoordinatorContext coordinatorContext; + private final BackendServiceProxy backendServiceProxy; + private final Map backendFragmentTasks; + + // mutable states + public PipelineExecutionTask( + CoordinatorContext coordinatorContext, + BackendServiceProxy backendServiceProxy, + Map fragmentTasks) { + // insert into stmt need latch to wait finish, but query stmt not need because result receiver can wait finish + super(new ChildrenRuntimeTasks<>(fragmentTasks)); + this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null"); + this.backendServiceProxy = Objects.requireNonNull(backendServiceProxy, "backendServiceProxy can not be null"); + this.timeoutDeadline = coordinatorContext.timeoutDeadline.get(); + + // flatten to fragment tasks to quickly index by BackendFragmentId, when receive the report message + ImmutableMap.Builder backendFragmentTasks + = ImmutableMap.builder(); + for (Entry backendTask : fragmentTasks.entrySet()) { + Long backendId = backendTask.getKey(); + for (Entry fragmentIdToTask : backendTask.getValue() + .getChildrenTasks().entrySet()) { + Integer fragmentId = fragmentIdToTask.getKey(); + SingleFragmentPipelineTask fragmentTask = fragmentIdToTask.getValue(); + backendFragmentTasks.put(new BackendFragmentId(backendId, fragmentId), fragmentTask); + } + } + this.backendFragmentTasks = backendFragmentTasks.build(); + } + + @Override + public void execute() throws Exception { + coordinatorContext.withLock(() -> { + sendAndWaitPhaseOneRpc(); + if (coordinatorContext.twoPhaseExecution()) { + sendAndWaitPhaseTwoRpc(); + } + return null; + }); + } + + @Override + public String toString() { + return "SqlPipelineTask(\n" + + childrenTasks.allTasks() + .stream() + .map(multiFragmentsPipelineTask -> " " + multiFragmentsPipelineTask) + .collect(Collectors.joining(",\n")) + + "\n)"; + } + + private void sendAndWaitPhaseOneRpc() throws UserException, RpcException { + List rpcs = Lists.newArrayList(); + for (MultiFragmentsPipelineTask fragmentsTask : childrenTasks.allTasks()) { + rpcs.add(new RpcInfo( + fragmentsTask, + DateTime.now().getMillis(), + fragmentsTask.sendPhaseOneRpc(coordinatorContext.twoPhaseExecution())) + ); + } + Map> rpcPhase1Latency = waitPipelineRpc(rpcs, + timeoutDeadline - System.currentTimeMillis(), "send fragments"); + + coordinatorContext.updateProfileIfPresent(profile -> profile.updateFragmentRpcCount(rpcs.size())); + coordinatorContext.updateProfileIfPresent(SummaryProfile::setFragmentSendPhase1Time); + coordinatorContext.updateProfileIfPresent(profile -> profile.setRpcPhase1Latency(rpcPhase1Latency)); + } + + private void sendAndWaitPhaseTwoRpc() throws RpcException, UserException { + List rpcs = Lists.newArrayList(); + for (MultiFragmentsPipelineTask fragmentTask : childrenTasks.allTasks()) { + rpcs.add(new RpcInfo( + fragmentTask, + DateTime.now().getMillis(), + fragmentTask.sendPhaseTwoRpc()) + ); + } + + Map> rpcPhase2Latency = waitPipelineRpc(rpcs, + timeoutDeadline - System.currentTimeMillis(), "send execution start"); + coordinatorContext.updateProfileIfPresent(profile -> profile.updateFragmentRpcCount(rpcs.size())); + coordinatorContext.updateProfileIfPresent(SummaryProfile::setFragmentSendPhase2Time); + coordinatorContext.updateProfileIfPresent(profile -> profile.setRpcPhase2Latency(rpcPhase2Latency)); + } + + private Map> waitPipelineRpc( + List rpcs, + long leftTimeMs, String operation) throws RpcException, UserException { + TQueryOptions queryOptions = coordinatorContext.queryOptions; + TUniqueId queryId = coordinatorContext.queryId; + + if (leftTimeMs <= 0) { + long currentTimeMillis = System.currentTimeMillis(); + long elapsed = (currentTimeMillis - timeoutDeadline) / 1000 + queryOptions.getExecutionTimeout(); + String msg = String.format( + "timeout before waiting %s rpc, query timeout:%d, already elapsed:%d, left for this:%d", + operation, queryOptions.getExecutionTimeout(), elapsed, leftTimeMs); + LOG.warn("Query {} {}", DebugUtil.printId(queryId), msg); + if (!queryOptions.isSetExecutionTimeout() || !queryOptions.isSetQueryTimeout()) { + LOG.warn("Query {} does not set timeout info, execution timeout: is_set:{}, value:{}" + + ", query timeout: is_set:{}, value: {}, " + + "coordinator timeout deadline {}, cur time millis: {}", + DebugUtil.printId(queryId), + queryOptions.isSetExecutionTimeout(), queryOptions.getExecutionTimeout(), + queryOptions.isSetQueryTimeout(), queryOptions.getQueryTimeout(), + timeoutDeadline, currentTimeMillis); + } + throw new UserException(msg); + } + + // BE -> (RPC latency from FE to BE, Execution latency on bthread, Duration of doing work, RPC latency from BE + // to FE) + Map> beToPrepareLatency = new HashMap<>(); + long timeoutMs = Math.min(leftTimeMs, Config.remote_fragment_exec_timeout_ms); + for (RpcInfo rpc : rpcs) { + TStatusCode code; + String errMsg = null; + Exception exception = null; + + Backend backend = rpc.task.getBackend(); + long beId = backend.getId(); + TNetworkAddress brpcAddress = backend.getBrpcAddress(); + + try { + PExecPlanFragmentResult result = rpc.future.get(timeoutMs, TimeUnit.MILLISECONDS); + long rpcDone = DateTime.now().getMillis(); + beToPrepareLatency.put(brpcAddress, + Lists.newArrayList(result.getReceivedTime() - rpc.startTime, + result.getExecutionTime() - result.getReceivedTime(), + result.getExecutionDoneTime() - result.getExecutionTime(), + rpcDone - result.getExecutionDoneTime())); + code = TStatusCode.findByValue(result.getStatus().getStatusCode()); + if (code == null) { + code = TStatusCode.INTERNAL_ERROR; + } + + if (code != TStatusCode.OK) { + if (!result.getStatus().getErrorMsgsList().isEmpty()) { + errMsg = result.getStatus().getErrorMsgsList().get(0); + } else { + errMsg = operation + " failed. backend id: " + beId; + } + } + } catch (ExecutionException e) { + exception = e; + code = TStatusCode.THRIFT_RPC_ERROR; + backendServiceProxy.removeProxy(brpcAddress); + } catch (InterruptedException e) { + exception = e; + code = TStatusCode.INTERNAL_ERROR; + backendServiceProxy.removeProxy(brpcAddress); + } catch (TimeoutException e) { + exception = e; + errMsg = String.format( + "timeout when waiting for %s rpc, query timeout:%d, left timeout for this operation:%d", + operation, queryOptions.getExecutionTimeout(), timeoutMs / 1000); + LOG.warn("Query {} {}", DebugUtil.printId(queryId), errMsg); + code = TStatusCode.TIMEOUT; + backendServiceProxy.removeProxy(brpcAddress); + } + + if (code != TStatusCode.OK) { + if (exception != null && errMsg == null) { + errMsg = operation + " failed. " + exception.getMessage(); + } + Status cancelStatus = new Status(TStatusCode.INTERNAL_ERROR, errMsg); + coordinatorContext.updateStatusIfOk(cancelStatus); + coordinatorContext.cancelSchedule(cancelStatus); + switch (code) { + case TIMEOUT: + MetricRepo.BE_COUNTER_QUERY_RPC_FAILED.getOrAdd(brpcAddress.hostname) + .increase(1L); + throw new RpcException(brpcAddress.hostname, errMsg, exception); + case THRIFT_RPC_ERROR: + MetricRepo.BE_COUNTER_QUERY_RPC_FAILED.getOrAdd(brpcAddress.hostname) + .increase(1L); + SimpleScheduler.addToBlacklist(beId, errMsg); + throw new RpcException(brpcAddress.hostname, errMsg, exception); + default: + throw new UserException(errMsg, exception); + } + } + } + return beToPrepareLatency; + } + + private static class RpcInfo { + public final MultiFragmentsPipelineTask task; + public final long startTime; + public final Future future; + + public RpcInfo(MultiFragmentsPipelineTask task, long startTime, Future future) { + this.task = task; + this.startTime = startTime; + this.future = future; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java new file mode 100644 index 00000000000000..fd00bf0e3e8536 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/PipelineExecutionTaskBuilder.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.protocol.TFastSerializer; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TPipelineFragmentParams; +import org.apache.doris.thrift.TPipelineFragmentParamsList; +import org.apache.doris.thrift.TPipelineInstanceParams; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import org.apache.thrift.protocol.TCompactProtocol.Factory; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +public class PipelineExecutionTaskBuilder { + private CoordinatorContext coordinatorContext; + + public PipelineExecutionTaskBuilder(CoordinatorContext coordinatorContext) { + this.coordinatorContext = coordinatorContext; + } + + public static PipelineExecutionTask build(CoordinatorContext coordinatorContext, + Map workerToFragmentsParam) { + PipelineExecutionTaskBuilder builder = new PipelineExecutionTaskBuilder(coordinatorContext); + return builder.buildTask(coordinatorContext, workerToFragmentsParam); + } + + private PipelineExecutionTask buildTask(CoordinatorContext coordinatorContext, + Map workerToFragmentsParam) { + BackendServiceProxy backendServiceProxy = BackendServiceProxy.getInstance(); + PipelineExecutionTask pipelineExecutionTask = new PipelineExecutionTask( + coordinatorContext, + backendServiceProxy, + buildMultiFragmentTasks(coordinatorContext, backendServiceProxy, workerToFragmentsParam) + ); + coordinatorContext.getJobProcessor().setSqlPipelineTask(pipelineExecutionTask); + return pipelineExecutionTask; + } + + private Map buildMultiFragmentTasks( + CoordinatorContext coordinatorContext, BackendServiceProxy backendServiceProxy, + Map workerToFragmentsParam) { + + Map workerToSerializeFragments = serializeFragments(workerToFragmentsParam); + + Map fragmentTasks = Maps.newLinkedHashMap(); + for (Entry kv : + workerToFragmentsParam.entrySet()) { + BackendWorker worker = (BackendWorker) kv.getKey(); + TPipelineFragmentParamsList fragmentParamsList = kv.getValue(); + ByteString serializeFragments = workerToSerializeFragments.get(worker); + + Backend backend = worker.getBackend(); + fragmentTasks.put( + worker.id(), + new MultiFragmentsPipelineTask( + coordinatorContext, + backend, + backendServiceProxy, + serializeFragments, + buildSingleFragmentPipelineTask(backend, fragmentParamsList) + ) + ); + } + return fragmentTasks; + } + + private Map buildSingleFragmentPipelineTask( + Backend backend, TPipelineFragmentParamsList fragmentParamsList) { + Map tasks = Maps.newLinkedHashMap(); + for (TPipelineFragmentParams fragmentParams : fragmentParamsList.getParamsList()) { + int fragmentId = fragmentParams.getFragmentId(); + Set instanceIds = fragmentParams.getLocalParams() + .stream() + .map(TPipelineInstanceParams::getFragmentInstanceId) + .collect(Collectors.toSet()); + tasks.put(fragmentId, new SingleFragmentPipelineTask(backend, fragmentId, instanceIds)); + } + return tasks; + } + + private Map serializeFragments( + Map workerToFragmentsParam) { + + AtomicLong compressedSize = new AtomicLong(0); + Map serializedFragments = workerToFragmentsParam.entrySet() + .parallelStream() + .map(kv -> { + try { + ByteString serializeString = + new TFastSerializer(1024, new Factory()).serialize(kv.getValue()); + return Pair.of(kv.getKey(), serializeString); + } catch (Throwable t) { + throw new IllegalStateException(t.getMessage(), t); + } + }) + .peek(kv -> compressedSize.addAndGet(kv.second.size())) + .collect(Collectors.toMap(Pair::key, Pair::value)); + + coordinatorContext.updateProfileIfPresent( + profile -> profile.updateFragmentCompressedSize(compressedSize.get()) + ); + coordinatorContext.updateProfileIfPresent(SummaryProfile::setFragmentSerializeTime); + + return serializedFragments; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java new file mode 100644 index 00000000000000..2ec38e8cc8e3ea --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/QueryProcessor.java @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.common.Status; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.ResultSink; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.JobProcessor; +import org.apache.doris.qe.ResultReceiver; +import org.apache.doris.qe.RowBatch; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatusCode; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.TException; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; + +public class QueryProcessor implements JobProcessor { + private static final Logger LOG = LogManager.getLogger(QueryProcessor.class); + + // constant fields + private final long limitRows; + + // mutable field + private Optional sqlPipelineTask; + private final CoordinatorContext coordinatorContext; + private final List runningReceivers; + private int receiverOffset; + private long numReceivedRows; + + public QueryProcessor(CoordinatorContext coordinatorContext, List runningReceivers) { + this.coordinatorContext = Objects.requireNonNull(coordinatorContext, "coordinatorContext can not be null"); + this.runningReceivers = new CopyOnWriteArrayList<>( + Objects.requireNonNull(runningReceivers, "runningReceivers can not be null") + ); + + this.limitRows = coordinatorContext.fragments.get(coordinatorContext.fragments.size() - 1) + .getPlanRoot() + .getLimit(); + + this.sqlPipelineTask = Optional.empty(); + } + + public static QueryProcessor build(CoordinatorContext coordinatorContext) { + PipelineDistributedPlan topFragment = coordinatorContext.topDistributedPlan; + DataSink topDataSink = coordinatorContext.dataSink; + Boolean enableParallelResultSink; + if (topDataSink instanceof ResultSink) { + enableParallelResultSink = coordinatorContext.queryOptions.isEnableParallelResultSink(); + } else { + enableParallelResultSink = coordinatorContext.queryOptions.isEnableParallelOutfile(); + } + + List topInstances = topFragment.getInstanceJobs(); + List receivers = Lists.newArrayListWithCapacity(topInstances.size()); + Map distinctWorkerJobs = Maps.newLinkedHashMap(); + for (AssignedJob topInstance : topInstances) { + distinctWorkerJobs.putIfAbsent(topInstance.getAssignedWorker().id(), topInstance); + } + + for (AssignedJob topInstance : distinctWorkerJobs.values()) { + DistributedPlanWorker topWorker = topInstance.getAssignedWorker(); + TNetworkAddress execBeAddr = new TNetworkAddress(topWorker.host(), topWorker.brpcPort()); + receivers.add( + new ResultReceiver( + coordinatorContext.queryId, + topInstance.instanceId(), + topWorker.id(), + execBeAddr, + coordinatorContext.timeoutDeadline.get(), + coordinatorContext.connectContext.getSessionVariable().getMaxMsgSizeOfResultReceiver(), + enableParallelResultSink + ) + ); + } + return new QueryProcessor(coordinatorContext, receivers); + } + + @Override + public void setSqlPipelineTask(PipelineExecutionTask pipelineExecutionTask) { + this.sqlPipelineTask = Optional.ofNullable(pipelineExecutionTask); + } + + public boolean isEos() { + return runningReceivers.isEmpty(); + } + + public RowBatch getNext() throws UserException, TException, RpcException { + ResultReceiver receiver = runningReceivers.get(receiverOffset); + Status status = new Status(); + RowBatch resultBatch = receiver.getNext(status); + if (!status.ok()) { + LOG.warn("Query {} coordinator get next fail, {}, need cancel.", + DebugUtil.printId(coordinatorContext.queryId), status.getErrorMsg()); + } + coordinatorContext.updateStatusIfOk(status); + + Status copyStatus = coordinatorContext.readCloneStatus(); + if (!copyStatus.ok()) { + if (Strings.isNullOrEmpty(copyStatus.getErrorMsg())) { + copyStatus.rewriteErrorMsg(); + } + if (copyStatus.isRpcError()) { + throw new RpcException(null, copyStatus.getErrorMsg()); + } else { + String errMsg = copyStatus.getErrorMsg(); + LOG.warn("query failed: {}", errMsg); + throw new UserException(errMsg); + } + } + + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().dryRunQuery) { + if (resultBatch.isEos()) { + numReceivedRows += resultBatch.getQueryStatistics().getReturnedRows(); + } + } else if (resultBatch.getBatch() != null) { + numReceivedRows += resultBatch.getBatch().getRowsSize(); + } + + if (resultBatch.isEos()) { + runningReceivers.remove(receiver); + if (!runningReceivers.isEmpty()) { + resultBatch.setEos(false); + } + + // if this query is a block query do not cancel. + boolean hasLimit = limitRows > 0; + if (!coordinatorContext.isBlockQuery + && coordinatorContext.instanceNum.get() > 1 + && hasLimit && numReceivedRows >= limitRows) { + if (LOG.isDebugEnabled()) { + LOG.debug("no block query, return num >= limit rows, need cancel"); + } + coordinatorContext.cancelSchedule(new Status(TStatusCode.LIMIT_REACH, "query reach limit")); + } + } + + if (!runningReceivers.isEmpty()) { + receiverOffset = (receiverOffset + 1) % runningReceivers.size(); + } + return resultBatch; + } + + public void cancel(Status cancelReason) { + for (ResultReceiver receiver : runningReceivers) { + receiver.cancel(cancelReason); + } + + this.sqlPipelineTask.ifPresent(sqlPipelineTask -> { + for (MultiFragmentsPipelineTask fragmentsTask : sqlPipelineTask.getChildrenTasks().values()) { + fragmentsTask.cancelExecute(cancelReason); + } + }); + } + + public int getReceiverOffset() { + return receiverOffset; + } + + public long getNumReceivedRows() { + return numReceivedRows; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java new file mode 100644 index 00000000000000..42cf08fb2e3b18 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/RuntimeFiltersThriftBuilder.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.worker.BackendWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.RuntimeFilter; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TRuntimeFilterParams; +import org.apache.doris.thrift.TRuntimeFilterTargetParams; +import org.apache.doris.thrift.TRuntimeFilterTargetParamsV2; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** RuntimeFiltersThriftBuilder */ +public class RuntimeFiltersThriftBuilder { + public final TNetworkAddress mergeAddress; + + private final List runtimeFilters; + private final AssignedJob mergeInstance; + private final Set broadcastRuntimeFilterIds; + private final Map> ridToTargets; + private final Map ridToBuilderNum; + + private RuntimeFiltersThriftBuilder( + TNetworkAddress mergeAddress, List runtimeFilters, + AssignedJob mergeInstance, Set broadcastRuntimeFilterIds, + Map> ridToTargets, + Map ridToBuilderNum) { + this.mergeAddress = mergeAddress; + this.runtimeFilters = runtimeFilters; + this.mergeInstance = mergeInstance; + this.broadcastRuntimeFilterIds = broadcastRuntimeFilterIds; + this.ridToTargets = ridToTargets; + this.ridToBuilderNum = ridToBuilderNum; + } + + public boolean isMergeRuntimeFilterInstance(AssignedJob instance) { + return mergeInstance == instance; + } + + public void setRuntimeFilterThriftParams(TRuntimeFilterParams runtimeFilterParams) { + for (RuntimeFilter rf : runtimeFilters) { + List targets = ridToTargets.get(rf.getFilterId()); + if (targets == null) { + continue; + } + + if (rf.hasRemoteTargets()) { + Map targetToParams = new LinkedHashMap<>(); + for (RuntimeFilterTarget target : targets) { + TRuntimeFilterTargetParamsV2 targetParams = targetToParams.computeIfAbsent( + target.address, address -> { + TRuntimeFilterTargetParamsV2 params = new TRuntimeFilterTargetParamsV2(); + params.target_fragment_instance_addr = address; + params.target_fragment_instance_ids = new ArrayList<>(); + return params; + }); + + targetParams.target_fragment_instance_ids.add(target.instanceId); + } + + runtimeFilterParams.putToRidToTargetParamv2( + rf.getFilterId().asInt(), new ArrayList<>(targetToParams.values()) + ); + } else { + List targetParams = Lists.newArrayList(); + for (RuntimeFilterTarget target : targets) { + targetParams.add(new TRuntimeFilterTargetParams(target.instanceId, target.address)); + } + runtimeFilterParams.putToRidToTargetParam(rf.getFilterId().asInt(), targetParams); + } + } + for (Map.Entry entry : ridToBuilderNum.entrySet()) { + boolean isBroadcastRuntimeFilter = broadcastRuntimeFilterIds.contains(entry.getKey().asInt()); + int builderNum = isBroadcastRuntimeFilter ? 1 : entry.getValue(); + runtimeFilterParams.putToRuntimeFilterBuilderNum(entry.getKey().asInt(), builderNum); + } + for (RuntimeFilter rf : runtimeFilters) { + runtimeFilterParams.putToRidToRuntimeFilter(rf.getFilterId().asInt(), rf.toThrift()); + } + } + + public static RuntimeFiltersThriftBuilder compute( + List runtimeFilters, List distributedPlans) { + PipelineDistributedPlan topMostPlan = distributedPlans.get(distributedPlans.size() - 1); + AssignedJob mergeInstance = topMostPlan.getInstanceJobs().get(0); + BackendWorker worker = (BackendWorker) mergeInstance.getAssignedWorker(); + TNetworkAddress mergeAddress = new TNetworkAddress(worker.host(), worker.brpcPort()); + + Set broadcastRuntimeFilterIds = runtimeFilters + .stream() + .filter(RuntimeFilter::isBroadcast) + .map(r -> r.getFilterId().asInt()) + .collect(Collectors.toSet()); + + Map> ridToTargetParam = Maps.newLinkedHashMap(); + Map ridToBuilderNum = Maps.newLinkedHashMap(); + for (PipelineDistributedPlan plan : distributedPlans) { + PlanFragment fragment = plan.getFragmentJob().getFragment(); + // Transform to + for (RuntimeFilterId rid : fragment.getTargetRuntimeFilterIds()) { + List targetFragments = + ridToTargetParam.computeIfAbsent(rid, k -> new ArrayList<>()); + for (AssignedJob instanceJob : plan.getInstanceJobs()) { + BackendWorker backendWorker = (BackendWorker) instanceJob.getAssignedWorker(); + Backend backend = backendWorker.getBackend(); + targetFragments.add(new RuntimeFilterTarget( + instanceJob.instanceId(), + new TNetworkAddress(backend.getHost(), backend.getBrpcPort()) + )); + } + } + + for (RuntimeFilterId rid : fragment.getBuilderRuntimeFilterIds()) { + int distinctWorkerNum = (int) plan.getInstanceJobs() + .stream() + .map(AssignedJob::getAssignedWorker) + .map(DistributedPlanWorker::id) + .distinct() + .count(); + ridToBuilderNum.merge(rid, distinctWorkerNum, Integer::sum); + } + } + return new RuntimeFiltersThriftBuilder( + mergeAddress, runtimeFilters, mergeInstance, + broadcastRuntimeFilterIds, ridToTargetParam, ridToBuilderNum + ); + } + + public static class RuntimeFilterTarget { + public final TUniqueId instanceId; + public final TNetworkAddress address; + + public RuntimeFilterTarget(TUniqueId instanceId, TNetworkAddress address) { + this.instanceId = instanceId; + this.address = address; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java new file mode 100644 index 00000000000000..2fba0654af9d67 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTask.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.qe.QueryStatisticsItem.FragmentInstanceInfo; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TReportExecStatusParams; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +public class SingleFragmentPipelineTask extends LeafRuntimeTask { + private static final Logger LOG = LogManager.getLogger(PipelineExecutionTask.class); + + // immutable parameters + private final Backend backend; + private final int fragmentId; + private final long lastMissingHeartbeatTime; + private final Set instanceIds; + + // mutate states + private final AtomicBoolean done = new AtomicBoolean(); + + public SingleFragmentPipelineTask(Backend backend, int fragmentId, Set instanceIds) { + this.backend = backend; + this.fragmentId = fragmentId; + this.instanceIds = instanceIds; + this.lastMissingHeartbeatTime = backend.getLastMissingHeartbeatTime(); + } + + // update profile. + // return true if profile is updated. Otherwise, return false. + // Has to use synchronized to ensure there are not concurrent update threads. Or the done + // state maybe update wrong and will lose data. see https://github.com/apache/doris/pull/29802/files. + public boolean processReportExecStatus(TReportExecStatusParams reportExecStatus) { + // The fragment or instance is not finished, not need update + if (!reportExecStatus.done) { + return false; + } + return this.done.compareAndSet(false, true); + } + + public boolean isBackendHealthy(long jobId) { + if (backend.getLastMissingHeartbeatTime() > lastMissingHeartbeatTime && !backend.isAlive()) { + LOG.warn("backend {} is down while joining the coordinator. job id: {}", backend.getId(), jobId); + return false; + } + return true; + } + + public boolean isDone() { + return done.get(); + } + + public Backend getBackend() { + return backend; + } + + public int getFragmentId() { + return fragmentId; + } + + public List buildFragmentInstanceInfo() { + TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + List infos = Lists.newArrayListWithCapacity(instanceIds.size()); + for (TUniqueId instanceId : instanceIds) { + infos.add( + new FragmentInstanceInfo.Builder() + .address(address) + .fragmentId(String.valueOf(fragmentId)) + .instanceId(instanceId) + .build() + ); + } + return infos; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java new file mode 100644 index 00000000000000..c04861cbf43693 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java @@ -0,0 +1,568 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.runtime; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan; +import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanRanges; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource; +import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedScanBucketOlapTableJob; +import org.apache.doris.nereids.trees.plans.physical.TopnFilter; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.DataStreamSink; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.MultiCastDataSink; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.OlapTableSink; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.planner.SortNode; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.thrift.PaloInternalServiceVersion; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPipelineFragmentParams; +import org.apache.doris.thrift.TPipelineFragmentParamsList; +import org.apache.doris.thrift.TPipelineInstanceParams; +import org.apache.doris.thrift.TPlanFragment; +import org.apache.doris.thrift.TPlanFragmentDestination; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TRuntimeFilterParams; +import org.apache.doris.thrift.TScanRangeParams; +import org.apache.doris.thrift.TTopnFilterDesc; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.LinkedHashMultiset; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multiset; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +public class ThriftPlansBuilder { + private static final Logger LOG = LogManager.getLogger(ThriftPlansBuilder.class); + + public static Map plansToThrift( + CoordinatorContext coordinatorContext) { + + List distributedPlans = coordinatorContext.distributedPlans; + + // we should set runtime predicate first, then we can use heap sort and to thrift + setRuntimePredicateIfNeed(coordinatorContext.scanNodes); + + RuntimeFiltersThriftBuilder runtimeFiltersThriftBuilder = RuntimeFiltersThriftBuilder.compute( + coordinatorContext.runtimeFilters, distributedPlans); + Supplier> topNFilterThriftSupplier + = topNFilterToThrift(coordinatorContext.topnFilters); + + Multiset workerProcessInstanceNum = computeInstanceNumPerWorker(distributedPlans); + Map fragmentsGroupByWorker = Maps.newLinkedHashMap(); + int currentInstanceIndex = 0; + Map sharedFileScanRangeParams = Maps.newLinkedHashMap(); + for (PipelineDistributedPlan currentFragmentPlan : distributedPlans) { + sharedFileScanRangeParams.putAll(computeFileScanRangeParams(currentFragmentPlan)); + + Map exchangeSenderNum = computeExchangeSenderNum(currentFragmentPlan); + ListMultimap instancesPerWorker + = groupInstancePerWorker(currentFragmentPlan); + Map workerToCurrentFragment = Maps.newLinkedHashMap(); + + for (AssignedJob instanceJob : currentFragmentPlan.getInstanceJobs()) { + TPipelineFragmentParams currentFragmentParam = fragmentToThriftIfAbsent( + currentFragmentPlan, instanceJob, workerToCurrentFragment, + instancesPerWorker, exchangeSenderNum, sharedFileScanRangeParams, + workerProcessInstanceNum, coordinatorContext); + + TPipelineInstanceParams instanceParam = instanceToThrift( + currentFragmentParam, instanceJob, runtimeFiltersThriftBuilder, + topNFilterThriftSupplier, currentInstanceIndex++ + ); + currentFragmentParam.getLocalParams().add(instanceParam); + } + + // arrange fragments by the same worker, + // so we can merge and send multiple fragment to a backend use one rpc + for (Entry kv : workerToCurrentFragment.entrySet()) { + TPipelineFragmentParamsList fragments = fragmentsGroupByWorker.computeIfAbsent( + kv.getKey(), w -> new TPipelineFragmentParamsList()); + fragments.addToParamsList(kv.getValue()); + } + } + + // backend should initialize fragment from target to source in backend, then + // it can bind the receiver fragment for the sender fragment, but frontend + // compute thrift message from source to fragment, so we need to reverse fragments. + for (DistributedPlanWorker worker : fragmentsGroupByWorker.keySet()) { + Collections.reverse(fragmentsGroupByWorker.get(worker).getParamsList()); + } + + setParamsForOlapTableSink(distributedPlans, fragmentsGroupByWorker); + + // remove redundant params to reduce rpc message size + for (Entry kv : fragmentsGroupByWorker.entrySet()) { + boolean isFirstFragmentInCurrentBackend = true; + for (TPipelineFragmentParams fragmentParams : kv.getValue().getParamsList()) { + if (!isFirstFragmentInCurrentBackend) { + fragmentParams.unsetDescTbl(); + fragmentParams.unsetFileScanParams(); + fragmentParams.unsetCoord(); + fragmentParams.unsetQueryGlobals(); + fragmentParams.unsetResourceInfo(); + fragmentParams.setIsSimplifiedParam(true); + } + isFirstFragmentInCurrentBackend = false; + } + } + return fragmentsGroupByWorker; + } + + private static ListMultimap + groupInstancePerWorker(PipelineDistributedPlan fragmentPlan) { + ListMultimap workerToInstances = ArrayListMultimap.create(); + for (AssignedJob instanceJob : fragmentPlan.getInstanceJobs()) { + workerToInstances.put(instanceJob.getAssignedWorker(), instanceJob); + } + return workerToInstances; + } + + private static void setRuntimePredicateIfNeed(Collection scanNodes) { + for (ScanNode scanNode : scanNodes) { + if (scanNode instanceof OlapScanNode) { + for (SortNode topnFilterSortNode : scanNode.getTopnFilterSortNodes()) { + topnFilterSortNode.setHasRuntimePredicate(); + } + } + } + } + + private static Supplier> topNFilterToThrift(List topnFilters) { + return Suppliers.memoize(() -> { + if (CollectionUtils.isEmpty(topnFilters)) { + return null; + } + + List filterDescs = new ArrayList<>(topnFilters.size()); + for (TopnFilter topnFilter : topnFilters) { + filterDescs.add(topnFilter.toThrift()); + } + return filterDescs; + }); + } + + private static void setParamsForOlapTableSink(List distributedPlans, + Map fragmentsGroupByWorker) { + int numBackendsWithSink = 0; + for (PipelineDistributedPlan distributedPlan : distributedPlans) { + PlanFragment fragment = distributedPlan.getFragmentJob().getFragment(); + if (fragment.getSink() instanceof OlapTableSink) { + numBackendsWithSink += (int) distributedPlan.getInstanceJobs() + .stream() + .map(AssignedJob::getAssignedWorker) + .distinct() + .count(); + } + } + + for (Entry kv : fragmentsGroupByWorker.entrySet()) { + TPipelineFragmentParamsList fragments = kv.getValue(); + for (TPipelineFragmentParams fragmentParams : fragments.getParamsList()) { + if (fragmentParams.getFragment().getOutputSink().getType() == TDataSinkType.OLAP_TABLE_SINK) { + int loadStreamPerNode = 1; + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) { + loadStreamPerNode = ConnectContext.get().getSessionVariable().getLoadStreamPerNode(); + } + fragmentParams.setLoadStreamPerNode(loadStreamPerNode); + fragmentParams.setTotalLoadStreams(numBackendsWithSink * loadStreamPerNode); + fragmentParams.setNumLocalSink(fragmentParams.getLocalParams().size()); + LOG.info("num local sink for backend {} is {}", fragmentParams.getBackendId(), + fragmentParams.getNumLocalSink()); + } + } + } + } + + private static Multiset computeInstanceNumPerWorker( + List distributedPlans) { + Multiset workerCounter = LinkedHashMultiset.create(); + for (PipelineDistributedPlan distributedPlan : distributedPlans) { + for (AssignedJob instanceJob : distributedPlan.getInstanceJobs()) { + workerCounter.add(instanceJob.getAssignedWorker()); + } + } + return workerCounter; + } + + private static Map computeExchangeSenderNum(PipelineDistributedPlan distributedPlan) { + Map senderNum = Maps.newLinkedHashMap(); + for (Entry kv : distributedPlan.getInputs().entries()) { + ExchangeNode exchangeNode = kv.getKey(); + PipelineDistributedPlan childPlan = (PipelineDistributedPlan) kv.getValue(); + senderNum.merge(exchangeNode.getId().asInt(), childPlan.getInstanceJobs().size(), Integer::sum); + } + return senderNum; + } + + private static void setMultiCastDestinationThrift(PipelineDistributedPlan fragmentPlan) { + MultiCastDataSink multiCastDataSink = (MultiCastDataSink) fragmentPlan.getFragmentJob().getFragment().getSink(); + List> destinationList = multiCastDataSink.getDestinations(); + + List dataStreamSinks = multiCastDataSink.getDataStreamSinks(); + for (int i = 0; i < dataStreamSinks.size(); i++) { + DataStreamSink realSink = dataStreamSinks.get(i); + List destinations = destinationList.get(i); + for (Entry> kv : fragmentPlan.getDestinations().entrySet()) { + DataSink sink = kv.getKey(); + if (sink == realSink) { + List destInstances = kv.getValue(); + for (AssignedJob destInstance : destInstances) { + destinations.add(instanceToDestination(destInstance)); + } + break; + } + } + } + } + + private static List nonMultiCastDestinationToThrift(PipelineDistributedPlan plan) { + Map> destinationsMapping = plan.getDestinations(); + List destinations = Lists.newArrayList(); + if (!destinationsMapping.isEmpty()) { + List destinationJobs = destinationsMapping.entrySet().iterator().next().getValue(); + for (AssignedJob destinationJob : destinationJobs) { + destinations.add(instanceToDestination(destinationJob)); + } + } + return destinations; + } + + private static TPlanFragmentDestination instanceToDestination(AssignedJob instance) { + DistributedPlanWorker worker = instance.getAssignedWorker(); + String host = worker.host(); + int port = worker.port(); + int brpcPort = worker.brpcPort(); + + TPlanFragmentDestination destination = new TPlanFragmentDestination(); + destination.setServer(new TNetworkAddress(host, port)); + destination.setBrpcServer(new TNetworkAddress(host, brpcPort)); + destination.setFragmentInstanceId(instance.instanceId()); + return destination; + } + + private static TPipelineFragmentParams fragmentToThriftIfAbsent( + PipelineDistributedPlan fragmentPlan, AssignedJob assignedJob, + Map workerToFragmentParams, + ListMultimap instancesPerWorker, + Map exchangeSenderNum, + Map fileScanRangeParamsMap, + Multiset workerProcessInstanceNum, + CoordinatorContext coordinatorContext) { + DistributedPlanWorker worker = assignedJob.getAssignedWorker(); + return workerToFragmentParams.computeIfAbsent(worker, w -> { + PlanFragment fragment = fragmentPlan.getFragmentJob().getFragment(); + ConnectContext connectContext = coordinatorContext.connectContext; + + TPipelineFragmentParams params = new TPipelineFragmentParams(); + params.setIsNereids(true); + params.setBackendId(worker.id()); + params.setProtocolVersion(PaloInternalServiceVersion.V1); + params.setDescTbl(coordinatorContext.descriptorTable); + params.setQueryId(coordinatorContext.queryId); + params.setFragmentId(fragment.getFragmentId().asInt()); + + // Each tParam will set the total number of Fragments that need to be executed on the same BE, + // and the BE will determine whether all Fragments have been executed based on this information. + // Notice. load fragment has a small probability that FragmentNumOnHost is 0, for unknown reasons. + params.setFragmentNumOnHost(workerProcessInstanceNum.count(worker)); + + params.setNeedWaitExecutionTrigger(coordinatorContext.twoPhaseExecution()); + params.setPerExchNumSenders(exchangeSenderNum); + + List nonMultiCastDestinations; + if (fragment.getSink() instanceof MultiCastDataSink) { + nonMultiCastDestinations = Lists.newArrayList(); + setMultiCastDestinationThrift(fragmentPlan); + } else { + nonMultiCastDestinations = nonMultiCastDestinationToThrift(fragmentPlan); + } + params.setDestinations(nonMultiCastDestinations); + + int instanceNumInThisFragment = fragmentPlan.getInstanceJobs().size(); + params.setNumSenders(instanceNumInThisFragment); + params.setTotalInstances(instanceNumInThisFragment); + + params.setCoord(coordinatorContext.coordinatorAddress); + params.setCurrentConnectFe(coordinatorContext.directConnectFrontendAddress.get()); + params.setQueryGlobals(coordinatorContext.queryGlobals); + params.setQueryOptions(new TQueryOptions(coordinatorContext.queryOptions)); + long memLimit = coordinatorContext.queryOptions.getMemLimit(); + // update memory limit for colocate join + if (connectContext != null + && !connectContext.getSessionVariable().isDisableColocatePlan() + && fragment.hasColocatePlanNode()) { + int rate = Math.min(Config.query_colocate_join_memory_limit_penalty_factor, instanceNumInThisFragment); + memLimit = coordinatorContext.queryOptions.getMemLimit() / rate; + } + params.getQueryOptions().setMemLimit(memLimit); + + params.setSendQueryStatisticsWithEveryBatch(fragment.isTransferQueryStatisticsWithEveryBatch()); + + TPlanFragment planThrift = fragment.toThrift(); + planThrift.query_cache_param = fragment.queryCacheParam; + params.setFragment(planThrift); + params.setLocalParams(Lists.newArrayList()); + params.setWorkloadGroups(coordinatorContext.getWorkloadGroups()); + + params.setFileScanParams(fileScanRangeParamsMap); + + if (fragmentPlan.getFragmentJob() instanceof UnassignedScanBucketOlapTableJob) { + int bucketNum = ((UnassignedScanBucketOlapTableJob) fragmentPlan.getFragmentJob()) + .getOlapScanNodes() + .get(0) + .getBucketNum(); + params.setNumBuckets(bucketNum); + } + + List instances = instancesPerWorker.get(worker); + Map instanceToIndex = instanceToIndex(instances); + + // local shuffle params: bucket_seq_to_instance_idx and shuffle_idx_to_instance_idx + params.setBucketSeqToInstanceIdx(computeBucketIdToInstanceId(fragmentPlan, w, instanceToIndex)); + params.setShuffleIdxToInstanceIdx(computeDestIdToInstanceId(fragmentPlan, w, instanceToIndex)); + return params; + }); + } + + private static Map instanceToIndex(List instances) { + Map instanceToIndex = Maps.newLinkedHashMap(); + for (int instanceIndex = 0; instanceIndex < instances.size(); instanceIndex++) { + instanceToIndex.put(instances.get(instanceIndex), instanceIndex); + } + + return instanceToIndex; + } + + private static Map computeFileScanRangeParams( + PipelineDistributedPlan distributedPlan) { + // scan node id -> TFileScanRangeParams + Map fileScanRangeParamsMap = Maps.newLinkedHashMap(); + for (ScanNode scanNode : distributedPlan.getFragmentJob().getScanNodes()) { + if (scanNode instanceof FileQueryScanNode) { + TFileScanRangeParams fileScanRangeParams = ((FileQueryScanNode) scanNode).getFileScanRangeParams(); + fileScanRangeParamsMap.put(scanNode.getId().asInt(), fileScanRangeParams); + } + } + + return fileScanRangeParamsMap; + } + + private static TPipelineInstanceParams instanceToThrift( + TPipelineFragmentParams currentFragmentParam, AssignedJob instance, + RuntimeFiltersThriftBuilder runtimeFiltersThriftBuilder, + Supplier> topNFilterThriftSupplier, int currentInstanceNum) { + TPipelineInstanceParams instanceParam = new TPipelineInstanceParams(); + instanceParam.setFragmentInstanceId(instance.instanceId()); + setScanSourceParam(currentFragmentParam, instance, instanceParam); + + instanceParam.setSenderId(instance.indexInUnassignedJob()); + instanceParam.setBackendNum(currentInstanceNum); + instanceParam.setRuntimeFilterParams(new TRuntimeFilterParams()); + + instanceParam.setTopnFilterDescs(topNFilterThriftSupplier.get()); + + // set for runtime filter + TRuntimeFilterParams runtimeFilterParams = new TRuntimeFilterParams(); + runtimeFilterParams.setRuntimeFilterMergeAddr(runtimeFiltersThriftBuilder.mergeAddress); + instanceParam.setRuntimeFilterParams(runtimeFilterParams); + if (runtimeFiltersThriftBuilder.isMergeRuntimeFilterInstance(instance)) { + runtimeFiltersThriftBuilder.setRuntimeFilterThriftParams(runtimeFilterParams); + } + return instanceParam; + } + + private static void setScanSourceParam( + TPipelineFragmentParams currentFragmentParam, AssignedJob instance, + TPipelineInstanceParams instanceParams) { + + boolean isLocalShuffle = instance instanceof LocalShuffleAssignedJob; + if (isLocalShuffle && ((LocalShuffleAssignedJob) instance).receiveDataFromLocal) { + // save thrift rpc message size, don't need perNodeScanRanges and perNodeSharedScans, + // but the perNodeScanRanges is required rpc field + instanceParams.setPerNodeScanRanges(Maps.newLinkedHashMap()); + return; + } + + ScanSource scanSource = instance.getScanSource(); + PerNodeScanParams scanParams; + if (scanSource instanceof BucketScanSource) { + scanParams = computeBucketScanSourceParam((BucketScanSource) scanSource); + } else { + scanParams = computeDefaultScanSourceParam((DefaultScanSource) scanSource); + } + // perNodeScanRanges is required + instanceParams.setPerNodeScanRanges(scanParams.perNodeScanRanges); + + if (isLocalShuffle) { + // a fragment in a backend only enable local shuffle once for the first local shuffle instance, + // because we just skip set scan params for LocalShuffleAssignedJob.receiveDataFromLocal == true + ignoreDataDistribution(currentFragmentParam); + } + } + + // local shuffle has two functions: + // 1. use 10 scan instances -> local shuffle -> 10 agg instances, this function can balance data in agg + // 2. use 1 scan instance -> local shuffle -> 10 agg, this function is ignore_data_distribution, + // it can add parallel in agg + private static void ignoreDataDistribution(TPipelineFragmentParams currentFragmentParam) { + // `parallel_instances == 1` is the switch of ignore_data_distribution, + // and backend will use 1 instance to scan a little data, and local shuffle to + // # SessionVariable.parallel_pipeline_task_num instances to increment parallel + currentFragmentParam.setParallelInstances(1); + } + + private static PerNodeScanParams computeDefaultScanSourceParam(DefaultScanSource defaultScanSource) { + Map> perNodeScanRanges = Maps.newLinkedHashMap(); + Map perNodeSharedScans = Maps.newLinkedHashMap(); + for (Entry kv : defaultScanSource.scanNodeToScanRanges.entrySet()) { + int scanNodeId = kv.getKey().getId().asInt(); + perNodeScanRanges.put(scanNodeId, kv.getValue().params); + perNodeSharedScans.put(scanNodeId, true); + } + + return new PerNodeScanParams(perNodeScanRanges, perNodeSharedScans); + } + + private static PerNodeScanParams computeBucketScanSourceParam(BucketScanSource bucketScanSource) { + Map> perNodeScanRanges = Maps.newLinkedHashMap(); + Map perNodeSharedScans = Maps.newLinkedHashMap(); + for (Entry> kv : + bucketScanSource.bucketIndexToScanNodeToTablets.entrySet()) { + Map scanNodeToRanges = kv.getValue(); + for (Entry kv2 : scanNodeToRanges.entrySet()) { + int scanNodeId = kv2.getKey().getId().asInt(); + List scanRanges = perNodeScanRanges.computeIfAbsent(scanNodeId, ArrayList::new); + scanRanges.addAll(kv2.getValue().params); + perNodeSharedScans.put(scanNodeId, true); + } + } + return new PerNodeScanParams(perNodeScanRanges, perNodeSharedScans); + } + + private static Map computeBucketIdToInstanceId( + PipelineDistributedPlan receivePlan, DistributedPlanWorker worker, + Map instanceToIndex) { + List instanceJobs = receivePlan.getInstanceJobs(); + if (instanceJobs.isEmpty() || !(instanceJobs.get(0).getScanSource() instanceof BucketScanSource)) { + // bucket_seq_to_instance_id is optional, so we can return null to save memory + return null; + } + + Map bucketIdToInstanceId = Maps.newLinkedHashMap(); + for (AssignedJob instanceJob : instanceJobs) { + if (instanceJob.getAssignedWorker().id() != worker.id()) { + continue; + } + if (instanceJob instanceof LocalShuffleAssignedJob + && ((LocalShuffleAssignedJob) instanceJob).receiveDataFromLocal) { + continue; + } + Integer instanceIndex = instanceToIndex.get(instanceJob); + BucketScanSource bucketScanSource = (BucketScanSource) instanceJob.getScanSource(); + for (Integer bucketIndex : bucketScanSource.bucketIndexToScanNodeToTablets.keySet()) { + bucketIdToInstanceId.put(bucketIndex, instanceIndex); + } + } + return bucketIdToInstanceId; + } + + private static Map computeDestIdToInstanceId( + PipelineDistributedPlan receivePlan, DistributedPlanWorker worker, + Map instanceToIndex) { + if (receivePlan.getInputs().isEmpty()) { + // shuffle_idx_to_index_id is required + return Maps.newLinkedHashMap(); + } + + Map destIdToInstanceId = Maps.newLinkedHashMap(); + filterInstancesWhichReceiveDataFromRemote( + receivePlan, worker, + (instanceJob, destId) -> destIdToInstanceId.put(destId, instanceToIndex.get(instanceJob)) + ); + return destIdToInstanceId; + } + + private static void filterInstancesWhichReceiveDataFromRemote( + PipelineDistributedPlan receivePlan, DistributedPlanWorker filterWorker, + BiConsumer computeFn) { + + // current only support all input plans have same destination with same order, + // so we can get first input plan to compute shuffle index to instance id + Set> exchangeToChildPlanSet = receivePlan.getInputs().entries(); + if (exchangeToChildPlanSet.isEmpty()) { + return; + } + Entry exchangeToChildPlan = exchangeToChildPlanSet.iterator().next(); + ExchangeNode linkNode = exchangeToChildPlan.getKey(); + PipelineDistributedPlan firstInputPlan = (PipelineDistributedPlan) exchangeToChildPlan.getValue(); + Map> sinkToDestInstances = firstInputPlan.getDestinations(); + for (Entry> kv : sinkToDestInstances.entrySet()) { + DataSink senderSink = kv.getKey(); + if (senderSink.getExchNodeId().asInt() == linkNode.getId().asInt()) { + for (int destId = 0; destId < kv.getValue().size(); destId++) { + AssignedJob assignedJob = kv.getValue().get(destId); + if (assignedJob.getAssignedWorker().id() == filterWorker.id()) { + computeFn.accept(assignedJob, destId); + } + } + break; + } + } + } + + private static class PerNodeScanParams { + Map> perNodeScanRanges; + Map perNodeSharedScans; + + public PerNodeScanParams(Map> perNodeScanRanges, + Map perNodeSharedScans) { + this.perNodeScanRanges = perNodeScanRanges; + this.perNodeSharedScans = perNodeSharedScans; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index 97a06176fef04d..41b7b79ad9ff76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -235,7 +235,7 @@ public Future execPlanFragmentsAsync(TN } public Future execPlanFragmentStartAsync(TNetworkAddress address, - PExecPlanFragmentStartRequest request) throws TException, RpcException { + PExecPlanFragmentStartRequest request) throws RpcException { try { final BackendServiceClient client = getProxy(address); return client.execPlanFragmentStartAsync(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index e3d543d9111dc8..31aef1ebb7c791 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2908,15 +2908,18 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c } // Step 3: get snapshot + String label = request.getLabelName(); TGetSnapshotResult result = new TGetSnapshotResult(); result.setStatus(new TStatus(TStatusCode.OK)); - Snapshot snapshot = Env.getCurrentEnv().getBackupHandler().getSnapshot(request.getLabelName()); + Snapshot snapshot = Env.getCurrentEnv().getBackupHandler().getSnapshot(label); if (snapshot == null) { result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_NOT_EXIST); - result.getStatus().addToErrorMsgs("snapshot not exist"); + result.getStatus().addToErrorMsgs(String.format("snapshot %s not exist", label)); } else { result.setMeta(snapshot.getMeta()); result.setJobInfo(snapshot.getJobInfo()); + LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}", + label, snapshot.getMeta().length, snapshot.getJobInfo().length); } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index 67de69c57dac87..c2479147ec7b20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -35,7 +35,7 @@ public class StatisticConstants { public static final String PARTITION_STATISTIC_TBL_NAME = "partition_statistics"; public static final String HISTOGRAM_TBL_NAME = "histogram_statistics"; - public static final int MAX_NAME_LEN = 64; + public static final int MAX_NAME_LEN = 1024; public static final int ID_LEN = 4096; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index a1cac53599556d..01eb92b9be3f40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -667,7 +667,7 @@ private static TFetchSchemaTableDataResult queriesMetadataResult(TSchemaTableReq trow.addToColumnValue(new TCell().setLongVal(-1)); } - List tgroupList = queryInfo.getCoord().gettWorkloadGroups(); + List tgroupList = queryInfo.getCoord().getTWorkloadGroups(); if (tgroupList != null && tgroupList.size() == 1) { trow.addToColumnValue(new TCell().setLongVal(tgroupList.get(0).id)); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java index be698776cac61a..c4120b9a199809 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.ClientPool; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.ThriftUtils; import org.apache.doris.system.Backend; import org.apache.doris.thrift.BackendService; import org.apache.doris.thrift.TAgentServiceVersion; @@ -208,6 +209,14 @@ public void run() { private static void submitTasks(long backendId, BackendService.Client client, List agentTaskRequests) throws TException { if (!agentTaskRequests.isEmpty()) { + if (LOG.isDebugEnabled()) { + long size = agentTaskRequests.stream() + .map(ThriftUtils::getBinaryMessageSize) + .reduce(0L, Long::sum); + TTaskType firstTaskType = agentTaskRequests.get(0).getTaskType(); + LOG.debug("submit {} tasks to backend[{}], total size: {}, first task type: {}", + agentTaskRequests.size(), backendId, size, firstTaskType); + } client.submitTasks(agentTaskRequests); } if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index c410f37e5c96ad..4824994c3b676f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -66,6 +66,7 @@ public class CreateReplicaTask extends AgentTask { private TStorageMedium storageMedium; private TCompressionType compressionType; private long rowStorePageSize; + private long storagePageSize; private List columns; @@ -156,7 +157,8 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition List rowStoreColumnUniqueIds, Map objectPool, long rowStorePageSize, - boolean variantEnableFlattenNested) { + boolean variantEnableFlattenNested, + long storagePageSize) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -204,6 +206,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.objectPool = objectPool; this.rowStorePageSize = rowStorePageSize; this.variantEnableFlattenNested = variantEnableFlattenNested; + this.storagePageSize = storagePageSize; } public void setIsRecoverTask(boolean isRecoverTask) { @@ -412,6 +415,7 @@ public TCreateTabletReq toThrift() { createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold); createTabletReq.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold); + createTabletReq.setStoragePageSize(storagePageSize); if (binlogConfig != null) { createTabletReq.setBinlogConfig(binlogConfig.toThrift()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/InternalSchemaInitializerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/InternalSchemaInitializerTest.java new file mode 100644 index 00000000000000..1eb003e81b9fee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/InternalSchemaInitializerTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.ModifyColumnClause; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.statistics.StatisticConstants; + +import com.google.common.collect.Lists; +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class InternalSchemaInitializerTest { + @Test + public void testGetModifyColumn() { + new MockUp() { + @Mock + public HMSExternalTable.DLAType getDlaType() { + return HMSExternalTable.DLAType.HUDI; + } + }; + + InternalSchemaInitializer initializer = new InternalSchemaInitializer(); + OlapTable table = new OlapTable(); + Column key1 = new Column("key1", ScalarType.createVarcharType(100), true, null, false, null, ""); + Column key2 = new Column("key2", ScalarType.createVarcharType(100), true, null, true, null, ""); + Column key3 = new Column("key3", ScalarType.createVarcharType(1024), true, null, null, ""); + Column key4 = new Column("key4", ScalarType.createVarcharType(1025), true, null, null, ""); + Column key5 = new Column("key5", ScalarType.INT, true, null, null, ""); + Column value1 = new Column("value1", ScalarType.INT, false, null, null, ""); + Column value2 = new Column("value2", ScalarType.createVarcharType(100), false, null, null, ""); + List schema = Lists.newArrayList(); + schema.add(key1); + schema.add(key2); + schema.add(key3); + schema.add(key4); + schema.add(key5); + schema.add(value1); + schema.add(value2); + table.fullSchema = schema; + List modifyColumnClauses = initializer.getModifyColumnClauses(table); + Assertions.assertEquals(2, modifyColumnClauses.size()); + ModifyColumnClause clause1 = (ModifyColumnClause) modifyColumnClauses.get(0); + Assertions.assertEquals("key1", clause1.getColumn().getName()); + Assertions.assertEquals(StatisticConstants.MAX_NAME_LEN, clause1.getColumn().getType().getLength()); + Assertions.assertFalse(clause1.getColumn().isAllowNull()); + + ModifyColumnClause clause2 = (ModifyColumnClause) modifyColumnClauses.get(1); + Assertions.assertEquals("key2", clause2.getColumn().getName()); + Assertions.assertEquals(StatisticConstants.MAX_NAME_LEN, clause2.getColumn().getType().getLength()); + Assertions.assertTrue(clause2.getColumn().isAllowNull()); + + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index eb3500d13d96fa..041ca89bfc5bde 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -188,4 +188,52 @@ public void testTag() throws AnalysisException { Assert.assertEquals(1, tagMap.size()); Assert.assertEquals(Tag.DEFAULT_BACKEND_TAG.value, tagMap.get(Tag.TYPE_LOCATION)); } + + @Test + public void testStoragePageSize() throws AnalysisException { + Map properties = Maps.newHashMap(); + + // Test default value + Assert.assertEquals(PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE, + PropertyAnalyzer.analyzeStoragePageSize(properties)); + + // Test valid value + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "8192"); // 8KB + Assert.assertEquals(8192, PropertyAnalyzer.analyzeStoragePageSize(properties)); + + // Test lower boundary value + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "4096"); // 4KB + Assert.assertEquals(4096, PropertyAnalyzer.analyzeStoragePageSize(properties)); + + // Test upper boundary value + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "10485760"); // 10MB + Assert.assertEquals(10485760, PropertyAnalyzer.analyzeStoragePageSize(properties)); + + // Test invalid number format + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "invalid"); + try { + PropertyAnalyzer.analyzeStoragePageSize(properties); + Assert.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("Invalid storage page size")); + } + + // Test value below minimum limit + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "1024"); // 1KB + try { + PropertyAnalyzer.analyzeStoragePageSize(properties); + Assert.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("Storage page size must be between 4KB and 10MB")); + } + + // Test value above maximum limit + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE, "20971520"); // 20MB + try { + PropertyAnalyzer.analyzeStoragePageSize(properties); + Assert.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("Storage page size must be between 4KB and 10MB")); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index a58b785dbca8ef..75a30453d756d4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; /** @@ -226,7 +227,7 @@ public void testEs8Mapping() throws IOException, URISyntaxException { public void testDateType() throws IOException, URISyntaxException { ObjectNode testDateFormat = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null, new ArrayList<>()); - List parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new ArrayList<>()); + List parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new ArrayList<>(), new HashMap<>()); Assertions.assertEquals(8, parseColumns.size()); for (Column column : parseColumns) { String name = column.getName(); @@ -259,7 +260,7 @@ public void testDateType() throws IOException, URISyntaxException { public void testFieldAlias() throws IOException, URISyntaxException { ObjectNode testFieldAlias = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/test_field_alias.json")), null, new ArrayList<>()); - List parseColumns = EsUtil.genColumnsFromEs("test_field_alias", null, testFieldAlias, true, new ArrayList<>()); + List parseColumns = EsUtil.genColumnsFromEs("test_field_alias", null, testFieldAlias, true, new ArrayList<>(), new HashMap<>()); Assertions.assertEquals("datetimev2(0)", parseColumns.get(2).getType().toSql()); Assertions.assertEquals("text", parseColumns.get(4).getType().toSql()); } @@ -269,7 +270,7 @@ public void testComplexType() throws IOException, URISyntaxException { ObjectNode testFieldAlias = EsUtil.getRootSchema( EsUtil.getMapping(loadJsonFromFile("data/es/es6_dynamic_complex_type.json")), null, new ArrayList<>()); List columns = EsUtil.genColumnsFromEs("test_complex_type", "complex_type", testFieldAlias, true, - new ArrayList<>()); + new ArrayList<>(), new HashMap<>()); Assertions.assertEquals(3, columns.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index ca5344990e7ec7..69eab5198aa214 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -71,30 +71,32 @@ public void testBinaryPredicateConvertEsDsl() { Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral); Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral); Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3)); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson()); + Map column2typeMap = new HashMap<>(); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(neExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson()); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson()); + QueryBuilders.toEsDsl(neExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr, column2typeMap).toJson()); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr, column2typeMap).toJson()); SlotRef k2 = new SlotRef(null, "k2"); Expr dateTimeLiteral = new StringLiteral("2023-02-19 22:00:00"); Expr dateTimeEqExpr = new BinaryPredicate(Operator.EQ, k2, dateTimeLiteral); Assertions.assertEquals("{\"term\":{\"k2\":\"2023-02-19T22:00:00.000+08:00\"}}", QueryBuilders.toEsDsl(dateTimeEqExpr, new ArrayList<>(), new HashMap<>(), - BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build()).toJson()); + BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build(), + new HashMap<>()).toJson()); SlotRef k3 = new SlotRef(null, "k3"); Expr stringLiteral = new StringLiteral(""); Expr stringNeExpr = new BinaryPredicate(Operator.NE, k3, stringLiteral); Assertions.assertEquals("{\"bool\":{\"must\":{\"exists\":{\"field\":\"k3\"}},\"must_not\":{\"term\":{\"k3\":\"\"}}}}", - QueryBuilders.toEsDsl(stringNeExpr).toJson()); + QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); stringLiteral = new StringLiteral("message"); stringNeExpr = new BinaryPredicate(Operator.NE, k3, stringLiteral); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k3\":\"message\"}}}}", - QueryBuilders.toEsDsl(stringNeExpr).toJson()); + QueryBuilders.toEsDsl(stringNeExpr, column2typeMap).toJson()); } @Test @@ -103,6 +105,7 @@ public void testCompoundPredicateConvertEsDsl() { IntLiteral intLiteral1 = new IntLiteral(3); SlotRef k2 = new SlotRef(null, "k2"); IntLiteral intLiteral2 = new IntLiteral(5); + Map column2typeMap = new HashMap<>(); BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1); BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2); CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1, @@ -111,21 +114,22 @@ public void testCompoundPredicateConvertEsDsl() { binaryPredicate2); CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null); Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(andPredicate).toJson()); + QueryBuilders.toEsDsl(andPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(orPredicate).toJson()); + QueryBuilders.toEsDsl(orPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(notPredicate).toJson()); + QueryBuilders.toEsDsl(notPredicate, column2typeMap).toJson()); } @Test public void testIsNullPredicateConvertEsDsl() { SlotRef k1 = new SlotRef(null, "k1"); + Map column2typeMap = new HashMap<>(); IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false); IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}", - QueryBuilders.toEsDsl(isNullPredicate).toJson()); - Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson()); + QueryBuilders.toEsDsl(isNullPredicate, column2typeMap).toJson()); + Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate, column2typeMap).toJson()); } @Test @@ -137,13 +141,28 @@ public void testLikePredicateConvertEsDsl() { LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1); LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2); LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson()); + Map column2typeMap = new HashMap<>(); + column2typeMap.put("k1", "keyword"); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1, column2typeMap).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate, column2typeMap).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2, column2typeMap).toJson()); List notPushDownList = new ArrayList<>(); Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), - BuilderOptions.builder().likePushDown(false).build())); + BuilderOptions.builder().likePushDown(false).build(), column2typeMap)); Assertions.assertFalse(notPushDownList.isEmpty()); + + // like can not push down + column2typeMap.clear(); + notPushDownList.clear(); + column2typeMap.put("k1", "text"); + Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate1, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertNull(QueryBuilders.toEsDsl(regexPredicate, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + + Assertions.assertEquals(3, notPushDownList.size()); } @Test @@ -156,9 +175,10 @@ public void testInPredicateConvertEsDsl() { intLiterals.add(intLiteral2); InPredicate isInPredicate = new InPredicate(k1, intLiterals, false); InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true); - Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson()); + Map column2typeMap = new HashMap<>(); + Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate, column2typeMap).toJson()); Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}", - QueryBuilders.toEsDsl(isNotInPredicate).toJson()); + QueryBuilders.toEsDsl(isNotInPredicate, column2typeMap).toJson()); } @Test @@ -166,11 +186,12 @@ public void testFunctionCallConvertEsDsl() { SlotRef k1 = new SlotRef(null, "k1"); String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}"; StringLiteral stringLiteral = new StringLiteral(str); + Map column2typeMap = new HashMap<>(); List exprs = new ArrayList<>(); exprs.add(k1); exprs.add(stringLiteral); FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs); - Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson()); + Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr, column2typeMap).toJson()); SlotRef k2 = new SlotRef(null, "k2"); IntLiteral intLiteral = new IntLiteral(5); @@ -179,7 +200,44 @@ public void testFunctionCallConvertEsDsl() { functionCallExpr); Assertions.assertEquals( "{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}", - QueryBuilders.toEsDsl(compoundPredicate).toJson()); + QueryBuilders.toEsDsl(compoundPredicate, column2typeMap).toJson()); + } + + @Test + public void testLikeFunctionCallConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + String str1 = "text%"; + StringLiteral stringLiteral1 = new StringLiteral(str1); + Map column2typeMap = new HashMap<>(); + column2typeMap.put("k1", "keyword"); + List exprs1 = new ArrayList<>(); + exprs1.add(k1); + exprs1.add(stringLiteral1); + FunctionCallExpr likeFunctionCallExpr = new FunctionCallExpr("like", exprs1); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"text*\"}}", + QueryBuilders.toEsDsl(likeFunctionCallExpr, column2typeMap).toJson()); + + SlotRef k2 = new SlotRef(null, "k2"); + String str2 = "text$"; + StringLiteral stringLiteral2 = new StringLiteral(str2); + List exprs2 = new ArrayList<>(); + exprs2.add(k2); + exprs2.add(stringLiteral2); + FunctionCallExpr regexFunctionCallExpr = new FunctionCallExpr("regexp", exprs2); + Assertions.assertEquals("{\"wildcard\":{\"k2\":\"text$\"}}", + QueryBuilders.toEsDsl(regexFunctionCallExpr, column2typeMap).toJson()); + + + column2typeMap.clear(); + List notPushDownList = new ArrayList<>(); + column2typeMap.put("k1", "text"); + Assertions.assertNull(QueryBuilders.toEsDsl(likeFunctionCallExpr, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap)); + Assertions.assertEquals("{\"wildcard\":{\"k2\":\"text$\"}}", + QueryBuilders.toEsDsl(regexFunctionCallExpr, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(true).build(), column2typeMap).toJson()); + + Assertions.assertEquals(1, notPushDownList.size()); } @Test @@ -189,8 +247,10 @@ public void testCastConvertEsDsl() { BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3)); List notPushDownList = new ArrayList<>(); Map fieldsContext = new HashMap<>(); + Map col2typeMap = new HashMap<>(); BuilderOptions builderOptions = BuilderOptions.builder().likePushDown(true).build(); - Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions)); + Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap)); Assertions.assertEquals(1, notPushDownList.size()); SlotRef k2 = new SlotRef(null, "k2"); @@ -199,7 +259,7 @@ public void testCastConvertEsDsl() { CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate, eqPredicate); - QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals(3, notPushDownList.size()); SlotRef k3 = new SlotRef(null, "k3"); @@ -207,7 +267,7 @@ public void testCastConvertEsDsl() { CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3); BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr, new FloatLiteral(3.0, Type.DOUBLE)); - QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals(3, notPushDownList.size()); SlotRef k4 = new SlotRef(null, "k4"); @@ -215,25 +275,29 @@ public void testCastConvertEsDsl() { CastExpr castFloatExpr = new CastExpr(Type.FLOAT, k4); BinaryPredicate castFloatPredicate = new BinaryPredicate(Operator.GE, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - QueryBuilders.QueryBuilder queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + QueryBuilders.QueryBuilder queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, + fieldsContext, builderOptions, col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"lte\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.LE, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"gte\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.LT, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"gt\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); castFloatPredicate = new BinaryPredicate(Operator.GT, new FloatLiteral(3.0, Type.FLOAT), castFloatExpr); - queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions, + col2typeMap); Assertions.assertEquals("{\"range\":{\"k4\":{\"lt\":3.0}}}", queryBuilder.toJson()); Assertions.assertEquals(3, notPushDownList.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java index 4371b9f578594b..80c2873aed337b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java @@ -33,7 +33,9 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * MappingTest @@ -343,6 +345,259 @@ public void testGenerateMapping5() { assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping); } + // Test more than two tables, and the same table num in source plan is equals to target plan + @Test + public void testGenerateMapping6() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " orders,\n" + + " lineitem l1,\n" + + " lineitem l2,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n" + + " AND l1.l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " lineitem l1,\n" + + " orders,\n" + + " lineitem l2,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n" + + " AND l2.l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(6, generateRelationMapping.size()); + + // expected table relation mapping is as following + // (1, 3), (2, 2), (3, 0), (0, 1) + // (1, 0), (2, 3), (3, 2), (0, 1) + // (1, 2), (2, 3), (3, 0), (0, 1) + // (1, 0), (2, 2), (3, 3), (0, 1) + // (1, 2), (2, 0), (3, 3), (0, 1) + // (1, 3), (2, 0), (3, 2), (0, 1) + Set> expectedRelationMappingSet = new HashSet<>(); + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(3)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(3), new RelationId(0)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(3)); + expectedRelationMapping.put(new RelationId(3), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(2), new RelationId(3)); + expectedRelationMapping.put(new RelationId(3), new RelationId(0)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(3), new RelationId(3)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(3), new RelationId(3)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(3)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(3), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet); + } + + // Test more than two tables, and the same table num in source plan is less then the num of target plan + @Test + public void testGenerateMapping7() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " orders,\n" + + " lineitem l1,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l3.l_orderkey\n" + + " AND l1.l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " lineitem l1,\n" + + " orders,\n" + + " lineitem l2,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n" + + " AND l2.l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(6, generateRelationMapping.size()); + + // expected table relation mapping is as following + // (1, 2), (2, 0), (0, 1) + // (1, 2), (2, 3), (0, 1) + // (1, 0), (2, 2), (0, 1) + // (1, 0), (2, 3), (0, 1) + // (1, 3), (2, 0), (0, 1) + // (1, 3), (2, 2), (0, 1) + Set> expectedRelationMappingSet = new HashSet<>(); + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(2), new RelationId(3)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(3)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(3)); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(3)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet); + } + + // Test more than two tables, and the same table num in source plan is more then the num of target plan + @Test + public void testGenerateMapping8() { + Plan sourcePlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " orders,\n" + + " lineitem l1,\n" + + " lineitem l2,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n" + + " AND l1.l_orderkey = o_orderkey") + .getPlan(); + + Plan targetPlan = PlanChecker.from(connectContext) + .analyze("SELECT orders.*, l1.l_orderkey, l3.l_orderkey " + + "FROM\n" + + " lineitem l1,\n" + + " orders,\n" + + " lineitem l3\n" + + "WHERE\n" + + " l1.l_orderkey = l3.l_orderkey\n" + + " AND l3.l_orderkey = o_orderkey") + .getPlan(); + List sourceRelations = new ArrayList<>(); + sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations); + + List targetRelations = new ArrayList<>(); + targetPlan.accept(RelationCollector.INSTANCE, targetRelations); + + List generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations); + Assertions.assertNotNull(generateRelationMapping); + Assertions.assertEquals(6, generateRelationMapping.size()); + + // expected table relation mapping is as following + // (1, 0), (2, 2), (0, 1) + // (1, 0), (3, 2), (0, 1) + // (2, 0), (1, 2), (0, 1) + // (3, 0), (2, 2), (0, 1) + // (2, 0), (3, 2), (0, 1) + // (3, 0), (1, 2), (0, 1) + Set> expectedRelationMappingSet = new HashSet<>(); + BiMap expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(1), new RelationId(0)); + expectedRelationMapping.put(new RelationId(3), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(3), new RelationId(0)); + expectedRelationMapping.put(new RelationId(2), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(2), new RelationId(0)); + expectedRelationMapping.put(new RelationId(3), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + expectedRelationMapping = HashBiMap.create(); + expectedRelationMapping.put(new RelationId(3), new RelationId(0)); + expectedRelationMapping.put(new RelationId(1), new RelationId(2)); + expectedRelationMapping.put(new RelationId(0), new RelationId(1)); + expectedRelationMappingSet.add(expectedRelationMapping); + + assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet); + } + private void assertRelationMapping(RelationMapping relationMapping, BiMap expectRelationMapping, BiMap expectSlotMapping) { @@ -362,6 +617,20 @@ private void assertRelationMapping(RelationMapping relationMapping, Assertions.assertEquals(generatedSlotMapping, expectSlotMapping); } + private void assertRelationMapping(Set relationMapping, + Set> expectRelationMapping) { + // check relation mapping if equals or not + Set> relationMappingSet = new HashSet<>(); + relationMapping.forEach(mapping -> { + BiMap generatedRelationMapping = HashBiMap.create(); + mapping.getMappedRelationMap().forEach((key, value) -> + generatedRelationMapping.put(key.getRelationId(), value.getRelationId())); + relationMappingSet.add(generatedRelationMapping); + } + ); + Assertions.assertEquals(relationMappingSet, expectRelationMapping); + } + protected static class RelationCollector extends DefaultPlanVisitor> { public static final RelationCollector INSTANCE = new RelationCollector(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index 6f2996207d03f0..f6c6201a934b0b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -74,6 +74,7 @@ public class AgentTaskTest { private TStorageType storageType = TStorageType.COLUMN; private long rowStorePageSize = 16384L; + private long storagePageSize = 65536L; private List columns; private MarkedCountDownLatch latch = new MarkedCountDownLatch(3); @@ -108,7 +109,8 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false); + TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, + storagePageSize); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); diff --git a/fe/pom.xml b/fe/pom.xml index d158b650a553b5..d78cfd50b819b4 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -259,7 +259,7 @@ under the License. 2.7 1.1.1 5.8.2 - 4.0.3 + 6.0.0 0.16.0 8.5.86 2.18.0 diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 324f45d59ff13a..8d93b973b96fa2 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -197,6 +197,7 @@ message ObjectStoreInfoPB { optional string user_id = 13; optional EncryptionInfoPB encryption_info = 14; optional bool sse_enabled = 15; + optional bool use_path_style = 16; } // The legacy ObjectStoreInfoPB is stored in InstanceInfoPB diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 30f74af7d671a6..7e65a4a9bf20c8 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -516,6 +516,7 @@ message TabletMetaPB { optional int64 time_series_compaction_time_threshold_seconds = 31 [default = 3600]; optional int64 time_series_compaction_empty_rowsets_threshold = 32 [default = 5]; optional int64 time_series_compaction_level_threshold = 33 [default = 1]; + optional int64 storage_page_size = 34 [default=65536]; // For cloud optional int64 index_id = 1000; @@ -572,6 +573,7 @@ message TabletMetaCloudPB { optional int64 group_commit_data_bytes = 36 [default = 134217728]; optional int64 time_series_compaction_empty_rowsets_threshold = 37 [default = 5]; optional int64 time_series_compaction_level_threshold = 38 [default = 1]; + optional int64 storage_page_size = 39 [default=65536]; // Use for selectdb-cloud optional string table_name = 101; diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index f02b8c0f88605a..9094c6eea2a05e 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -184,6 +184,7 @@ struct TCreateTabletReq { 27: optional i64 time_series_compaction_level_threshold = 1 28: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.DEFAULT // Deprecated 29: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2 + 30: optional i64 storage_page_size = 65536 // For cloud 1000: optional bool is_in_memory = false diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 8319b846695270..b80ce5cac3b969 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -256,7 +256,7 @@ struct TOlapTableSchemaParam { 11: optional string auto_increment_column 12: optional i32 auto_increment_column_unique_id = -1 13: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V1 - 14: optional Types.TUniqueKeyUpdateMode unique_key_update_mode = Types.TUniqueKeyUpdateMode.UPSERT + 14: optional Types.TUniqueKeyUpdateMode unique_key_update_mode 15: optional i32 sequence_map_col_unique_id = -1 } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index f531db3028224a..29fecc27539ff4 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -352,6 +352,10 @@ struct TQueryOptions { 136: optional bool enable_phrase_query_sequential_opt = true; 137: optional bool enable_auto_create_when_overwrite = false; + + 138: optional i64 orc_tiny_stripe_threshold_bytes = 8388608; + 139: optional i64 orc_once_max_read_bytes = 8388608; + 140: optional i64 orc_max_merge_distance_bytes = 1048576; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index eb5266942c023a..ec4497b267b22f 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -321,17 +321,18 @@ struct TPaimonDeletionFileDesc { struct TPaimonFileDesc { 1: optional string paimon_split 2: optional string paimon_column_names - 3: optional string db_name - 4: optional string table_name + 3: optional string db_name // deprecated + 4: optional string table_name // deprecated 5: optional string paimon_predicate - 6: optional map paimon_options - 7: optional i64 ctl_id - 8: optional i64 db_id - 9: optional i64 tbl_id - 10: optional i64 last_update_time + 6: optional map paimon_options // deprecated + 7: optional i64 ctl_id // deprecated + 8: optional i64 db_id // deprecated + 9: optional i64 tbl_id // deprecated + 10: optional i64 last_update_time // deprecated 11: optional string file_format 12: optional TPaimonDeletionFileDesc deletion_file; - 13: optional map hadoop_conf + 13: optional map hadoop_conf // deprecated + 14: optional string paimon_table } struct TTrinoConnectorFileDesc { diff --git a/regression-test/data/doc/data-operate/import/import-way/insert-into-manual.md.out b/regression-test/data/doc/data-operate/import/import-way/insert-into-manual.md.out new file mode 100644 index 00000000000000..9de2e13f029c18 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/insert-into-manual.md.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +5 + +-- !sql -- +3 + diff --git a/regression-test/data/doc/data-operate/import/import-way/routine-load-manual.md.out b/regression-test/data/doc/data-operate/import/import-way/routine-load-manual.md.out new file mode 100644 index 00000000000000..4c0c1a9e6b7c41 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/routine-load-manual.md.out @@ -0,0 +1,123 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +1 Emily 25 +2 Benjamin 35 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql4 -- +1 Benjamin 18 +2 Emily 20 + +-- !sql5 -- +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql6 -- +1 Emily 25 +2 Benjamin 35 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql7 -- +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql8 -- +1 Benjamin 18 2024-02-04T10:00 + +-- !sql9 -- +1 Benjamin 18 2024-02-04T10:00 +2 Emily 20 2024-02-05T11:00 + +-- !sql10 -- +1 Emily 25 +2 Benjamin 35 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql11 -- +1 Emily 25 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql12 -- +1 Benjamin 18 180 +2 Emily 20 200 +3 Alexander 22 220 + +-- !sql13 -- +1 Benjamin 18 +2 Emily 20 +3 Alexander 22 + +-- !sql14 -- +1 Benjamin 18 180 +2 Emily 20 200 +3 Alexander 22 220 + +-- !sql15 -- +1 Benjamin 18 +2 Emily 20 +3 Alexander 22 + +-- !sql16 -- +1 Benjamin 18 [1, 2, 3, 4, 5] +2 Emily 20 [6, 7, 8, 9, 10] +3 Alexander 22 [11, 12, 13, 14, 15] + +-- !sql17 -- +1 Benjamin 18 {"a":100, "b":200} +2 Emily 20 {"c":300, "d":400} +3 Alexander 22 {"e":500, "f":600} + +-- !sql18 -- +1 Benjamin 18 \N \N +2 Emily 20 \N \N +3 Alexander 22 \N \N + +-- !sql19 -- +2022-05-05 10001 Test01 Beijing windows \N +2022-05-06 10001 Test01 Shanghai windows \N +2022-05-05 10002 Test01 Beijing linux \N +2022-05-06 10002 Test01 Shanghai linux \N +2022-05-05 10003 Test01 Beijing macos \N +2022-05-06 10003 Test01 Jiangsu macos \N +2022-05-05 10004 Test01 Hebei windows \N +2022-05-06 10004 Test01 Shaanxi windows \N + diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_array.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_array.json new file mode 100644 index 00000000000000..8260cad6b774b8 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_array.json @@ -0,0 +1,3 @@ +{ "id" : 1, "name" : "Benjamin", "age":18, "array":[1,2,3,4,5]} +{ "id" : 2, "name" : "Emily", "age":20, "array":[6,7,8,9,10]} +{ "id" : 3, "name" : "Alexander", "age":22, "array":[11,12,13,14,15]} \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_bitmap.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_bitmap.json new file mode 100644 index 00000000000000..73bb59becb4651 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_bitmap.json @@ -0,0 +1,3 @@ +{ "id" : 1, "name" : "Benjamin", "age":18, "bitmap_id":243} +{ "id" : 2, "name" : "Emily", "age":20, "bitmap_id":28574} +{ "id" : 3, "name" : "Alexander", "age":22, "bitmap_id":8573} \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_column_mapping.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_column_mapping.csv new file mode 100644 index 00000000000000..7eb3f2c7998930 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_column_mapping.csv @@ -0,0 +1,3 @@ +1,Benjamin,18 +2,Emily,20 +3,Alexander,22 \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_csv.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_csv.csv new file mode 100644 index 00000000000000..9e401297ab29c3 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_csv.csv @@ -0,0 +1,10 @@ +1,Emily,25 +2,Benjamin,35 +3,Olivia,28 +4,Alexander,60 +5,Ava,17 +6,William,69 +7,Sophia,32 +8,James,64 +9,Emma,37 +10,Liam,64 \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_delete.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_delete.csv new file mode 100644 index 00000000000000..11d1c8468f2513 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_delete.csv @@ -0,0 +1,2 @@ +3,Alexander,22 +5,William,26 \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_hll.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_hll.csv new file mode 100644 index 00000000000000..a7d55471221009 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_hll.csv @@ -0,0 +1,8 @@ +2022-05-05,10001,Test01,Beijing,windows +2022-05-05,10002,Test01,Beijing,linux +2022-05-05,10003,Test01,Beijing,macos +2022-05-05,10004,Test01,Hebei,windows +2022-05-06,10001,Test01,Shanghai,windows +2022-05-06,10002,Test01,Shanghai,linux +2022-05-06,10003,Test01,Jiangsu,macos +2022-05-06,10004,Test01,Shaanxi,windows \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_json.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_json.json new file mode 100644 index 00000000000000..6e06fe67521c57 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_json.json @@ -0,0 +1,3 @@ +{ "id" : 1, "name" : "Benjamin", "age":18 } +{ "id" : 2, "name" : "Emily", "age":20 } +{ "id" : 3, "name" : "Alexander", "age":22 } \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_json_path.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_json_path.json new file mode 100644 index 00000000000000..f96e2cd232bdbd --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_json_path.json @@ -0,0 +1,3 @@ +{ "name" : "Benjamin", "id" : 1, "num":180 , "age":18 } +{ "name" : "Emily", "id" : 2, "num":200 , "age":20 } +{ "name" : "Alexander", "id" : 3, "num":220 , "age":22 } \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_json_root.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_json_root.json new file mode 100644 index 00000000000000..60df381670c905 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_json_root.json @@ -0,0 +1,3 @@ +{"id": 1231, "source" :{ "id" : 1, "name" : "Benjamin", "age":18 }} +{"id": 1232, "source" :{ "id" : 2, "name" : "Emily", "age":20 }} +{"id": 1233, "source" :{ "id" : 3, "name" : "Alexander", "age":22 }} \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_map.json b/regression-test/data/doc/data-operate/import/import-way/test_rl_map.json new file mode 100644 index 00000000000000..d87689824956e2 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_map.json @@ -0,0 +1,3 @@ +{ "id" : 1, "name" : "Benjamin", "age":18, "map":{"a": 100, "b": 200}} +{ "id" : 2, "name" : "Emily", "age":20, "map":{"c": 300, "d": 400}} +{ "id" : 3, "name" : "Alexander", "age":22, "map":{"e": 500, "f": 600}} \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_max_filter_ratio.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_max_filter_ratio.csv new file mode 100644 index 00000000000000..1e07d97228de94 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_max_filter_ratio.csv @@ -0,0 +1,3 @@ +1,Benjamin,18 +2,Emily,20 +3,Alexander,dirty_data \ No newline at end of file diff --git a/regression-test/data/doc/data-operate/import/import-way/test_rl_partition.csv b/regression-test/data/doc/data-operate/import/import-way/test_rl_partition.csv new file mode 100644 index 00000000000000..f41efd837e1fe6 --- /dev/null +++ b/regression-test/data/doc/data-operate/import/import-way/test_rl_partition.csv @@ -0,0 +1,3 @@ +1,Benjamin,18,2024-02-04 10:00:00 +2,Emily,20,2024-02-05 11:00:00 +3,Alexander,22,2024-02-06 12:00:00 \ No newline at end of file diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index 6d761babed7428..312aef06652ce5 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -220,6 +220,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_5_28 -- value1 value2 +-- !sql_5_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_5_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -349,6 +360,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_6_28 -- value1 value2 +-- !sql_6_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_6_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -517,6 +539,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_7_35 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_7_36 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_50 -- value1 value2 @@ -685,6 +720,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_8_33 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_8_34 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql01 -- ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] @@ -906,6 +954,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_5_28 -- value1 value2 +-- !sql_5_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_5_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -1035,6 +1094,17 @@ Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] -- !sql_6_28 -- value1 value2 +-- !sql_6_29 -- +string1 text#1 +string2 text2 +string3 text3_4*5 + +-- !sql_6_30 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -1203,6 +1273,19 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_7_35 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_7_36 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + -- !sql_7_50 -- value1 value2 @@ -1371,3 +1454,16 @@ Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] +-- !sql_8_33 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 + +-- !sql_8_34 -- +string1 text#1 +string2 text2 +string3 text3_4*5 +string4 text3_4*5 +string_ignore_above_10 text_ignore_above_10 + diff --git a/regression-test/data/external_table_p0/hive/test_orc_tiny_stripes.out b/regression-test/data/external_table_p0/hive/test_orc_tiny_stripes.out new file mode 100644 index 00000000000000..d08eb7c887263e --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_orc_tiny_stripes.out @@ -0,0 +1,2311 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + +-- !test_1 -- +372 + +-- !test_2 -- +1 str_1 10000000001 +1 str_1 10000000001 + +-- !test_3 -- +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 + +-- !test_4 -- +4 str_4 10000000004 +4 str_4 10000000004 + +-- !test_5 -- +348 + +-- !test_6 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_7 -- +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 + +-- !test_8 -- +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 + +-- !test_9 -- +10 + +-- !test_10 -- +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW + +-- !test_11 -- +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 + +-- !test_12 -- +0 +0 +6 + +-- !test_13 -- +20 +60 + +-- !test_14 -- +0 +0 +40 + diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index 1caeec713e269c..b2101cd96c87b7 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -436,6 +436,15 @@ t2 text Yes false \N NONE varchar varchar(65533) Yes true \N int_u bigint Yes false \N NONE +-- !sql -- +int_u bigint Yes true \N +text varchar(65533) Yes true \N +t2 varchar(65533) Yes false \N NONE + +-- !sql -- +varchar varchar(65533) Yes true \N +int_u bigint Yes false \N NONE + -- !sql -- internal @@ -870,3 +879,12 @@ t2 text Yes false \N NONE varchar varchar(65533) Yes true \N int_u bigint Yes false \N NONE +-- !sql -- +int_u bigint Yes true \N +text varchar(65533) Yes true \N +t2 varchar(65533) Yes false \N NONE + +-- !sql -- +varchar varchar(65533) Yes true \N +int_u bigint Yes false \N NONE + diff --git a/regression-test/data/insert_p0/test_insert_nan.out b/regression-test/data/insert_p0/test_insert_nan.out new file mode 100644 index 00000000000000..f0d5ba9c2d6c85 --- /dev/null +++ b/regression-test/data/insert_p0/test_insert_nan.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 \N + diff --git a/regression-test/data/inverted_index_p0/test_array_with_inverted_index_all_type.out b/regression-test/data/inverted_index_p0/test_array_with_inverted_index_all_type.out index 966ff5a6d5a811..8801dd5356694a 100644 --- a/regression-test/data/inverted_index_p0/test_array_with_inverted_index_all_type.out +++ b/regression-test/data/inverted_index_p0/test_array_with_inverted_index_all_type.out @@ -1,6 +1,24 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_null -- +101 \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !sql_null -- +101 \N \N \N \N + +-- !sql_null -- +101 \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !sql_null -- +101 \N \N \N \N + +-- !sql_null -- +101 \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !sql_null -- +101 \N \N \N \N + -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -900,6 +918,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 @@ -1003,7 +1022,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -1497,6 +1516,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 63 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 @@ -1600,7 +1620,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 63 -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -2500,6 +2520,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 @@ -2603,7 +2624,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -3097,6 +3118,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 63 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 @@ -3200,7 +3222,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 63 -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -4100,6 +4122,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 40 @@ -4203,7 +4226,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 65 -- !sql_count -- -101 +102 -- !sql_array_contains -- 0 1 @@ -4697,6 +4720,7 @@ ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 63 -- !sql -- +\N 0 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 38 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 ["2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29", "2024-04-29"] 41 diff --git a/regression-test/data/inverted_index_p2/documents-1000.json b/regression-test/data/inverted_index_p2/documents-1000.json new file mode 100644 index 00000000000000..a1b0aa8342edc8 --- /dev/null +++ b/regression-test/data/inverted_index_p2/documents-1000.json @@ -0,0 +1,1000 @@ +{"@timestamp": 893964617, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964653, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964672, "clientip":"26.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964679, "clientip":"247.37.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893964682, "clientip":"247.37.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964687, "clientip":"252.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964689, "clientip":"247.37.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964689, "clientip":"247.37.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964692, "clientip":"247.37.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893964703, "clientip":"247.37.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964704, "clientip":"247.37.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 1139} +{"@timestamp": 893964712, "clientip":"2.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964712, "clientip":"247.37.0.0", "request": "GET /french/tickets/body.html HTTP/1.0", "status": 200, "size": 3029} +{"@timestamp": 893964726, "clientip":"120.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964736, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964736, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964737, "clientip":"247.37.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964750, "clientip":"247.37.0.0", "request": "GET /french/tickets/tck_0804.htm HTTP/1.0", "status": 200, "size": 14521} +{"@timestamp": 893964753, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_quest_bg2.jpg HTTP/1.0", "status": 200, "size": 11324} +{"@timestamp": 893964755, "clientip":"126.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_header.gif HTTP/1.0", "status": 200, "size": 671} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_abroad2.gif HTTP/1.0", "status": 200, "size": 1512} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_infrance2.gif HTTP/1.0", "status": 200, "size": 1136} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 1647} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_quest2.gif HTTP/1.0", "status": 200, "size": 1271} +{"@timestamp": 893964772, "clientip":"247.37.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893964778, "clientip":"247.37.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964779, "clientip":"247.37.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893964779, "clientip":"247.37.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893964785, "clientip":"13.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964789, "clientip":"138.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964800, "clientip":"28.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964809, "clientip":"31.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964812, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893964815, "clientip":"29.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964825, "clientip":"92.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964841, "clientip":"134.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964844, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893964864, "clientip":"137.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964932, "clientip":"167.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964936, "clientip":"174.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964961, "clientip":"97.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964973, "clientip":"174.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965016, "clientip":"121.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965036, "clientip":"2.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965050, "clientip":"91.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965059, "clientip":"47.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965070, "clientip":"142.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965084, "clientip":"104.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965106, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965115, "clientip":"109.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965123, "clientip":"184.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965139, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965149, "clientip":"56.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965150, "clientip":"161.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965150, "clientip":"237.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965154, "clientip":"41.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965162, "clientip":"131.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965172, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965182, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893965193, "clientip":"180.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965198, "clientip":"176.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965200, "clientip":"192.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965278, "clientip":"4.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965293, "clientip":"76.6.0.0", "request": "GET /images/teams_hm_bg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965300, "clientip":"235.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965310, "clientip":"183.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965318, "clientip":"248.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965322, "clientip":"249.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965352, "clientip":"218.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965382, "clientip":"128.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965405, "clientip":"139.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965412, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965424, "clientip":"249.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965449, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965464, "clientip":"221.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965467, "clientip":"175.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965479, "clientip":"43.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965504, "clientip":"201.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965518, "clientip":"17.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965550, "clientip":"138.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965554, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965578, "clientip":"76.6.0.0", "request": "GET /images/nantes.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965585, "clientip":"252.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965598, "clientip":"120.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965616, "clientip":"2.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965622, "clientip":"44.135.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965627, "clientip":"44.135.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965628, "clientip":"44.135.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893965631, "clientip":"44.135.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965632, "clientip":"44.135.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /french/images/nav_comp_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965638, "clientip":"44.135.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965648, "clientip":"126.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965649, "clientip":"44.135.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965651, "clientip":"44.135.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893965651, "clientip":"44.135.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965652, "clientip":"44.135.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965655, "clientip":"44.135.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965659, "clientip":"44.135.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965659, "clientip":"44.135.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893965660, "clientip":"44.135.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965661, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893965661, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965663, "clientip":"44.135.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893965664, "clientip":"44.135.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893965668, "clientip":"26.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965684, "clientip":"13.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965700, "clientip":"28.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965700, "clientip":"44.135.0.0", "request": "GET /french/news/newsprr.htm HTTP/1.0", "status": 200, "size": 28486} +{"@timestamp": 893965705, "clientip":"44.135.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_press_on.gif HTTP/1.0", "status": 200, "size": 1757} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893965708, "clientip":"44.135.0.0", "request": "GET /french/images/news_hd_press.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965708, "clientip":"44.135.0.0", "request": "GET /images/news_arrow.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965712, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965724, "clientip":"31.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965727, "clientip":"92.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965727, "clientip":"134.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965731, "clientip":"29.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965738, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965753, "clientip":"237.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965796, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893965849, "clientip":"174.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965856, "clientip":"167.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965865, "clientip":"137.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965882, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965883, "clientip":"97.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965896, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/argentina78.html HTTP/1.0", "status": 200, "size": 20916} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/history_hm_header.gif HTTP/1.0", "status": 200, "size": 1034} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/football.GIF HTTP/1.0", "status": 200, "size": 1452} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/france98b.GIF HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/images/infrance.GIF HTTP/1.0", "status": 200, "size": 1017} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/images/reading.GIF HTTP/1.0", "status": 200, "size": 1094} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_30_off.gif HTTP/1.0", "status": 200, "size": 406} +{"@timestamp": 893965902, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_38_off.gif HTTP/1.0", "status": 200, "size": 436} +{"@timestamp": 893965903, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_54_off.gif HTTP/1.0", "status": 200, "size": 403} +{"@timestamp": 893965904, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_62_off.gif HTTP/1.0", "status": 200, "size": 437} +{"@timestamp": 893965905, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_70_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893965905, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_78_on.gif HTTP/1.0", "status": 200, "size": 501} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_94_off.gif HTTP/1.0", "status": 200, "size": 432} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_86_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/posters/argentina78.gif HTTP/1.0", "status": 200, "size": 4033} +{"@timestamp": 893965907, "clientip":"45.135.0.0", "request": "GET /images/hist7802.jpg HTTP/1.0", "status": 200, "size": 10813} +{"@timestamp": 893965907, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_trophy2.gif HTTP/1.0", "status": 200, "size": 443} +{"@timestamp": 893965908, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_50_off.gif HTTP/1.0", "status": 200, "size": 392} +{"@timestamp": 893965908, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_header.gif HTTP/1.0", "status": 200, "size": 2077} +{"@timestamp": 893965910, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_66_off.gif HTTP/1.0", "status": 200, "size": 435} +{"@timestamp": 893965911, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_82_off.gif HTTP/1.0", "status": 200, "size": 433} +{"@timestamp": 893965912, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bracket_top.gif HTTP/1.0", "status": 200, "size": 289} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /images/hist7801.jpg HTTP/1.0", "status": 200, "size": 12855} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_74_off.gif HTTP/1.0", "status": 200, "size": 423} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_34_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893965915, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bracket_bot.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_58_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_90_off.gif HTTP/1.0", "status": 200, "size": 429} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/images/thecup.GIF HTTP/1.0", "status": 200, "size": 1036} +{"@timestamp": 893965930, "clientip":"174.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965932, "clientip":"121.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965937, "clientip":"91.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965976, "clientip":"142.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965979, "clientip":"104.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965988, "clientip":"47.135.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 200, "size": 2739} +{"@timestamp": 893965992, "clientip":"109.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893965994, "clientip":"47.135.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 855} +{"@timestamp": 893965994, "clientip":"47.135.0.0", "request": "GET /french/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 995} +{"@timestamp": 893965998, "clientip":"47.135.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 413} +{"@timestamp": 893966000, "clientip":"47.135.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 982} +{"@timestamp": 893966001, "clientip":"47.135.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 1139} +{"@timestamp": 893966002, "clientip":"47.135.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966002, "clientip":"47.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966003, "clientip":"47.135.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 976} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/images/hm_official.gif HTTP/1.0", "status": 200, "size": 972} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966004, "clientip":"46.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966005, "clientip":"47.135.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966006, "clientip":"47.135.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 966} +{"@timestamp": 893966008, "clientip":"47.135.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 945} +{"@timestamp": 893966010, "clientip":"47.135.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966011, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966017, "clientip":"47.135.0.0", "request": "GET /images/hm_anime_f.gif HTTP/1.0", "status": 200, "size": 15529} +{"@timestamp": 893966017, "clientip":"47.135.0.0", "request": "GET /images/hm_day_f.gif HTTP/1.0", "status": 200, "size": 574} +{"@timestamp": 893966019, "clientip":"47.135.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966021, "clientip":"47.135.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966024, "clientip":"47.135.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966028, "clientip":"47.135.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966028, "clientip":"47.135.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966031, "clientip":"47.135.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966048, "clientip":"131.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966055, "clientip":"41.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966068, "clientip":"161.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966068, "clientip":"56.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966084, "clientip":"176.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966085, "clientip":"47.135.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966088, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966090, "clientip":"47.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 200, "size": 4573} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966092, "clientip":"180.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966096, "clientip":"47.135.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966097, "clientip":"47.135.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966097, "clientip":"47.135.0.0", "request": "GET /images/dburton.jpg HTTP/1.0", "status": 200, "size": 12009} +{"@timestamp": 893966098, "clientip":"47.135.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966100, "clientip":"47.135.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966102, "clientip":"47.135.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966102, "clientip":"47.135.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966104, "clientip":"47.135.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966104, "clientip":"47.135.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966108, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893966109, "clientip":"47.135.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1285} +{"@timestamp": 893966109, "clientip":"47.135.0.0", "request": "GET /french/images/today_new.gif HTTP/1.0", "status": 200, "size": 869} +{"@timestamp": 893966110, "clientip":"47.135.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966111, "clientip":"47.135.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966112, "clientip":"47.135.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/archives.gif HTTP/1.0", "status": 200, "size": 569} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1078} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893966115, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966135, "clientip":"184.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966178, "clientip":"4.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966210, "clientip":"183.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966218, "clientip":"249.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966220, "clientip":"235.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966221, "clientip":"47.135.0.0", "request": "GET /french/news/3004tick.htm HTTP/1.0", "status": 200, "size": 4234} +{"@timestamp": 893966223, "clientip":"47.135.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966224, "clientip":"47.135.0.0", "request": "GET /images/ps_bdr_r.gif HTTP/1.0", "status": 200, "size": 281} +{"@timestamp": 893966224, "clientip":"47.135.0.0", "request": "GET /images/ps_bdr_l.gif HTTP/1.0", "status": 200, "size": 346} +{"@timestamp": 893966232, "clientip":"248.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966259, "clientip":"2.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966273, "clientip":"218.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966280, "clientip":"48.4.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966296, "clientip":"128.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966306, "clientip":"139.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966311, "clientip":"141.78.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893966322, "clientip":"249.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966327, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966342, "clientip":"47.135.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966343, "clientip":"141.78.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_anime_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_day_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 206, "size": 838} +{"@timestamp": 893966347, "clientip":"141.78.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 206, "size": 107} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966349, "clientip":"141.78.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966350, "clientip":"141.78.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966351, "clientip":"141.78.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966351, "clientip":"141.78.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966352, "clientip":"141.78.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966352, "clientip":"141.78.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966368, "clientip":"221.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966381, "clientip":"43.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966384, "clientip":"175.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966384, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966386, "clientip":"141.78.0.0", "request": "GET /french/competition/maincomp.htm HTTP/1.0", "status": 200, "size": 3048} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /images/comp_hm_brac.gif HTTP/1.0", "status": 200, "size": 254} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/comp_bg2_hm.jpg HTTP/1.0", "status": 200, "size": 25676} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/bar.jpg HTTP/1.0", "status": 200, "size": 686} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/comp_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 2207} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966389, "clientip":"141.78.0.0", "request": "GET /french/images/comp_hm_nav.gif HTTP/1.0", "status": 200, "size": 11652} +{"@timestamp": 893966390, "clientip":"141.78.0.0", "request": "GET /images/comp_hm_archive.gif HTTP/1.0", "status": 200, "size": 1644} +{"@timestamp": 893966391, "clientip":"141.78.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /french/images/today_new.gif HTTP/1.0", "status": 200, "size": 869} +{"@timestamp": 893966394, "clientip":"141.78.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966394, "clientip":"141.78.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893966396, "clientip":"141.78.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/dburton.jpg HTTP/1.0", "status": 200, "size": 12009} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966397, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1285} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /french/images/archives.gif HTTP/1.0", "status": 200, "size": 569} +{"@timestamp": 893966400, "clientip":"141.78.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1078} +{"@timestamp": 893966402, "clientip":"0.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966403, "clientip":"2.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966404, "clientip":"2.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966404, "clientip":"1.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966404, "clientip":"3.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966404, "clientip":"4.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966404, "clientip":"5.0.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.1", "status": 200, "size": 915} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_concacaf.gif HTTP/1.0", "status": 200, "size": 764} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_afc.gif HTTP/1.0", "status": 200, "size": 475} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_caf.gif HTTP/1.0", "status": 200, "size": 473} +{"@timestamp": 893966405, "clientip":"7.0.0.0", "request": "GET /english/playing/mascot/mascot.html HTTP/1.0", "status": 200, "size": 5521} +{"@timestamp": 893966405, "clientip":"1.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 200, "size": 996} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_topr.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/posters/france38.gif HTTP/1.0", "status": 200, "size": 4649} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/38-1.jpg HTTP/1.0", "status": 200, "size": 14315} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_nant.gif HTTP/1.0", "status": 200, "size": 359} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_on.gif HTTP/1.0", "status": 200, "size": 507} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc.gif HTTP/1.0", "status": 200, "size": 82} +{"@timestamp": 893966405, "clientip":"11.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_lyon.gif HTTP/1.0", "status": 200, "size": 286} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966405, "clientip":"1.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_mars.gif HTTP/1.0", "status": 200, "size": 377} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_toul.gif HTTP/1.0", "status": 200, "size": 380} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966406, "clientip":"12.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 206, "size": 124} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bg2_hm.jpg HTTP/1.0", "status": 200, "size": 25676} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/comp_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal_mont.gif HTTP/1.0", "status": 200, "size": 316} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"2.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal_steti.gif HTTP/1.0", "status": 200, "size": 1125} +{"@timestamp": 893966406, "clientip":"1.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966406, "clientip":"5.0.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.1", "status": 200, "size": 1795} +{"@timestamp": 893966406, "clientip":"1.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/comp_hm_header.gif HTTP/1.0", "status": 200, "size": 1189} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"14.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.1", "status": 200, "size": 1504} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"15.0.0.0", "request": "GET /cgi-bin/trivia/Trivia.pl?ENG HTTP/1.0", "status": 200, "size": 6213} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966406, "clientip":"16.0.0.0", "request": "GET /images/s102336.gif HTTP/1.0", "status": 200, "size": 177} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/france98b.GIF HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/football.GIF HTTP/1.0", "status": 200, "size": 1170} +{"@timestamp": 893966406, "clientip":"18.0.0.0", "request": "GET /english/competition/headtohead78.htm HTTP/1.0", "status": 200, "size": 20126} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"12.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/infrance.GIF HTTP/1.0", "status": 200, "size": 990} +{"@timestamp": 893966406, "clientip":"19.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /images/ps_bdr_r.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /french/news/3004tick.htm HTTP/1.0", "status": 200, "size": 4234} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /images/ps_bdr_l.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966407, "clientip":"19.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966407, "clientip":"19.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966407, "clientip":"8.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"1.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/10982.gif HTTP/1.0", "status": 200, "size": 183} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/11101.gif HTTP/1.0", "status": 200, "size": 415} +{"@timestamp": 893966407, "clientip":"0.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966407, "clientip":"5.0.0.0", "request": "GET /french/images/space.gif HTTP/1.1", "status": 200, "size": 42} +{"@timestamp": 893966407, "clientip":"5.0.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.1", "status": 200, "size": 1317} +{"@timestamp": 893966407, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_stage2n.gif HTTP/1.0", "status": 200, "size": 984} +{"@timestamp": 893966407, "clientip":"2.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966407, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_groupsn_on.gif HTTP/1.0", "status": 200, "size": 963} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/bordeaux.gif HTTP/1.0", "status": 200, "size": 723} +{"@timestamp": 893966407, "clientip":"10.0.0.0", "request": "GET /images/cal_bord.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966407, "clientip":"2.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966407, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/banner.jpg HTTP/1.0", "status": 200, "size": 19967} +{"@timestamp": 893966407, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/01test.gif HTTP/1.0", "status": 200, "size": 1664} +{"@timestamp": 893966407, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_refsn.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/quizbg.gif HTTP/1.0", "status": 200, "size": 1460} +{"@timestamp": 893966408, "clientip":"20.0.0.0", "request": "GET /images/lyon.gif HTTP/1.0", "status": 200, "size": 599} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966408, "clientip":"2.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966408, "clientip":"2.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966408, "clientip":"21.0.0.0", "request": "GET /english/news/11415.htm HTTP/1.1", "status": 200, "size": 21300} +{"@timestamp": 893966408, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966408, "clientip":"17.0.0.0", "request": "GET /english/history/images/reading.GIF HTTP/1.0", "status": 200, "size": 1171} +{"@timestamp": 893966408, "clientip":"22.0.0.0", "request": "GET /english/teams/teamqualify124.htm HTTP/1.0", "status": 200, "size": 3866} +{"@timestamp": 893966408, "clientip":"23.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966408, "clientip":"24.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"24.0.0.0", "request": "GET /english/playing/images/anim/trivia_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /french/images/today_new.gif HTTP/1.1", "status": 200, "size": 869} +{"@timestamp": 893966409, "clientip":"20.0.0.0", "request": "GET /images/saintdenis.gif HTTP/1.0", "status": 200, "size": 702} +{"@timestamp": 893966409, "clientip":"24.0.0.0", "request": "GET /english/playing/images/play_hm_mascot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.1", "status": 200, "size": 1078} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /images/case5.gif HTTP/1.1", "status": 200, "size": 1362} +{"@timestamp": 893966409, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_30_off.gif HTTP/1.0", "status": 200, "size": 406} +{"@timestamp": 893966409, "clientip":"20.0.0.0", "request": "GET /images/toulouse.gif HTTP/1.0", "status": 200, "size": 704} +{"@timestamp": 893966409, "clientip":"25.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.1", "status": 200, "size": 2557} +{"@timestamp": 893966409, "clientip":"26.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966409, "clientip":"27.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966409, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_refsn.gif HTTP/1.0", "status": 200, "size": 933} +{"@timestamp": 893966409, "clientip":"28.0.0.0", "request": "GET /english/individuals/player13893.htm HTTP/1.0", "status": 200, "size": 6472} +{"@timestamp": 893966409, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_calendar.gif HTTP/1.0", "status": 200, "size": 1197} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966410, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/02how.gif HTTP/1.0", "status": 200, "size": 754} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966410, "clientip":"29.0.0.0", "request": "GET /images/10511.jpg HTTP/1.0", "status": 200, "size": 15543} +{"@timestamp": 893966410, "clientip":"20.0.0.0", "request": "GET /images/11289.jpg HTTP/1.0", "status": 200, "size": 6444} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966410, "clientip":"20.0.0.0", "request": "GET /images/11288.jpg HTTP/1.0", "status": 200, "size": 5874} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966410, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966410, "clientip":"5.0.0.0", "request": "GET /images/dburton.jpg HTTP/1.1", "status": 200, "size": 12009} +{"@timestamp": 893966410, "clientip":"2.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966410, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/04luck.gif HTTP/1.0", "status": 200, "size": 744} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966410, "clientip":"2.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966410, "clientip":"5.0.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.1", "status": 200, "size": 333} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966411, "clientip":"30.0.0.0", "request": "GET /english/playing/download/download.html HTTP/1.0", "status": 200, "size": 13898} +{"@timestamp": 893966411, "clientip":"2.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966411, "clientip":"16.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 200, "size": 142} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966411, "clientip":"12.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"5.0.0.0", "request": "GET /images/bord_stories.gif HTTP/1.1", "status": 200, "size": 520} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"31.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.1", "status": 200, "size": 934} +{"@timestamp": 893966411, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/03score.gif HTTP/1.0", "status": 200, "size": 572} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"31.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.1", "status": 200, "size": 914} +{"@timestamp": 893966411, "clientip":"32.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.1", "status": 200, "size": 60349} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966412, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/submit.gif HTTP/1.0", "status": 200, "size": 1808} +{"@timestamp": 893966412, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_header.gif HTTP/1.0", "status": 200, "size": 1989} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966412, "clientip":"5.0.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.1", "status": 200, "size": 1285} +{"@timestamp": 893966412, "clientip":"19.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966412, "clientip":"25.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.1", "status": 200, "size": 2861} +{"@timestamp": 893966412, "clientip":"19.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966412, "clientip":"33.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966413, "clientip":"25.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/images/thecup.GIF HTTP/1.0", "status": 200, "size": 1293} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966413, "clientip":"12.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966413, "clientip":"16.0.0.0", "request": "GET /images/s102338.gif HTTP/1.0", "status": 200, "size": 138} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_off.gif HTTP/1.0", "status": 200, "size": 436} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966413, "clientip":"20.0.0.0", "request": "GET /images/11103.gif HTTP/1.0", "status": 200, "size": 513} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"34.0.0.0", "request": "GET /english/playing/body.html HTTP/1.0", "status": 200, "size": 5033} +{"@timestamp": 893966413, "clientip":"19.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966414, "clientip":"35.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966414, "clientip":"36.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966414, "clientip":"20.0.0.0", "request": "GET /images/marseille.gif HTTP/1.0", "status": 200, "size": 722} +{"@timestamp": 893966414, "clientip":"5.0.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.1", "status": 200, "size": 1056} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"24.0.0.0", "request": "GET /cgi-bin/trivia/Trivia.pl?ENG HTTP/1.0", "status": 200, "size": 6228} +{"@timestamp": 893966414, "clientip":"5.0.0.0", "request": "GET /images/ligneb01.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966414, "clientip":"12.0.0.0", "request": "GET /english/tickets/body.html HTTP/1.0", "status": 200, "size": 2925} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_50_on.gif HTTP/1.0", "status": 200, "size": 455} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966414, "clientip":"3.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966414, "clientip":"37.0.0.0", "request": "GET /french/competition/schedule.htm HTTP/1.0", "status": 200, "size": 49256} +{"@timestamp": 893966414, "clientip":"3.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966414, "clientip":"26.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"232.41.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8749} +{"@timestamp": 893966415, "clientip":"232.41.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966415, "clientip":"19.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966415, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_58_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_bg.jpg HTTP/1.0", "status": 200, "size": 20929} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 200, "size": 1226} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 200, "size": 11253} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /images/arw_red_lk.gif HTTP/1.0", "status": 200, "size": 1068} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /english/images/news_hm_header.gif HTTP/1.1", "status": 200, "size": 527} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /images/bg_generic.jpg HTTP/1.1", "status": 200, "size": 21127} +{"@timestamp": 893966415, "clientip":"38.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /english/images/space.gif HTTP/1.1", "status": 200, "size": 42} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966415, "clientip":"34.0.0.0", "request": "GET /english/playing/images/play_hm_bg_opt1.jpg HTTP/1.0", "status": 200, "size": 44502} +{"@timestamp": 893966415, "clientip":"5.0.0.0", "request": "GET /images/ligne.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966415, "clientip":"3.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 200, "size": 669} +{"@timestamp": 893966415, "clientip":"19.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966415, "clientip":"35.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966416, "clientip":"5.0.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.1", "status": 200, "size": 965} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966416, "clientip":"19.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_62_off.gif HTTP/1.0", "status": 200, "size": 437} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_66_off.gif HTTP/1.0", "status": 200, "size": 435} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_70_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893966416, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.1", "status": 200, "size": 688} +{"@timestamp": 893966416, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_posters.jpg HTTP/1.1", "status": 200, "size": 33296} +{"@timestamp": 893966416, "clientip":"201.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966417, "clientip":"25.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.1", "status": 200, "size": 1927} +{"@timestamp": 893966417, "clientip":"23.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966417, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_74_off.gif HTTP/1.0", "status": 200, "size": 423} +{"@timestamp": 893966417, "clientip":"1.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966417, "clientip":"5.0.0.0", "request": "GET /images/ligneb.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966417, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_78_off.gif HTTP/1.0", "status": 200, "size": 427} +{"@timestamp": 893966417, "clientip":"16.0.0.0", "request": "GET /english/images/team_group_header_d.gif HTTP/1.0", "status": 200, "size": 646} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_82_off.gif HTTP/1.0", "status": 200, "size": 433} +{"@timestamp": 893966418, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/italy34.html HTTP/1.1", "status": 200, "size": 13022} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_86_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966418, "clientip":"1.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966418, "clientip":"37.0.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966418, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_groupsn_on.gif HTTP/1.0", "status": 200, "size": 963} +{"@timestamp": 893966418, "clientip":"22.0.0.0", "request": "GET /english/images/team_bu_detail2.gif HTTP/1.0", "status": 200, "size": 1438} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_90_off.gif HTTP/1.0", "status": 200, "size": 429} +{"@timestamp": 893966418, "clientip":"38.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/nantes.gif HTTP/1.0", "status": 200, "size": 677} +{"@timestamp": 893966419, "clientip":"16.0.0.0", "request": "GET /images/s102326.gif HTTP/1.0", "status": 200, "size": 248} +{"@timestamp": 893966419, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_94_off.gif HTTP/1.0", "status": 200, "size": 432} +{"@timestamp": 893966419, "clientip":"1.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966419, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_top.gif HTTP/1.0", "status": 200, "size": 289} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/saintetienne.gif HTTP/1.0", "status": 200, "size": 761} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/11287.jpg HTTP/1.0", "status": 200, "size": 6431} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/posters/brasil50.gif HTTP/1.0", "status": 200, "size": 5003} +{"@timestamp": 893966420, "clientip":"20.0.0.0", "request": "GET /images/lens.gif HTTP/1.0", "status": 200, "size": 582} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_bot.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_trophy.gif HTTP/1.0", "status": 200, "size": 800} +{"@timestamp": 893966420, "clientip":"41.0.0.0", "request": "GET /images/col.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966421, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/50-1.jpg HTTP/1.0", "status": 200, "size": 15314} +{"@timestamp": 893966421, "clientip":"1.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966421, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_30_off.gif HTTP/1.1", "status": 200, "size": 406} +{"@timestamp": 893966421, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_on.gif HTTP/1.1", "status": 200, "size": 474} +{"@timestamp": 893966421, "clientip":"16.0.0.0", "request": "GET /images/s102477.gif HTTP/1.0", "status": 200, "size": 214} +{"@timestamp": 893966421, "clientip":"26.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966421, "clientip":"26.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966421, "clientip":"8.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966421, "clientip":"38.0.0.0", "request": "GET /images/s102325.gif HTTP/1.0", "status": 200, "size": 187} +{"@timestamp": 893966422, "clientip":"15.0.0.0", "request": "GET /english/playing/links.html HTTP/1.0", "status": 200, "size": 18694} +{"@timestamp": 893966422, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/34-1.jpg HTTP/1.1", "status": 200, "size": 14275} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /images/teams_hm_bg.jpg HTTP/1.0", "status": 200, "size": 18794} +{"@timestamp": 893966422, "clientip":"42.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966422, "clientip":"25.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.1", "status": 200, "size": 2491} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 200, "size": 639} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/teams_bu_confed_off.gif HTTP/1.0", "status": 200, "size": 1105} +{"@timestamp": 893966422, "clientip":"43.0.0.0", "request": "GET /english/individuals/player111722.htm HTTP/1.0", "status": 200, "size": 6523} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/montpellier.gif HTTP/1.0", "status": 200, "size": 728} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/out_france.html HTTP/1.0", "status": 200, "size": 6143} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/11105.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_quest_bg2.jpg HTTP/1.0", "status": 200, "size": 11324} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_header.gif HTTP/1.0", "status": 200, "size": 453} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 1647} +{"@timestamp": 893966423, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_press_off.gif HTTP/1.1", "status": 200, "size": 847} +{"@timestamp": 893966423, "clientip":"44.0.0.0", "request": "GET /english/teams/teambio169.htm HTTP/1.1", "status": 200, "size": 11157} +{"@timestamp": 893966423, "clientip":"38.0.0.0", "request": "GET /images/s102327.gif HTTP/1.0", "status": 200, "size": 97} +{"@timestamp": 893966423, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_letter_off.gif HTTP/1.1", "status": 200, "size": 940} +{"@timestamp": 893966423, "clientip":"45.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.1", "status": 200, "size": 12800} +{"@timestamp": 893966423, "clientip":"32.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.1", "status": 200, "size": 2672} +{"@timestamp": 893966423, "clientip":"15.0.0.0", "request": "GET /english/playing/images/backg.gif HTTP/1.0", "status": 200, "size": 1462} +{"@timestamp": 893966423, "clientip":"15.0.0.0", "request": "GET /english/playing/images/play_header.gif HTTP/1.0", "status": 200, "size": 2262} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/paris_off.gif HTTP/1.0", "status": 200, "size": 658} +{"@timestamp": 893966423, "clientip":"8.0.0.0", "request": "GET /images/teams_hm_bracket.gif HTTP/1.0", "status": 200, "size": 655} +{"@timestamp": 893966424, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 200, "size": 670} +{"@timestamp": 893966424, "clientip":"29.0.0.0", "request": "GET /images/32t49811.jpg HTTP/1.0", "status": 200, "size": 4753} +{"@timestamp": 893966424, "clientip":"37.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 200, "size": 284} +{"@timestamp": 893966424, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/posters/italy34.gif HTTP/1.1", "status": 200, "size": 4474} +{"@timestamp": 893966424, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_quest2.gif HTTP/1.0", "status": 200, "size": 1201} +{"@timestamp": 893966424, "clientip":"38.0.0.0", "request": "GET /images/s102424.gif HTTP/1.0", "status": 200, "size": 164} +{"@timestamp": 893966424, "clientip":"20.0.0.0", "request": "GET /images/11102.gif HTTP/1.0", "status": 200, "size": 417} +{"@timestamp": 893966424, "clientip":"15.0.0.0", "request": "GET /english/playing/images/france98b.GIF HTTP/1.0", "status": 200, "size": 597} +{"@timestamp": 893966424, "clientip":"39.0.0.0", "request": "GET /english/history/images/france98b.GIF HTTP/1.1", "status": 200, "size": 915} +{"@timestamp": 893966424, "clientip":"39.0.0.0", "request": "GET /english/history/images/football.GIF HTTP/1.1", "status": 200, "size": 1170} +{"@timestamp": 893966424, "clientip":"46.0.0.0", "request": "GET /english/venues/venues/saint-denis.html HTTP/1.0", "status": 200, "size": 22760} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_abroad2_on.gif HTTP/1.0", "status": 200, "size": 2020} +{"@timestamp": 893966425, "clientip":"47.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_abroad_header.gif HTTP/1.0", "status": 200, "size": 3606} +{"@timestamp": 893966425, "clientip":"15.0.0.0", "request": "GET /english/playing/images/mascots.GIF HTTP/1.0", "status": 200, "size": 1275} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/10512.jpg HTTP/1.1", "status": 200, "size": 17227} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/32t49809.jpg HTTP/1.1", "status": 200, "size": 5623} +{"@timestamp": 893966425, "clientip":"32.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.1", "status": 200, "size": 8389} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_infrance2.gif HTTP/1.0", "status": 200, "size": 1134} +{"@timestamp": 893966425, "clientip":"43.0.0.0", "request": "GET /images/32p49815.jpg HTTP/1.0", "status": 200, "size": 11735} +{"@timestamp": 893966425, "clientip":"43.0.0.0", "request": "GET /english/images/team_bu_detail_off.gif HTTP/1.0", "status": 200, "size": 918} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/esp.gif HTTP/1.1", "status": 200, "size": 1892} +{"@timestamp": 893966425, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 200, "size": 639} +{"@timestamp": 893966425, "clientip":"48.0.0.0", "request": "GET /english/help/site.html HTTP/1.0", "status": 200, "size": 7697} +{"@timestamp": 893966425, "clientip":"15.0.0.0", "request": "GET /english/playing/images/downloads.GIF HTTP/1.0", "status": 200, "size": 1294} +{"@timestamp": 893966426, "clientip":"37.0.0.0", "request": "GET /french/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1547} +{"@timestamp": 893966426, "clientip":"25.0.0.0", "request": "GET /french/index.html HTTP/1.1", "status": 200, "size": 954} +{"@timestamp": 893966426, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_kits_off.gif HTTP/1.1", "status": 200, "size": 1027} +{"@timestamp": 893966426, "clientip":"15.0.0.0", "request": "GET /english/playing/images/links_on.GIF HTTP/1.0", "status": 200, "size": 1393} +{"@timestamp": 893966426, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia.GIF HTTP/1.0", "status": 200, "size": 999} +{"@timestamp": 893966426, "clientip":"0.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966426, "clientip":"32.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.1", "status": 200, "size": 374} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /english/venues/cities/images/denis/venue_denn_bg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"8.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 1379} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /images/11295.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /images/11294.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_b.gif HTTP/1.0", "status": 200, "size": 665} +{"@timestamp": 893966427, "clientip":"38.0.0.0", "request": "GET /images/s102330.gif HTTP/1.0", "status": 200, "size": 259} +{"@timestamp": 893966427, "clientip":"15.0.0.0", "request": "GET /english/playing/images/banner2.gif HTTP/1.0", "status": 200, "size": 15328} +{"@timestamp": 893966427, "clientip":"49.0.0.0", "request": "GET /english/news/2704nevi.htm HTTP/1.0", "status": 200, "size": 3132} +{"@timestamp": 893966427, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_b.gif HTTP/1.0", "status": 200, "size": 665} +{"@timestamp": 893966427, "clientip":"15.0.0.0", "request": "GET /english/playing/images/fifa_logo_sm.gif HTTP/1.0", "status": 200, "size": 2900} +{"@timestamp": 893966427, "clientip":"39.0.0.0", "request": "GET /english/history/images/infrance.GIF HTTP/1.1", "status": 200, "size": 990} +{"@timestamp": 893966427, "clientip":"47.135.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893966428, "clientip":"50.0.0.0", "request": "GET /english/playing/download/images/big.bird.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966428, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_e.gif HTTP/1.0", "status": 200, "size": 643} +{"@timestamp": 893966429, "clientip":"41.0.0.0", "request": "GET /english/images/team_bu_detail_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966429, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/misc.gif HTTP/1.0", "status": 200, "size": 2997} +{"@timestamp": 893966429, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_g.gif HTTP/1.0", "status": 200, "size": 641} +{"@timestamp": 893966429, "clientip":"38.0.0.0", "request": "GET /images/s102382.gif HTTP/1.0", "status": 200, "size": 102} +{"@timestamp": 893966429, "clientip":"12.0.0.0", "request": "GET /english/tickets/10484.htm HTTP/1.0", "status": 200, "size": 16722} +{"@timestamp": 893966429, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_h.gif HTTP/1.0", "status": 200, "size": 628} +{"@timestamp": 893966429, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_abroad2.gif HTTP/1.0", "status": 200, "size": 1486} +{"@timestamp": 893966430, "clientip":"12.0.0.0", "request": "GET /images/11103.gif HTTP/1.0", "status": 200, "size": 513} +{"@timestamp": 893966430, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_d.gif HTTP/1.0", "status": 200, "size": 646} +{"@timestamp": 893966430, "clientip":"5.0.0.0", "request": "GET /french/images/archives.gif HTTP/1.1", "status": 200, "size": 569} +{"@timestamp": 893966430, "clientip":"39.0.0.0", "request": "GET /english/history/images/reading.GIF HTTP/1.1", "status": 200, "size": 1171} +{"@timestamp": 893966430, "clientip":"32.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.1", "status": 200, "size": 6507} +{"@timestamp": 893966430, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966430, "clientip":"26.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966430, "clientip":"47.135.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893966430, "clientip":"47.135.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893966431, "clientip":"12.0.0.0", "request": "GET /images/10982.gif HTTP/1.0", "status": 200, "size": 183} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 200, "size": 142} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /images/s102328.gif HTTP/1.0", "status": 200, "size": 236} +{"@timestamp": 893966431, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_header.gif HTTP/1.1", "status": 200, "size": 1989} +{"@timestamp": 893966431, "clientip":"51.0.0.0", "request": "GET /english/teams/teambio160.htm HTTP/1.1", "status": 200, "size": 11097} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966431, "clientip":"52.0.0.0", "request": "GET /english/member/body.html HTTP/1.1", "status": 200, "size": 5097} +{"@timestamp": 893966432, "clientip":"10.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966432, "clientip":"53.0.0.0", "request": "GET /english/competition/headtohead75.htm HTTP/1.1", "status": 200, "size": 5152} +{"@timestamp": 893966432, "clientip":"25.0.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.1", "status": 200, "size": 374} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966432, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.1", "status": 200, "size": 688} +{"@timestamp": 893966432, "clientip":"47.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966432, "clientip":"37.0.0.0", "request": "GET /images/cal_nant.gif HTTP/1.0", "status": 200, "size": 359} +{"@timestamp": 893966432, "clientip":"37.0.0.0", "request": "GET /images/cal_stdenis.gif HTTP/1.0", "status": 200, "size": 402} +{"@timestamp": 893966432, "clientip":"41.0.0.0", "request": "GET /images/32t49807.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966433, "clientip":"32.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.1", "status": 200, "size": 853} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102320.gif HTTP/1.0", "status": 200, "size": 259} +{"@timestamp": 893966433, "clientip":"42.0.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 200, "size": 4573} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102438.gif HTTP/1.0", "status": 200, "size": 297} +{"@timestamp": 893966433, "clientip":"54.0.0.0", "request": "GET /english/playing/download/images/big.bird.gif HTTP/1.0", "status": 200, "size": 4870} +{"@timestamp": 893966433, "clientip":"0.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102329.gif HTTP/1.0", "status": 200, "size": 159} +{"@timestamp": 893966433, "clientip":"10.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966434, "clientip":"26.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966434, "clientip":"0.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966434, "clientip":"41.0.0.0", "request": "GET /english/images/team_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966434, "clientip":"32.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.1", "status": 200, "size": 776} +{"@timestamp": 893966434, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 200, "size": 670} +{"@timestamp": 893966434, "clientip":"52.0.0.0", "request": "GET /english/member/images/submit.gif HTTP/1.1", "status": 200, "size": 447} +{"@timestamp": 893966434, "clientip":"12.0.0.0", "request": "GET /images/10981.gif HTTP/1.0", "status": 200, "size": 173} +{"@timestamp": 893966434, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_off.gif HTTP/1.1", "status": 200, "size": 397} +{"@timestamp": 893966434, "clientip":"38.0.0.0", "request": "GET /images/s102377.gif HTTP/1.0", "status": 200, "size": 173} +{"@timestamp": 893966434, "clientip":"12.0.0.0", "request": "GET /images/11116.gif HTTP/1.0", "status": 200, "size": 667} +{"@timestamp": 893966435, "clientip":"26.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11106.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11105.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11102.gif HTTP/1.0", "status": 200, "size": 417} +{"@timestamp": 893966435, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_off.gif HTTP/1.1", "status": 200, "size": 436} +{"@timestamp": 893966435, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_h.gif HTTP/1.0", "status": 200, "size": 628} +{"@timestamp": 893966435, "clientip":"25.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.1", "status": 200, "size": 929} +{"@timestamp": 893966435, "clientip":"52.0.0.0", "request": "GET / HTTP/1.1", "status": 200, "size": 8712} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11101.gif HTTP/1.0", "status": 200, "size": 415} +{"@timestamp": 893966435, "clientip":"32.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.1", "status": 200, "size": 870} +{"@timestamp": 893966436, "clientip":"55.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966436, "clientip":"56.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966436, "clientip":"38.0.0.0", "request": "GET /images/s102443.gif HTTP/1.0", "status": 200, "size": 165} +{"@timestamp": 893966436, "clientip":"46.0.0.0", "request": "GET /images/paris.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966436, "clientip":"38.0.0.0", "request": "GET /images/s140875.gif HTTP/1.0", "status": 200, "size": 184} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_bord.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_steti.gif HTTP/1.0", "status": 200, "size": 1125} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_mont.gif HTTP/1.0", "status": 200, "size": 316} +{"@timestamp": 893966436, "clientip":"51.0.0.0", "request": "GET /images/32t49813.jpg HTTP/1.1", "status": 200, "size": 4714} +{"@timestamp": 893966436, "clientip":"2.0.0.0", "request": "GET /english/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/body.html HTTP/1.0", "status": 200, "size": 2925} +{"@timestamp": 893966437, "clientip":"52.0.0.0", "request": "GET /english/member/images/cfologo.gif HTTP/1.1", "status": 200, "size": 2820} +{"@timestamp": 893966437, "clientip":"52.0.0.0", "request": "GET /english/member/images/member_header.jpg HTTP/1.1", "status": 200, "size": 10457} +{"@timestamp": 893966437, "clientip":"32.0.0.0", "request": "GET /english/news/3004bres.htm HTTP/1.1", "status": 200, "size": 5765} +{"@timestamp": 893966437, "clientip":"38.0.0.0", "request": "GET /images/s102386.gif HTTP/1.0", "status": 200, "size": 279} +{"@timestamp": 893966437, "clientip":"41.0.0.0", "request": "GET /images/news_hm_arw.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966437, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_54_off.gif HTTP/1.1", "status": 200, "size": 403} +{"@timestamp": 893966437, "clientip":"38.0.0.0", "request": "GET /images/s102326.gif HTTP/1.0", "status": 200, "size": 248} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_bg.jpg HTTP/1.0", "status": 200, "size": 20929} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 200, "size": 1226} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 200, "size": 11253} +{"@timestamp": 893966438, "clientip":"33.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.1", "status": 200, "size": 327} +{"@timestamp": 893966438, "clientip":"38.0.0.0", "request": "GET /english/competition/maincomp.htm HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"46.0.0.0", "request": "GET /images/11291.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"47.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_58_off.gif HTTP/1.1", "status": 200, "size": 397} +{"@timestamp": 893966439, "clientip":"25.0.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.1", "status": 200, "size": 855} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966439, "clientip":"47.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966439, "clientip":"58.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /english/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1210} +{"@timestamp": 893966440, "clientip":"47.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966440, "clientip":"47.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966440, "clientip":"38.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 1379} +{"@timestamp": 893966440, "clientip":"38.0.0.0", "request": "GET /english/images/comp_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"25.0.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.1", "status": 200, "size": 945} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"59.0.0.0", "request": "GET /english/news/2704leag.htm HTTP/1.0", "status": 200, "size": 4801} +{"@timestamp": 893966441, "clientip":"60.0.0.0", "request": "GET /images/10531.jpg HTTP/1.0", "status": 200, "size": 16554} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"61.0.0.0", "request": "GET /english/venues/cities/montpellier.html HTTP/1.0", "status": 200, "size": 9686} +{"@timestamp": 893966442, "clientip":"62.0.0.0", "request": "GET /english/playing/links.html HTTP/1.0", "status": 200, "size": 18694} +{"@timestamp": 893966442, "clientip":"23.0.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 200, "size": 669} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"46.0.0.0", "request": "GET /images/sdffr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_62_off.gif HTTP/1.1", "status": 200, "size": 437} +{"@timestamp": 893966442, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_66_off.gif HTTP/1.1", "status": 200, "size": 435} +{"@timestamp": 893966443, "clientip":"64.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966443, "clientip":"38.0.0.0", "request": "GET /english/images/comp_hm_nav.gif HTTP/1.0", "status": 206, "size": 10902} +{"@timestamp": 893966443, "clientip":"25.0.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.1", "status": 200, "size": 965} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966443, "clientip":"65.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966443, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/button.03.gif HTTP/1.0", "status": 200, "size": 893} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966444, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_70_off.gif HTTP/1.1", "status": 200, "size": 409} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966444, "clientip":"6.0.0.0", "request": "GET /french/venues/body.html HTTP/1.0", "status": 200, "size": 2042} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966444, "clientip":"64.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966444, "clientip":"66.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966444, "clientip":"62.0.0.0", "request": "GET /english/playing/images/backg.gif HTTP/1.0", "status": 200, "size": 1462} +{"@timestamp": 893966444, "clientip":"67.0.0.0", "request": "GET /english/individuals/player389.htm HTTP/1.0", "status": 200, "size": 7001} +{"@timestamp": 893966444, "clientip":"47.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966445, "clientip":"7.0.0.0", "request": "GET /english/playing/images/play_header.gif HTTP/1.0", "status": 200, "size": 2262} +{"@timestamp": 893966445, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/footix.test.gif HTTP/1.0", "status": 200, "size": 20385} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_mid_off.gif HTTP/1.0", "status": 200, "size": 9911} +{"@timestamp": 893966445, "clientip":"25.0.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.1", "status": 200, "size": 982} +{"@timestamp": 893966445, "clientip":"64.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_top_off.gif HTTP/1.0", "status": 200, "size": 8369} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_hm_nav.gif HTTP/1.0", "status": 200, "size": 7644} +{"@timestamp": 893966445, "clientip":"68.0.0.0", "request": "GET /english/venues/cities/images/montpellier/venue_mont_header.jpg HTTP/1.0", "status": 200, "size": 11562} +{"@timestamp": 893966445, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/cup.gif HTTP/1.0", "status": 200, "size": 5469} +{"@timestamp": 893966445, "clientip":"34.0.0.0", "request": "GET /english/history/body.html HTTP/1.0", "status": 200, "size": 2909} +{"@timestamp": 893966445, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_c.gif HTTP/1.0", "status": 200, "size": 369} +{"@timestamp": 893966446, "clientip":"64.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966446, "clientip":"66.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966446, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_bot_off.gif HTTP/1.0", "status": 200, "size": 7397} +{"@timestamp": 893966446, "clientip":"71.0.0.0", "request": "GET /english/teams/teambio148.htm HTTP/1.0", "status": 200, "size": 10857} +{"@timestamp": 893966446, "clientip":"72.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966446, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_78_off.gif HTTP/1.1", "status": 200, "size": 427} +{"@timestamp": 893966446, "clientip":"67.0.0.0", "request": "GET /images/102373.gif HTTP/1.0", "status": 200, "size": 1703} +{"@timestamp": 893966446, "clientip":"47.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966446, "clientip":"67.0.0.0", "request": "GET /images/32p49804.jpg HTTP/1.0", "status": 200, "size": 11023} +{"@timestamp": 893966446, "clientip":"47.0.0.0", "request": "GET /english/images/teams_bu_confed_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"71.0.0.0", "request": "GET /images/32t49812.jpg HTTP/1.0", "status": 200, "size": 4132} +{"@timestamp": 893966447, "clientip":"71.0.0.0", "request": "GET /images/ger.gif HTTP/1.0", "status": 200, "size": 1603} +{"@timestamp": 893966447, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_82_off.gif HTTP/1.1", "status": 200, "size": 433} +{"@timestamp": 893966447, "clientip":"25.0.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.1", "status": 200, "size": 976} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966447, "clientip":"47.0.0.0", "request": "GET /images/s102325.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966447, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102424.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_86_off.gif HTTP/1.1", "status": 200, "size": 409} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102487.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966448, "clientip":"73.0.0.0", "request": "GET / HTTP/1.1", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"62.0.0.0", "request": "GET /english/playing/images/links_on.GIF HTTP/1.0", "status": 200, "size": 1393} +{"@timestamp": 893966448, "clientip":"62.0.0.0", "request": "GET /english/playing/images/banner2.gif HTTP/1.0", "status": 200, "size": 15328} +{"@timestamp": 893966448, "clientip":"66.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966448, "clientip":"57.0.0.0", "request": "GET /english/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1431} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102477.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_e.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966449, "clientip":"74.0.0.0", "request": "GET / HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"75.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966449, "clientip":"47.0.0.0", "request": "GET /images/s102324.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"25.0.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.1", "status": 200, "size": 413} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966449, "clientip":"14.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.1", "status": 200, "size": 2557} +{"@timestamp": 893966449, "clientip":"47.0.0.0", "request": "GET /images/s102376.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/france38.html HTTP/1.1", "status": 200, "size": 12669} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.1", "status": 200, "size": 2843} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.1", "status": 200, "size": 60349} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966449, "clientip":"76.0.0.0", "request": "GET /english/competition/stage1.htm HTTP/1.1", "status": 200, "size": 36783} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966449, "clientip":"53.0.0.0", "request": "GET /english/competition/headtohead76.htm HTTP/1.1", "status": 200, "size": 16909} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966449, "clientip":"2.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.1", "status": 200, "size": 2861} +{"@timestamp": 893966450, "clientip":"2.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966450, "clientip":"66.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966450, "clientip":"66.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966450, "clientip":"77.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966450, "clientip":"77.0.0.0", "request": "GET /english/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966450, "clientip":"57.0.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966450, "clientip":"47.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966450, "clientip":"47.0.0.0", "request": "GET /images/s102443.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966450, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966451, "clientip":"66.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 200, "size": 996} +{"@timestamp": 893966451, "clientip":"57.0.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966451, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_94_off.gif HTTP/1.1", "status": 200, "size": 432} +{"@timestamp": 893966451, "clientip":"77.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s140875.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s102321.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/venue_denn_bg.jpg HTTP/1.0", "status": 200, "size": 21003} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_header.gif HTTP/1.0", "status": 200, "size": 740} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_topr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"25.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.1", "status": 200, "size": 33665} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_bu_city_on.gif HTTP/1.0", "status": 200, "size": 1061} +{"@timestamp": 893966451, "clientip":"66.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_g.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s102357.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966451, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/coup_du_mondtxt.gif HTTP/1.0", "status": 200, "size": 5053} +{"@timestamp": 893966451, "clientip":"73.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"78.0.0.0", "request": "GET /english/venues/cities/images/montpellier/12eglise.jpg HTTP/1.0", "status": 200, "size": 3035} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /images/s102383.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/12maisons.jpg HTTP/1.0", "status": 200, "size": 2559} +{"@timestamp": 893966452, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/denis_a.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966452, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/montpellier/17mystery.gif HTTP/1.0", "status": 200, "size": 542} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_bg2.jpg HTTP/1.0", "status": 200, "size": 22515} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.0", "status": 200, "size": 688} +{"@timestamp": 893966452, "clientip":"2.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966452, "clientip":"1.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966452, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/montpellier/18corum.gif HTTP/1.0", "status": 200, "size": 650} +{"@timestamp": 893966452, "clientip":"66.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966452, "clientip":"80.0.0.0", "request": "GET /english/news/player01.htm HTTP/1.0", "status": 200, "size": 42591} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_3094.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_posters.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_top.gif HTTP/1.1", "status": 200, "size": 289} +{"@timestamp": 893966452, "clientip":"2.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /images/s102380.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/16tradition.gif HTTP/1.0", "status": 200, "size": 387} +{"@timestamp": 893966452, "clientip":"66.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966453, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_t.gif HTTP/1.0", "status": 200, "size": 353} +{"@timestamp": 893966453, "clientip":"73.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.1", "status": 200, "size": 327} +{"@timestamp": 893966453, "clientip":"81.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966453, "clientip":"5.0.0.0", "request": "GET /french/news/3004arge.htm HTTP/1.1", "status": 200, "size": 4869} +{"@timestamp": 893966453, "clientip":"78.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_i.gif HTTP/1.0", "status": 200, "size": 314} +{"@timestamp": 893966453, "clientip":"1.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966453, "clientip":"77.0.0.0", "request": "GET /english/images/comp_bu_stage2n.gif HTTP/1.0", "status": 200, "size": 984} +{"@timestamp": 893966453, "clientip":"2.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966453, "clientip":"62.0.0.0", "request": "GET /english/playing/images/fifa_logo_sm.gif HTTP/1.0", "status": 200, "size": 2900} +{"@timestamp": 893966453, "clientip":"68.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_a.gif HTTP/1.0", "status": 200, "size": 434} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102320.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 568} +{"@timestamp": 893966453, "clientip":"1.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 852} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102353.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/17commercial.gif HTTP/1.0", "status": 200, "size": 614} +{"@timestamp": 893966453, "clientip":"57.0.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102329.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 933} +{"@timestamp": 893966453, "clientip":"57.0.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966453, "clientip":"7.0.0.0", "request": "GET /english/playing/images/links.GIF HTTP/1.0", "status": 200, "size": 1394} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102377.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102328.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"66.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102323.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"34.0.0.0", "request": "GET /english/history/images/history_hm_3094.gif HTTP/1.0", "status": 200, "size": 1031} +{"@timestamp": 893966454, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/trophytxt.gif HTTP/1.0", "status": 200, "size": 1425} +{"@timestamp": 893966454, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/19historical.gif HTTP/1.0", "status": 200, "size": 616} +{"@timestamp": 893966454, "clientip":"57.0.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966454, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/18collect.gif HTTP/1.0", "status": 200, "size": 714} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102442.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966455, "clientip":"34.0.0.0", "request": "GET /english/history/images/history_hm_nav.gif HTTP/1.0", "status": 200, "size": 18328} diff --git a/regression-test/data/load_p0/routine_load/test_routine_load_with_udf.out b/regression-test/data/load_p0/routine_load/test_routine_load_with_udf.out new file mode 100644 index 00000000000000..027890b8fe8df0 --- /dev/null +++ b/regression-test/data/load_p0/routine_load/test_routine_load_with_udf.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_topic_udf -- +1 eab 2023-07-15 def 2023-07-20T05:48:31 defdoris udf load + diff --git a/regression-test/data/load_p0/stream_load/test_stream_load_udf.csv b/regression-test/data/load_p0/stream_load/test_stream_load_udf.csv new file mode 100644 index 00000000000000..619df81596dddb --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_stream_load_udf.csv @@ -0,0 +1,3 @@ +1,asd +2,xxx +3,\N diff --git a/regression-test/data/load_p0/stream_load/test_stream_load_with_udf.out b/regression-test/data/load_p0/stream_load/test_stream_load_with_udf.out new file mode 100644 index 00000000000000..898c2cb4a9e4ce --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_stream_load_with_udf.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1001 asd +1002 xxx +1003 \N + diff --git a/regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv b/regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv new file mode 100644 index 00000000000000..21344d61bcb3ff --- /dev/null +++ b/regression-test/data/nereids_function_p0/fn_test_ip_invalid.csv @@ -0,0 +1,31 @@ +1;256.0.0.1;2001:db8::12345;'256.0.0.1';'2001:db8::12345' +2;192.168.1.999;2001:db8:::1;'192.168.1.999';'2001:db8:::1' +3;300.300.300.300;::fffff:0:0;'300.300.300.300';'::fffff:0:0' +4;192.168.1.1.1;2001:db8::g123;'192.168.1.1.1';'2001:db8::g123' +5;1234.56.78.90;2001:db8::zzz;'1234.56.78.90';'2001:db8::zzz' +6;192.168.1.;2001:db8:85a3::8a2e:0370:;'192.168.1.';'2001:db8:85a3::8a2e:0370:' +7;192.168.1.256;2001:0db8:85a3:0000:0000:8a2e:0370:7334:1234;'192.168.1.256';'2001:0db8:85a3:0000:0000:8a2e:0370:7334:1234' +8;192.168.1.-1;2001:db8:85a3:8a2e::7334;'192.168.1.-1';'2001:db8:85a3:8a2e::7334' +9;192.168.1.256;2001:db8:85a3:0::12345;'192.168.1.256';'2001:db8:85a3:0::12345' +10;abc.def.ghi.jkl;::g1;'abc.def.ghi.jkl';'::g1' +11;10.0.0.0.0;2001:db8::85a3::7334;'10.0.0.0.0';'2001:db8::85a3::7334' +12;192.168.1.1.1.1;2001:db8:85a3:0000:0000:8a2e:0370:7334:;'192.168.1.1.1.1';'2001:db8:85a3:0000:0000:8a2e:0370:7334:' +13;123.456.78.90;2001:db8:85a3:0000:0000:8a2e::gg;'123.456.78.90';'2001:db8:85a3:0000:0000:8a2e::gg' +14;192.168.1.1a;1234:5678:9abc:def::567;'192.168.1.1a';'1234:5678:9abc:def::567' +15;192.168.1.;fe80:1234::85a3::7334;'192.168.1.';'fe80:1234::85a3::7334' +16;192.168.1.1;::12345:abcd;'192.168.1.1';'::12345:abcd' +17;10.0.0.1;2001:db8:85a3:0::123::;'10.0.0.1';'2001:db8:85a3:0::123::' +18;172.16.0.1;2001:db8:::;'172.16.0.1';'2001:db8:::' +19;192.168.1.1;2001:db8:85a3:0000:0000:8a2e:0370:12345;'192.168.1.1';'2001:db8:85a3:0000:0000:8a2e:0370:12345' +20;172.16.0.0;2001:db8:85a3:0:0:0:0:0:0;'172.16.0.0';'2001:db8:85a3:0:0:0:0:0:0' +21;10.0.0.1;fe80:85a3:::7334;'10.0.0.1';'fe80:85a3:::7334' +22;192.168.1.1;2001:db8::85a3::7334;'192.168.1.1';'2001:db8::85a3::7334' +23;172.16.0.1;g123:abcd::85a3::7334;'172.16.0.1';'g123:abcd::85a3::7334' +24;10.0.0.1;2001:db8:85a3:0000:0000:8a2e:0370:abcdg;'10.0.0.1';'2001:db8:85a3:0000:0000:8a2e:0370:abcdg' +25;192.168.0.1;fe80:0000:0000:0000:0000:0000:0000:0001;'192.168.0.1';'fe80:0000:0000:0000:0000:0000:0000:0001' +26;192.168.1.1;12345::abcd;'192.168.1.1';'12345::abcd' +27;10.0.0.1;::85a3:::7334;'10.0.0.1';'::85a3:::7334' +28;172.16.0.0;2001:db8:85a3:0000:0000:8a2e:0370:abcd::;'172.16.0.0';'2001:db8:85a3:0000:0000:8a2e:0370:abcd::' +29;192.168.1.1;2001:db8:85a3:0000::g123;'192.168.1.1';'2001:db8:85a3:0000::g123' +30;10.0.0.1;2001:db8:85a3:0000::0370:abcd:12345;'10.0.0.1';'2001:db8:85a3:0000::0370:abcd:12345' +31;aaaa;bbb;'aaaa';'bbb' \ No newline at end of file diff --git a/regression-test/data/nereids_function_p0/fn_test_ip_normal.csv b/regression-test/data/nereids_function_p0/fn_test_ip_normal.csv new file mode 100644 index 00000000000000..7798cde2455dc3 --- /dev/null +++ b/regression-test/data/nereids_function_p0/fn_test_ip_normal.csv @@ -0,0 +1,72 @@ +29;0.0.0.1;2001:0db8:0:0:0:0:0:1;'0.0.0.1';'2001:0db8:0:0:0:0:0:1' +30;0.0.1.10;2001:0db8:0:0:0:0:0:2;'0.0.1.10';'2001:0db8:0:0:0:0:0:2' +31;0.0.2.20;2001:0db8:0:0:0:0:0:3;'0.0.2.20';'2001:0db8:0:0:0:0:0:3' +32;0.0.3.30;2001:0db8:0:0:0:0:0:4;'0.0.3.30';'2001:0db8:0:0:0:0:0:4' +33;10.0.0.5;2001:0db8:0:0:0:0:0:5;'10.0.0.5';'2001:0db8:0:0:0:0:0:5' +34;10.0.1.15;2001:0db8:0:0:0:0:0:6;'10.0.1.15';'2001:0db8:0:0:0:0:0:6' +35;10.1.0.25;2001:0db8:0:0:0:0:0:7;'10.1.0.25';'2001:0db8:0:0:0:0:0:7' +36;10.2.0.35;2001:0db8:0:0:0:0:0:8;'10.2.0.35';'2001:0db8:0:0:0:0:0:8' +37;10.3.0.45;2001:0db8:0:0:0:0:0:9;'10.3.0.45';'2001:0db8:0:0:0:0:0:9' +38;10.4.0.55;2001:0db8:0:0:0:0:0:a;'10.4.0.55';'2001:0db8:0:0:0:0:0:a' +39;10.5.0.65;2001:0db8:0:0:0:0:0:b;'10.5.0.65';'2001:0db8:0:0:0:0:0:b' +40;10.6.0.75;2001:0db8:0:0:0:0:0:c;'10.6.0.75';'2001:0db8:0:0:0:0:0:c' +41;10.7.0.85;2001:0db8:0:0:0:0:0:d;'10.7.0.85';'2001:0db8:0:0:0:0:0:d' +42;10.8.0.95;2001:0db8:0:0:0:0:0:e;'10.8.0.95';'2001:0db8:0:0:0:0:0:e' +43;10.9.1.5;2001:0db8:0:0:0:0:0:f;'10.9.1.5';'2001:0db8:0:0:0:0:0:f' +44;10.10.1.15;2001:0db8:0:0:0:0:0:10;'10.10.1.15';'2001:0db8:0:0:0:0:0:10' +45;10.11.1.25;2001:0db8:0:0:0:0:0:11;'10.11.1.25';'2001:0db8:0:0:0:0:0:11' +46;10.12.1.35;2001:0db8:0:0:0:0:0:12;'10.12.1.35';'2001:0db8:0:0:0:0:0:12' +47;10.13.1.45;2001:0db8:0:0:0:0:0:13;'10.13.1.45';'2001:0db8:0:0:0:0:0:13' +48;10.14.1.55;2001:0db8:0:0:0:0:0:14;'10.14.1.55';'2001:0db8:0:0:0:0:0:14' +49;10.15.1.65;2001:0db8:0:0:0:0:0:15;'10.15.1.65';'2001:0db8:0:0:0:0:0:15' +50;10.16.1.75;2001:0db8:0:0:0:0:0:16;'10.16.1.75';'2001:0db8:0:0:0:0:0:16' +51;128.0.0.1;2001:0db8:0:0:0:0:0:17;'128.0.0.1';'2001:0db8:0:0:0:0:0:17' +52;128.1.0.10;2001:0db8:0:0:0:0:0:18;'128.1.0.10';'2001:0db8:0:0:0:0:0:18' +53;128.2.0.20;2001:0db8:0:0:0:0:0:19;'128.2.0.20';'2001:0db8:0:0:0:0:0:19' +54;128.3.0.30;2001:0db8:0:0:0:0:0:1a;'128.3.0.30';'2001:0db8:0:0:0:0:0:1a' +55;128.4.0.40;2001:0db8:0:0:0:0:0:1b;'128.4.0.40';'2001:0db8:0:0:0:0:0:1b' +56;128.5.0.50;2001:0db8:0:0:0:0:0:1c;'128.5.0.50';'2001:0db8:0:0:0:0:0:1c' +57;128.6.0.60;2001:0db8:0:0:0:0:0:1d;'128.6.0.60';'2001:0db8:0:0:0:0:0:1d' +58;128.7.0.70;2001:0db8:0:0:0:0:0:1e;'128.7.0.70';'2001:0db8:0:0:0:0:0:1e' +59;128.8.0.80;2001:0db8:0:0:0:0:0:1f;'128.8.0.80';'2001:0db8:0:0:0:0:0:1f' +60;128.9.0.90;2001:0db8:0:0:0:0:0:20;'128.9.0.90';'2001:0db8:0:0:0:0:0:20' +61;172.16.0.1;2001:0db8:0:0:0:0:0:21;'172.16.0.1';'2001:0db8:0:0:0:0:0:21' +62;172.16.0.5;2001:0db8:0:0:0:0:0:22;'172.16.0.5';'2001:0db8:0:0:0:0:0:22' +63;172.16.0.10;2001:0db8:0:0:0:0:0:23;'172.16.0.10';'2001:0db8:0:0:0:0:0:23' +64;172.16.0.15;2001:0db8:0:0:0:0:0:24;'172.16.0.15';'2001:0db8:0:0:0:0:0:24' +65;172.16.0.20;2001:0db8:0:0:0:0:0:25;'172.16.0.20';'2001:0db8:0:0:0:0:0:25' +66;172.16.0.25;2001:0db8:0:0:0:0:0:26;'172.16.0.25';'2001:0db8:0:0:0:0:0:26' +67;172.16.0.30;2001:0db8:0:0:0:0:0:27;'172.16.0.30';'2001:0db8:0:0:0:0:0:27' +68;172.16.0.35;2001:0db8:0:0:0:0:0:28;'172.16.0.35';'2001:0db8:0:0:0:0:0:28' +69;192.0.0.1;2001:0db8:0:0:0:0:0:29;'192.0.0.1';'2001:0db8:0:0:0:0:0:29' +70;192.0.0.2;2001:0db8:0:0:0:0:0:2a;'192.0.0.2';'2001:0db8:0:0:0:0:0:2a' +71;192.0.0.3;2001:0db8:0:0:0:0:0:2b;'192.0.0.3';'2001:0db8:0:0:0:0:0:2b' +72;192.0.0.4;2001:0db8:0:0:0:0:0:2c;'192.0.0.4';'2001:0db8:0:0:0:0:0:2c' +73;192.0.0.5;2001:0db8:0:0:0:0:0:2d;'192.0.0.5';'2001:0db8:0:0:0:0:0:2d' +74;192.0.0.6;2001:0db8:0:0:0:0:0:2e;'192.0.0.6';'2001:0db8:0:0:0:0:0:2e' +75;192.0.0.7;2001:0db8:0:0:0:0:0:2f;'192.0.0.7';'2001:0db8:0:0:0:0:0:2f' +76;192.0.0.8;2001:0db8:0:0:0:0:0:30;'192.0.0.8';'2001:0db8:0:0:0:0:0:30' +77;192.0.0.9;2001:0db8:0:0:0:0:0:31;'192.0.0.9';'2001:0db8:0:0:0:0:0:31' +78;192.0.0.10;2001:0db8:0:0:0:0:0:32;'192.0.0.10';'2001:0db8:0:0:0:0:0:32' +79;192.168.0.1;2001:0db8:0:0:0:0:0:33;'192.168.0.1';'2001:0db8:0:0:0:0:0:33' +80;192.168.0.2;2001:0db8:0:0:0:0:0:34;'192.168.0.2';'2001:0db8:0:0:0:0:0:34' +81;192.168.1.5;2001:0db8:0:0:0:0:0:35;'192.168.1.5';'2001:0db8:0:0:0:0:0:35' +82;192.168.1.10;2001:0db8:0:0:0:0:0:36;'192.168.1.10';'2001:0db8:0:0:0:0:0:36' +83;192.168.1.15;2001:0db8:0:0:0:0:0:37;'192.168.1.15';'2001:0db8:0:0:0:0:0:37' +84;192.168.1.20;2001:0db8:0:0:0:0:0:38;'192.168.1.20';'2001:0db8:0:0:0:0:0:38' +85;192.168.2.5;2001:0db8:0:0:0:0:0:39;'192.168.2.5';'2001:0db8:0:0:0:0:0:39' +86;192.168.2.10;2001:0db8:0:0:0:0:0:3a;'192.168.2.10';'2001:0db8:0:0:0:0:0:3a' +87;192.168.5.20;2001:0db8:0:0:0:0:0:3b;'192.168.5.20';'2001:0db8:0:0:0:0:0:3b' +88;224.0.0.1;ff01::1;'224.0.0.1';'ff01::1' +89;224.0.0.2;ff01::2;'224.0.0.2';'ff01::2' +90;224.0.0.3;ff01::3;'224.0.0.3';'ff01::3' +91;224.0.0.4;ff01::4;'224.0.0.4';'ff01::4' +92;224.0.0.5;ff01::5;'224.0.0.5';'ff01::5' +93;224.0.0.6;ff01::6;'224.0.0.6';'ff01::6' +94;224.0.0.7;ff01::7;'224.0.0.7';'ff01::7' +95;224.0.0.8;ff01::8;'224.0.0.8';'ff01::8' +96;224.0.0.9;ff01::9;'224.0.0.9';'ff01::9' +97;224.0.0.10;ff01::a;'224.0.0.10';'ff01::a' +98;224.0.0.11;ff01::b;'224.0.0.11';'ff01::b' +99;224.0.0.12;ff01::c;'224.0.0.12';'ff01::c' +100;224.0.0.13;ff01::d;'224.0.0.13';'ff01::d' \ No newline at end of file diff --git a/regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv b/regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv new file mode 100644 index 00000000000000..937357921ae240 --- /dev/null +++ b/regression-test/data/nereids_function_p0/fn_test_ip_nullable.csv @@ -0,0 +1,100 @@ +1;127.0.0.1;::1 +2;10.0.0.0;null +3;null;fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4;172.16.0.0;fe80:: +5;null;febf:ffff:ffff:ffff:ffff +6;192.168.0.0;null +7;null;ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8;224.0.0.0;ff01:: +9;null;null +10;255.255.255.255;ff05::1 +11;null;ff02::2 +12;169.254.255.255;null +13;null;2001:db8::1 +14;192.0.2.0;null +15;198.51.100.0;ff02::5 +16;null;ff05::5 +17;240.0.0.0;null +18;100.64.0.0;2001::1 +19;null;2001::2 +20;192.0.0.9;ff02::1:ff00:1 +21;null;null +22;126.255.255.255;null +23;128.0.0.0;null +24;null;ff02::dead +25;192.0.0.0;ff02::1 +26;null;null +27;240.0.0.1;ff02::7 +28;null;ff03::1 +29;192.88.99.1;null +30;192.0.0.170;ff05::2 +31;null;ff06::1 +32;198.18.0.0;null +33;null;2001:db8::abcd:ef12 +34;233.252.0.0;null +35;null;ff0e::1 +36;null;ff0f::1 +37;172.20.10.5;fe80::1234:abcd +38;172.31.254.254;null +39;null;fd00::1234 +40;10.200.200.200;fdff::feed:beef +41;100.100.100.100;null +42;null;ff01::cafe +43;null;ff02::babe +44;8.8.8.8;2001:4860:4860::8888 +45;null;2001:4860:4860::8844 +46;null;null +47;9.9.9.9;2620:fe::fe +48;null;2600::3 +49;4.4.4.4;2602::4 +50;null;ff0f::8 +51;240.0.0.8;null +52;null;ff0d::1234 +53;10.10.10.10;null +54;192.168.2.254;ff01::5678 +55;null;fc01::1 +56;198.18.1.1;null +57;null;fd00::1234:5678 +58;192.168.200.200;null +59;100.100.10.10;ff02::9 +60;null;2001:4860:4860::8888 +61;1.1.1.1;null +62;198.18.255.255;2001:db8::dead:beef +63;null;fe80::dead:beef +64;169.254.100.100;null +65;null;ff05::dead +66;192.88.99.10;ff0e::feed +67;240.255.255.255;null +68;null;fd12::cafe +69;192.168.3.1;null +70;100.100.100.102;ff05::5678 +71;null;fe80::feed:beef +72;198.18.200.200;null +73;203.0.113.123;fd00::dead:beef +74;null;ff0f::abcd +75;null;fd12::dead +76;172.20.30.40;null +77;10.100.100.100;ff09::1234 +78;null;ff02::feed +79;null;fe80::feed +80;192.0.0.100;null +81;100.127.127.127;null +82;null;2001:db8::abcd:ef12 +83;null;null +84;null;fd12:3456:789a:1::1 +85;null;ff02::2 +86;192.168.1.1;2001:0db8:1234:5678:90ab:cdef:1234:5678 +87;null;fe80::1234:5678:9abc:def0 +88;null;ff05::1:3 +89;null;null +90;null;2001:db8::abcd:ef12 +91;null;ff08::1 +92;null;ff0e::1 +93;192.168.0.10;ff0f::1 +94;null;fe80::1234:abcd +95;192.0.2.123;null +96;null;fd00::cafe:babe +97;192.168.5.5;fdff::feed:beef +98;null;null +99;192.168.7.7;ff01::cafe +100;10.10.10.10;null \ No newline at end of file diff --git a/regression-test/data/nereids_function_p0/fn_test_ip_special.csv b/regression-test/data/nereids_function_p0/fn_test_ip_special.csv new file mode 100644 index 00000000000000..14266428cc6008 --- /dev/null +++ b/regression-test/data/nereids_function_p0/fn_test_ip_special.csv @@ -0,0 +1,29 @@ +1;127.0.0.1;::1;"127.0.0.1";"::1" +2;10.0.0.0;fc00::;"10.0.0.0";"fc00::" +3;10.255.255.255;fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"10.255.255.255";"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +4;172.16.0.0;fc00::;"172.16.0.0";"fc00::" +5;172.31.255.255;febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"172.31.255.255";"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +6;192.168.0.0;fe80::;"192.168.0.0";"fe80::" +7;192.168.255.255;ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"192.168.255.255";"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +8;169.254.0.0;fe80::;"169.254.0.0";"fe80::" +9;8.8.8.8;2001:4860:4860::8888;"8.8.8.8";"2001:4860:4860::8888" +10;1.1.1.1;2606:4700:4700::1111;"1.1.1.1";"2606:4700:4700::1111" +11;224.0.0.0;ff01::;"224.0.0.0";"ff01::" +12;239.255.255.255;ff02::1;"239.255.255.255";"ff02::1" +13;192.0.2.0;2001:0db8:85a3::8a2e:0370:7334;"192.0.2.0";"2001:0db8:85a3::8a2e:0370:7334" +14;203.0.113.0;2001:db8::1;"203.0.113.0";"2001:db8::1" +15;198.51.100.0;2001:db8::2;"198.51.100.0";"2001:db8::2" +16;localhost;::1;"localhost";"::1" +17;240.0.0.0;null;"240.0.0.0";"null" +18;255.255.255.255;null;"255.255.255.255";"null" +19;null;fd00::;"null";"fd00::" +20;0.0.0.0;null;"0.0.0.0";"null" +21;127.255.255.255;null;"127.255.255.255";"null" +22;128.0.0.0;null;"128.0.0.0";"null" +23;191.255.255.255;null;"191.255.255.255";"null" +24;192.0.0.0;null;"192.0.0.0";"null" +25;223.255.255.255;null;"223.255.255.255";"null" +26;224.0.0.0;ff01::;"224.0.0.0";"ff01::" +27;239.255.255.255;ff02::1;"239.255.255.255";"ff02::1" +28;null;ff00::;"null";"ff00::" + diff --git a/regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv b/regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv new file mode 100644 index 00000000000000..3519d27183cb62 --- /dev/null +++ b/regression-test/data/nereids_function_p0/fn_test_ip_special_no_null.csv @@ -0,0 +1,28 @@ +1;127.0.0.1;::1;"127.0.0.1";"::1" +2;10.0.0.0;fc00::;"10.0.0.0";"fc00::" +3;10.255.255.255;fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"10.255.255.255";"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +4;172.16.0.0;fc00::;"172.16.0.0";"fc00::" +5;172.31.255.255;febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"172.31.255.255";"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +6;192.168.0.0;fe80::;"192.168.0.0";"fe80::" +7;192.168.255.255;ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff;"192.168.255.255";"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" +8;169.254.0.0;fe80::;"169.254.0.0";"fe80::" +9;8.8.8.8;2001:4860:4860::8888;"8.8.8.8";"2001:4860:4860::8888" +10;1.1.1.1;2606:4700:4700::1111;"1.1.1.1";"2606:4700:4700::1111" +11;224.0.0.0;ff01::;"224.0.0.0";"ff01::" +12;239.255.255.255;ff02::1;"239.255.255.255";"ff02::1" +13;192.0.2.0;2001:0db8:85a3::8a2e:0370:7334;"192.0.2.0";"2001:0db8:85a3::8a2e:0370:7334" +14;203.0.113.0;2001:db8::1;"203.0.113.0";"2001:db8::1" +15;198.51.100.0;2001:db8::2;"198.51.100.0";"2001:db8::2" +16;localhost;::1;"localhost";"::1" +17;240.0.0.0;::1;"240.0.0.0";"::1" +18;255.255.255.255;ff00::9;"255.255.255.255";"ff00::9" +19;240.100.100.101;fd00::;"240.100.100.101";"fd00::" +20;0.0.0.0;fc00::babe;"0.0.0.0";"fc00::babe" +21;127.255.255.255;ff01::f0f0;"127.255.255.255";"ff01::f0f0" +22;128.0.0.0;ff09::1234;"128.0.0.0";"ff09::1234" +23;191.255.255.255;fd00::dead;"191.255.255.255";"fd00::dead" +24;192.0.0.0;fd00::feed;"192.0.0.0";"fd00::feed" +25;223.255.255.255;fd06::dead;"223.255.255.255";"fd06::dead" +26;224.0.0.0;ff01::;"224.0.0.0";"ff01::" +27;239.255.255.255;ff02::1;"239.255.255.255";"ff02::1" +28;100.127.127.127;ff00::;"100.127.127.127";"ff00::" diff --git a/regression-test/data/nereids_function_p0/scalar_function/IP.out b/regression-test/data/nereids_function_p0/scalar_function/IP.out new file mode 100644 index 00000000000000..5b44453ceb6232 --- /dev/null +++ b/regression-test/data/nereids_function_p0/scalar_function/IP.out @@ -0,0 +1,11779 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +100 + +-- !sql_cidr_ipv6 -- +1 :: ::ffff:ffff:ffff:ffff +2 fc00:: fc00::ffff:ffff:ffff:ffff +3 fdff:ffff:ffff:ffff:: fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: fc00::ffff:ffff:ffff:ffff +5 febf:ffff:ffff:ffff:: febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: fe80::ffff:ffff:ffff:ffff +7 ffff:ffff:ffff:ffff:: ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: fe80::ffff:ffff:ffff:ffff +9 2001:4860:4860:: 2001:4860:4860:0:ffff:ffff:ffff:ffff +10 2606:4700:4700:: 2606:4700:4700:0:ffff:ffff:ffff:ffff +11 ff01:: ff01::ffff:ffff:ffff:ffff +12 ff02:: ff02::ffff:ffff:ffff:ffff +13 2001:db8:85a3:: 2001:db8:85a3:0:ffff:ffff:ffff:ffff +14 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +15 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +16 :: ::ffff:ffff:ffff:ffff +17 \N \N +18 \N \N +19 fd00:: fd00::ffff:ffff:ffff:ffff +20 \N \N +21 \N \N +22 \N \N +23 \N \N +24 \N \N +25 \N \N +26 ff01:: ff01::ffff:ffff:ffff:ffff +27 ff02:: ff02::ffff:ffff:ffff:ffff +28 ff00:: ff00::ffff:ffff:ffff:ffff +29 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +30 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +31 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +32 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +33 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +34 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +35 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +36 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +37 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +38 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +39 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +40 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +41 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +42 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +43 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +44 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +45 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +46 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +47 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +48 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +49 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +50 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +51 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +52 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +53 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +54 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +55 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +56 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +57 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +58 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +59 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +60 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +61 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +62 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +63 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +64 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +65 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +66 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +67 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +68 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +69 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +70 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +71 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +72 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +73 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +74 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +75 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +76 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +77 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +78 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +79 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +80 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +81 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +82 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +83 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +84 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +85 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +86 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +87 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +88 ff01:: ff01::ffff:ffff:ffff:ffff +89 ff01:: ff01::ffff:ffff:ffff:ffff +90 ff01:: ff01::ffff:ffff:ffff:ffff +91 ff01:: ff01::ffff:ffff:ffff:ffff +92 ff01:: ff01::ffff:ffff:ffff:ffff +93 ff01:: ff01::ffff:ffff:ffff:ffff +94 ff01:: ff01::ffff:ffff:ffff:ffff +95 ff01:: ff01::ffff:ffff:ffff:ffff +96 ff01:: ff01::ffff:ffff:ffff:ffff +97 ff01:: ff01::ffff:ffff:ffff:ffff +98 ff01:: ff01::ffff:ffff:ffff:ffff +99 ff01:: ff01::ffff:ffff:ffff:ffff +100 ff01:: ff01::ffff:ffff:ffff:ffff + +-- !sql_cidr_ipv4 -- +1 127.0.0.1 127.0.0.1 +2 10.0.0.0 10.0.0.0 +3 10.255.255.255 10.255.255.255 +4 172.16.0.0 172.16.0.0 +5 172.31.255.255 172.31.255.255 +6 192.168.0.0 192.168.0.0 +7 192.168.255.255 192.168.255.255 +8 169.254.0.0 169.254.0.0 +9 8.8.8.8 8.8.8.8 +10 1.1.1.1 1.1.1.1 +11 224.0.0.0 224.0.0.0 +12 239.255.255.255 239.255.255.255 +13 192.0.2.0 192.0.2.0 +14 203.0.113.0 203.0.113.0 +15 198.51.100.0 198.51.100.0 +16 \N \N +17 240.0.0.0 240.0.0.0 +18 255.255.255.255 255.255.255.255 +19 \N \N +20 0.0.0.0 0.0.0.0 +21 127.255.255.255 127.255.255.255 +22 128.0.0.0 128.0.0.0 +23 191.255.255.255 191.255.255.255 +24 192.0.0.0 192.0.0.0 +25 223.255.255.255 223.255.255.255 +26 224.0.0.0 224.0.0.0 +27 239.255.255.255 239.255.255.255 +28 \N \N +29 0.0.0.1 0.0.0.1 +30 0.0.1.10 0.0.1.10 +31 0.0.2.20 0.0.2.20 +32 0.0.3.30 0.0.3.30 +33 10.0.0.5 10.0.0.5 +34 10.0.1.15 10.0.1.15 +35 10.1.0.25 10.1.0.25 +36 10.2.0.35 10.2.0.35 +37 10.3.0.45 10.3.0.45 +38 10.4.0.55 10.4.0.55 +39 10.5.0.65 10.5.0.65 +40 10.6.0.75 10.6.0.75 +41 10.7.0.85 10.7.0.85 +42 10.8.0.95 10.8.0.95 +43 10.9.1.5 10.9.1.5 +44 10.10.1.15 10.10.1.15 +45 10.11.1.25 10.11.1.25 +46 10.12.1.35 10.12.1.35 +47 10.13.1.45 10.13.1.45 +48 10.14.1.55 10.14.1.55 +49 10.15.1.65 10.15.1.65 +50 10.16.1.75 10.16.1.75 +51 128.0.0.1 128.0.0.1 +52 128.1.0.10 128.1.0.10 +53 128.2.0.20 128.2.0.20 +54 128.3.0.30 128.3.0.30 +55 128.4.0.40 128.4.0.40 +56 128.5.0.50 128.5.0.50 +57 128.6.0.60 128.6.0.60 +58 128.7.0.70 128.7.0.70 +59 128.8.0.80 128.8.0.80 +60 128.9.0.90 128.9.0.90 +61 172.16.0.1 172.16.0.1 +62 172.16.0.5 172.16.0.5 +63 172.16.0.10 172.16.0.10 +64 172.16.0.15 172.16.0.15 +65 172.16.0.20 172.16.0.20 +66 172.16.0.25 172.16.0.25 +67 172.16.0.30 172.16.0.30 +68 172.16.0.35 172.16.0.35 +69 192.0.0.1 192.0.0.1 +70 192.0.0.2 192.0.0.2 +71 192.0.0.3 192.0.0.3 +72 192.0.0.4 192.0.0.4 +73 192.0.0.5 192.0.0.5 +74 192.0.0.6 192.0.0.6 +75 192.0.0.7 192.0.0.7 +76 192.0.0.8 192.0.0.8 +77 192.0.0.9 192.0.0.9 +78 192.0.0.10 192.0.0.10 +79 192.168.0.1 192.168.0.1 +80 192.168.0.2 192.168.0.2 +81 192.168.1.5 192.168.1.5 +82 192.168.1.10 192.168.1.10 +83 192.168.1.15 192.168.1.15 +84 192.168.1.20 192.168.1.20 +85 192.168.2.5 192.168.2.5 +86 192.168.2.10 192.168.2.10 +87 192.168.5.20 192.168.5.20 +88 224.0.0.1 224.0.0.1 +89 224.0.0.2 224.0.0.2 +90 224.0.0.3 224.0.0.3 +91 224.0.0.4 224.0.0.4 +92 224.0.0.5 224.0.0.5 +93 224.0.0.6 224.0.0.6 +94 224.0.0.7 224.0.0.7 +95 224.0.0.8 224.0.0.8 +96 224.0.0.9 224.0.0.9 +97 224.0.0.10 224.0.0.10 +98 224.0.0.11 224.0.0.11 +99 224.0.0.12 224.0.0.12 +100 224.0.0.13 224.0.0.13 + +-- !sql_cidr_ipv6_all -- +1 {"min":"::", "max":"0:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +2 {"min":"fc00::", "max":"fc00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +3 {"min":"fdff::", "max":"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +4 {"min":"fc00::", "max":"fc00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +5 {"min":"febf::", "max":"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +6 {"min":"fe80::", "max":"fe80:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +7 {"min":"ffff::", "max":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +8 {"min":"fe80::", "max":"fe80:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +9 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +10 {"min":"2606::", "max":"2606:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +11 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +12 {"min":"ff02::", "max":"ff02:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +13 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +14 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +15 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +16 {"min":"::", "max":"0:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +17 \N +18 \N +19 {"min":"fd00::", "max":"fd00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +27 {"min":"ff02::", "max":"ff02:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +28 {"min":"ff00::", "max":"ff00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +29 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +30 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +31 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +32 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +33 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +34 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +35 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +36 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +37 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +38 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +39 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +40 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +41 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +42 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +43 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +44 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +45 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +46 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +47 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +48 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +49 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +50 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +51 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +52 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +53 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +54 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +55 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +56 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +57 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +58 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +59 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +60 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +61 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +62 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +63 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +64 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +65 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +66 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +67 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +68 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +69 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +70 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +71 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +72 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +73 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +74 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +75 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +76 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +77 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +78 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +79 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +80 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +81 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +82 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +83 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +84 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +85 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +86 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +87 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +88 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +89 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +90 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +91 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +92 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +93 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +94 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +95 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +96 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +97 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +98 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +99 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +100 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} + +-- !sql_cidr_ipv4_all -- +1 {"min":"127.0.0.0", "max":"127.0.255.255"} +2 {"min":"10.0.0.0", "max":"10.0.255.255"} +3 {"min":"10.255.0.0", "max":"10.255.255.255"} +4 {"min":"172.16.0.0", "max":"172.16.255.255"} +5 {"min":"172.31.0.0", "max":"172.31.255.255"} +6 {"min":"192.168.0.0", "max":"192.168.255.255"} +7 {"min":"192.168.0.0", "max":"192.168.255.255"} +8 {"min":"169.254.0.0", "max":"169.254.255.255"} +9 {"min":"8.8.0.0", "max":"8.8.255.255"} +10 {"min":"1.1.0.0", "max":"1.1.255.255"} +11 {"min":"224.0.0.0", "max":"224.0.255.255"} +12 {"min":"239.255.0.0", "max":"239.255.255.255"} +13 {"min":"192.0.0.0", "max":"192.0.255.255"} +14 {"min":"203.0.0.0", "max":"203.0.255.255"} +15 {"min":"198.51.0.0", "max":"198.51.255.255"} +16 \N +17 {"min":"240.0.0.0", "max":"240.0.255.255"} +18 {"min":"255.255.0.0", "max":"255.255.255.255"} +19 \N +20 {"min":"0.0.0.0", "max":"0.0.255.255"} +21 {"min":"127.255.0.0", "max":"127.255.255.255"} +22 {"min":"128.0.0.0", "max":"128.0.255.255"} +23 {"min":"191.255.0.0", "max":"191.255.255.255"} +24 {"min":"192.0.0.0", "max":"192.0.255.255"} +25 {"min":"223.255.0.0", "max":"223.255.255.255"} +26 {"min":"224.0.0.0", "max":"224.0.255.255"} +27 {"min":"239.255.0.0", "max":"239.255.255.255"} +28 \N +29 {"min":"0.0.0.0", "max":"0.0.255.255"} +30 {"min":"0.0.0.0", "max":"0.0.255.255"} +31 {"min":"0.0.0.0", "max":"0.0.255.255"} +32 {"min":"0.0.0.0", "max":"0.0.255.255"} +33 {"min":"10.0.0.0", "max":"10.0.255.255"} +34 {"min":"10.0.0.0", "max":"10.0.255.255"} +35 {"min":"10.1.0.0", "max":"10.1.255.255"} +36 {"min":"10.2.0.0", "max":"10.2.255.255"} +37 {"min":"10.3.0.0", "max":"10.3.255.255"} +38 {"min":"10.4.0.0", "max":"10.4.255.255"} +39 {"min":"10.5.0.0", "max":"10.5.255.255"} +40 {"min":"10.6.0.0", "max":"10.6.255.255"} +41 {"min":"10.7.0.0", "max":"10.7.255.255"} +42 {"min":"10.8.0.0", "max":"10.8.255.255"} +43 {"min":"10.9.0.0", "max":"10.9.255.255"} +44 {"min":"10.10.0.0", "max":"10.10.255.255"} +45 {"min":"10.11.0.0", "max":"10.11.255.255"} +46 {"min":"10.12.0.0", "max":"10.12.255.255"} +47 {"min":"10.13.0.0", "max":"10.13.255.255"} +48 {"min":"10.14.0.0", "max":"10.14.255.255"} +49 {"min":"10.15.0.0", "max":"10.15.255.255"} +50 {"min":"10.16.0.0", "max":"10.16.255.255"} +51 {"min":"128.0.0.0", "max":"128.0.255.255"} +52 {"min":"128.1.0.0", "max":"128.1.255.255"} +53 {"min":"128.2.0.0", "max":"128.2.255.255"} +54 {"min":"128.3.0.0", "max":"128.3.255.255"} +55 {"min":"128.4.0.0", "max":"128.4.255.255"} +56 {"min":"128.5.0.0", "max":"128.5.255.255"} +57 {"min":"128.6.0.0", "max":"128.6.255.255"} +58 {"min":"128.7.0.0", "max":"128.7.255.255"} +59 {"min":"128.8.0.0", "max":"128.8.255.255"} +60 {"min":"128.9.0.0", "max":"128.9.255.255"} +61 {"min":"172.16.0.0", "max":"172.16.255.255"} +62 {"min":"172.16.0.0", "max":"172.16.255.255"} +63 {"min":"172.16.0.0", "max":"172.16.255.255"} +64 {"min":"172.16.0.0", "max":"172.16.255.255"} +65 {"min":"172.16.0.0", "max":"172.16.255.255"} +66 {"min":"172.16.0.0", "max":"172.16.255.255"} +67 {"min":"172.16.0.0", "max":"172.16.255.255"} +68 {"min":"172.16.0.0", "max":"172.16.255.255"} +69 {"min":"192.0.0.0", "max":"192.0.255.255"} +70 {"min":"192.0.0.0", "max":"192.0.255.255"} +71 {"min":"192.0.0.0", "max":"192.0.255.255"} +72 {"min":"192.0.0.0", "max":"192.0.255.255"} +73 {"min":"192.0.0.0", "max":"192.0.255.255"} +74 {"min":"192.0.0.0", "max":"192.0.255.255"} +75 {"min":"192.0.0.0", "max":"192.0.255.255"} +76 {"min":"192.0.0.0", "max":"192.0.255.255"} +77 {"min":"192.0.0.0", "max":"192.0.255.255"} +78 {"min":"192.0.0.0", "max":"192.0.255.255"} +79 {"min":"192.168.0.0", "max":"192.168.255.255"} +80 {"min":"192.168.0.0", "max":"192.168.255.255"} +81 {"min":"192.168.0.0", "max":"192.168.255.255"} +82 {"min":"192.168.0.0", "max":"192.168.255.255"} +83 {"min":"192.168.0.0", "max":"192.168.255.255"} +84 {"min":"192.168.0.0", "max":"192.168.255.255"} +85 {"min":"192.168.0.0", "max":"192.168.255.255"} +86 {"min":"192.168.0.0", "max":"192.168.255.255"} +87 {"min":"192.168.0.0", "max":"192.168.255.255"} +88 {"min":"224.0.0.0", "max":"224.0.255.255"} +89 {"min":"224.0.0.0", "max":"224.0.255.255"} +90 {"min":"224.0.0.0", "max":"224.0.255.255"} +91 {"min":"224.0.0.0", "max":"224.0.255.255"} +92 {"min":"224.0.0.0", "max":"224.0.255.255"} +93 {"min":"224.0.0.0", "max":"224.0.255.255"} +94 {"min":"224.0.0.0", "max":"224.0.255.255"} +95 {"min":"224.0.0.0", "max":"224.0.255.255"} +96 {"min":"224.0.0.0", "max":"224.0.255.255"} +97 {"min":"224.0.0.0", "max":"224.0.255.255"} +98 {"min":"224.0.0.0", "max":"224.0.255.255"} +99 {"min":"224.0.0.0", "max":"224.0.255.255"} +100 {"min":"224.0.0.0", "max":"224.0.255.255"} + +-- !sql_num2string_ipv6 -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +16 ::1 +17 :: +18 :: +19 fd00:: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_num2string_ipv6_str -- +1 223a:3a31:2200:: +2 2266:6330:303a:3a22:: +3 \N +4 2266:6330:303a:3a22:: +5 \N +6 2266:6538:303a:3a22:: +7 \N +8 2266:6538:303a:3a22:: +9 \N +10 \N +11 2266:6630:313a:3a22:: +12 2266:6630:323a:3a31:2200:: +13 \N +14 2232:3030:313a:6462:383a:3a31:2200:0 +15 2232:3030:313a:6462:383a:3a32:2200:0 +16 223a:3a31:2200:: +17 226e:756c:6c22:: +18 226e:756c:6c22:: +19 2266:6430:303a:3a22:: +20 226e:756c:6c22:: +21 226e:756c:6c22:: +22 226e:756c:6c22:: +23 226e:756c:6c22:: +24 226e:756c:6c22:: +25 226e:756c:6c22:: +26 2266:6630:313a:3a22:: +27 2266:6630:323a:3a31:2200:: +28 2266:6630:303a:3a22:: +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 2766:6630:313a:3a31:2700:: +89 2766:6630:313a:3a32:2700:: +90 2766:6630:313a:3a33:2700:: +91 2766:6630:313a:3a34:2700:: +92 2766:6630:313a:3a35:2700:: +93 2766:6630:313a:3a36:2700:: +94 2766:6630:313a:3a37:2700:: +95 2766:6630:313a:3a38:2700:: +96 2766:6630:313a:3a39:2700:: +97 2766:6630:313a:3a61:2700:: +98 2766:6630:313a:3a62:2700:: +99 2766:6630:313a:3a63:2700:: +100 2766:6630:313a:3a64:2700:: + +-- !sql_num2string_ipv4 -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +16 0.0.0.0 +17 240.0.0.0 +18 255.255.255.255 +19 0.0.0.0 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 0.0.0.0 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_num2string_ipv4_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_inet6_ntoa -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +16 ::1 +17 :: +18 :: +19 fd00:: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_inet6_ntoa_str -- +1 223a:3a31:2200:: +2 2266:6330:303a:3a22:: +3 \N +4 2266:6330:303a:3a22:: +5 \N +6 2266:6538:303a:3a22:: +7 \N +8 2266:6538:303a:3a22:: +9 \N +10 \N +11 2266:6630:313a:3a22:: +12 2266:6630:323a:3a31:2200:: +13 \N +14 2232:3030:313a:6462:383a:3a31:2200:0 +15 2232:3030:313a:6462:383a:3a32:2200:0 +16 223a:3a31:2200:: +17 226e:756c:6c22:: +18 226e:756c:6c22:: +19 2266:6430:303a:3a22:: +20 226e:756c:6c22:: +21 226e:756c:6c22:: +22 226e:756c:6c22:: +23 226e:756c:6c22:: +24 226e:756c:6c22:: +25 226e:756c:6c22:: +26 2266:6630:313a:3a22:: +27 2266:6630:323a:3a31:2200:: +28 2266:6630:303a:3a22:: +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 2766:6630:313a:3a31:2700:: +89 2766:6630:313a:3a32:2700:: +90 2766:6630:313a:3a33:2700:: +91 2766:6630:313a:3a34:2700:: +92 2766:6630:313a:3a35:2700:: +93 2766:6630:313a:3a36:2700:: +94 2766:6630:313a:3a37:2700:: +95 2766:6630:313a:3a38:2700:: +96 2766:6630:313a:3a39:2700:: +97 2766:6630:313a:3a61:2700:: +98 2766:6630:313a:3a62:2700:: +99 2766:6630:313a:3a63:2700:: +100 2766:6630:313a:3a64:2700:: + +-- !sql_inet_ntoa -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +16 0.0.0.0 +17 240.0.0.0 +18 255.255.255.255 +19 0.0.0.0 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 0.0.0.0 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_inet_ntoa_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_string2num_or_default_ipv6 -- +1 00000000000000000000000000000001 +2 FC000000000000000000000000000000 +3 FDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +4 FC000000000000000000000000000000 +5 FEBFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +6 FE800000000000000000000000000000 +7 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +8 FE800000000000000000000000000000 +9 20014860486000000000000000008888 +10 26064700470000000000000000001111 +11 FF010000000000000000000000000000 +12 FF020000000000000000000000000001 +13 20010DB885A3000000008A2E03707334 +14 20010DB8000000000000000000000001 +15 20010DB8000000000000000000000002 +16 00000000000000000000000000000001 +17 00000000000000000000000000000000 +18 00000000000000000000000000000000 +19 FD000000000000000000000000000000 +20 00000000000000000000000000000000 +21 00000000000000000000000000000000 +22 00000000000000000000000000000000 +23 00000000000000000000000000000000 +24 00000000000000000000000000000000 +25 00000000000000000000000000000000 +26 FF010000000000000000000000000000 +27 FF020000000000000000000000000001 +28 FF000000000000000000000000000000 +29 20010DB8000000000000000000000001 +30 20010DB8000000000000000000000002 +31 20010DB8000000000000000000000003 +32 20010DB8000000000000000000000004 +33 20010DB8000000000000000000000005 +34 20010DB8000000000000000000000006 +35 20010DB8000000000000000000000007 +36 20010DB8000000000000000000000008 +37 20010DB8000000000000000000000009 +38 20010DB800000000000000000000000A +39 20010DB800000000000000000000000B +40 20010DB800000000000000000000000C +41 20010DB800000000000000000000000D +42 20010DB800000000000000000000000E +43 20010DB800000000000000000000000F +44 20010DB8000000000000000000000010 +45 20010DB8000000000000000000000011 +46 20010DB8000000000000000000000012 +47 20010DB8000000000000000000000013 +48 20010DB8000000000000000000000014 +49 20010DB8000000000000000000000015 +50 20010DB8000000000000000000000016 +51 20010DB8000000000000000000000017 +52 20010DB8000000000000000000000018 +53 20010DB8000000000000000000000019 +54 20010DB800000000000000000000001A +55 20010DB800000000000000000000001B +56 20010DB800000000000000000000001C +57 20010DB800000000000000000000001D +58 20010DB800000000000000000000001E +59 20010DB800000000000000000000001F +60 20010DB8000000000000000000000020 +61 20010DB8000000000000000000000021 +62 20010DB8000000000000000000000022 +63 20010DB8000000000000000000000023 +64 20010DB8000000000000000000000024 +65 20010DB8000000000000000000000025 +66 20010DB8000000000000000000000026 +67 20010DB8000000000000000000000027 +68 20010DB8000000000000000000000028 +69 20010DB8000000000000000000000029 +70 20010DB800000000000000000000002A +71 20010DB800000000000000000000002B +72 20010DB800000000000000000000002C +73 20010DB800000000000000000000002D +74 20010DB800000000000000000000002E +75 20010DB800000000000000000000002F +76 20010DB8000000000000000000000030 +77 20010DB8000000000000000000000031 +78 20010DB8000000000000000000000032 +79 20010DB8000000000000000000000033 +80 20010DB8000000000000000000000034 +81 20010DB8000000000000000000000035 +82 20010DB8000000000000000000000036 +83 20010DB8000000000000000000000037 +84 20010DB8000000000000000000000038 +85 20010DB8000000000000000000000039 +86 20010DB800000000000000000000003A +87 20010DB800000000000000000000003B +88 FF010000000000000000000000000001 +89 FF010000000000000000000000000002 +90 FF010000000000000000000000000003 +91 FF010000000000000000000000000004 +92 FF010000000000000000000000000005 +93 FF010000000000000000000000000006 +94 FF010000000000000000000000000007 +95 FF010000000000000000000000000008 +96 FF010000000000000000000000000009 +97 FF01000000000000000000000000000A +98 FF01000000000000000000000000000B +99 FF01000000000000000000000000000C +100 FF01000000000000000000000000000D + +-- !sql_string2num_or_default_ipv6_str -- +1 00000000000000000000000000000000 +2 00000000000000000000000000000000 +3 00000000000000000000000000000000 +4 00000000000000000000000000000000 +5 00000000000000000000000000000000 +6 00000000000000000000000000000000 +7 00000000000000000000000000000000 +8 00000000000000000000000000000000 +9 00000000000000000000000000000000 +10 00000000000000000000000000000000 +11 00000000000000000000000000000000 +12 00000000000000000000000000000000 +13 00000000000000000000000000000000 +14 00000000000000000000000000000000 +15 00000000000000000000000000000000 +16 00000000000000000000000000000000 +17 00000000000000000000000000000000 +18 00000000000000000000000000000000 +19 00000000000000000000000000000000 +20 00000000000000000000000000000000 +21 00000000000000000000000000000000 +22 00000000000000000000000000000000 +23 00000000000000000000000000000000 +24 00000000000000000000000000000000 +25 00000000000000000000000000000000 +26 00000000000000000000000000000000 +27 00000000000000000000000000000000 +28 00000000000000000000000000000000 +29 00000000000000000000000000000000 +30 00000000000000000000000000000000 +31 00000000000000000000000000000000 +32 00000000000000000000000000000000 +33 00000000000000000000000000000000 +34 00000000000000000000000000000000 +35 00000000000000000000000000000000 +36 00000000000000000000000000000000 +37 00000000000000000000000000000000 +38 00000000000000000000000000000000 +39 00000000000000000000000000000000 +40 00000000000000000000000000000000 +41 00000000000000000000000000000000 +42 00000000000000000000000000000000 +43 00000000000000000000000000000000 +44 00000000000000000000000000000000 +45 00000000000000000000000000000000 +46 00000000000000000000000000000000 +47 00000000000000000000000000000000 +48 00000000000000000000000000000000 +49 00000000000000000000000000000000 +50 00000000000000000000000000000000 +51 00000000000000000000000000000000 +52 00000000000000000000000000000000 +53 00000000000000000000000000000000 +54 00000000000000000000000000000000 +55 00000000000000000000000000000000 +56 00000000000000000000000000000000 +57 00000000000000000000000000000000 +58 00000000000000000000000000000000 +59 00000000000000000000000000000000 +60 00000000000000000000000000000000 +61 00000000000000000000000000000000 +62 00000000000000000000000000000000 +63 00000000000000000000000000000000 +64 00000000000000000000000000000000 +65 00000000000000000000000000000000 +66 00000000000000000000000000000000 +67 00000000000000000000000000000000 +68 00000000000000000000000000000000 +69 00000000000000000000000000000000 +70 00000000000000000000000000000000 +71 00000000000000000000000000000000 +72 00000000000000000000000000000000 +73 00000000000000000000000000000000 +74 00000000000000000000000000000000 +75 00000000000000000000000000000000 +76 00000000000000000000000000000000 +77 00000000000000000000000000000000 +78 00000000000000000000000000000000 +79 00000000000000000000000000000000 +80 00000000000000000000000000000000 +81 00000000000000000000000000000000 +82 00000000000000000000000000000000 +83 00000000000000000000000000000000 +84 00000000000000000000000000000000 +85 00000000000000000000000000000000 +86 00000000000000000000000000000000 +87 00000000000000000000000000000000 +88 00000000000000000000000000000000 +89 00000000000000000000000000000000 +90 00000000000000000000000000000000 +91 00000000000000000000000000000000 +92 00000000000000000000000000000000 +93 00000000000000000000000000000000 +94 00000000000000000000000000000000 +95 00000000000000000000000000000000 +96 00000000000000000000000000000000 +97 00000000000000000000000000000000 +98 00000000000000000000000000000000 +99 00000000000000000000000000000000 +100 00000000000000000000000000000000 + +-- !sql_string2num_or_default_ipv4 -- +1 2130706433 +2 167772160 +3 184549375 +4 2886729728 +5 2887778303 +6 3232235520 +7 3232301055 +8 2851995648 +9 134744072 +10 16843009 +11 3758096384 +12 4026531839 +13 3221225984 +14 3405803776 +15 3325256704 +16 0 +17 4026531840 +18 4294967295 +19 0 +20 0 +21 2147483647 +22 2147483648 +23 3221225471 +24 3221225472 +25 3758096383 +26 3758096384 +27 4026531839 +28 0 +29 1 +30 266 +31 532 +32 798 +33 167772165 +34 167772431 +35 167837721 +36 167903267 +37 167968813 +38 168034359 +39 168099905 +40 168165451 +41 168230997 +42 168296543 +43 168362245 +44 168427791 +45 168493337 +46 168558883 +47 168624429 +48 168689975 +49 168755521 +50 168821067 +51 2147483649 +52 2147549194 +53 2147614740 +54 2147680286 +55 2147745832 +56 2147811378 +57 2147876924 +58 2147942470 +59 2148008016 +60 2148073562 +61 2886729729 +62 2886729733 +63 2886729738 +64 2886729743 +65 2886729748 +66 2886729753 +67 2886729758 +68 2886729763 +69 3221225473 +70 3221225474 +71 3221225475 +72 3221225476 +73 3221225477 +74 3221225478 +75 3221225479 +76 3221225480 +77 3221225481 +78 3221225482 +79 3232235521 +80 3232235522 +81 3232235781 +82 3232235786 +83 3232235791 +84 3232235796 +85 3232236037 +86 3232236042 +87 3232236820 +88 3758096385 +89 3758096386 +90 3758096387 +91 3758096388 +92 3758096389 +93 3758096390 +94 3758096391 +95 3758096392 +96 3758096393 +97 3758096394 +98 3758096395 +99 3758096396 +100 3758096397 + +-- !sql_string2num_or_default_ipv4_str -- +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 +10 0 +11 0 +12 0 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +21 0 +22 0 +23 0 +24 0 +25 0 +26 0 +27 0 +28 0 +29 0 +30 0 +31 0 +32 0 +33 0 +34 0 +35 0 +36 0 +37 0 +38 0 +39 0 +40 0 +41 0 +42 0 +43 0 +44 0 +45 0 +46 0 +47 0 +48 0 +49 0 +50 0 +51 0 +52 0 +53 0 +54 0 +55 0 +56 0 +57 0 +58 0 +59 0 +60 0 +61 0 +62 0 +63 0 +64 0 +65 0 +66 0 +67 0 +68 0 +69 0 +70 0 +71 0 +72 0 +73 0 +74 0 +75 0 +76 0 +77 0 +78 0 +79 0 +80 0 +81 0 +82 0 +83 0 +84 0 +85 0 +86 0 +87 0 +88 0 +89 0 +90 0 +91 0 +92 0 +93 0 +94 0 +95 0 +96 0 +97 0 +98 0 +99 0 +100 0 + +-- !sql_string2num_or_null_ipv6 -- +1 00000000000000000000000000000001 +2 FC000000000000000000000000000000 +3 FDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +4 FC000000000000000000000000000000 +5 FEBFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +6 FE800000000000000000000000000000 +7 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +8 FE800000000000000000000000000000 +9 20014860486000000000000000008888 +10 26064700470000000000000000001111 +11 FF010000000000000000000000000000 +12 FF020000000000000000000000000001 +13 20010DB885A3000000008A2E03707334 +14 20010DB8000000000000000000000001 +15 20010DB8000000000000000000000002 +16 00000000000000000000000000000001 +17 \N +18 \N +19 FD000000000000000000000000000000 +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 FF010000000000000000000000000000 +27 FF020000000000000000000000000001 +28 FF000000000000000000000000000000 +29 20010DB8000000000000000000000001 +30 20010DB8000000000000000000000002 +31 20010DB8000000000000000000000003 +32 20010DB8000000000000000000000004 +33 20010DB8000000000000000000000005 +34 20010DB8000000000000000000000006 +35 20010DB8000000000000000000000007 +36 20010DB8000000000000000000000008 +37 20010DB8000000000000000000000009 +38 20010DB800000000000000000000000A +39 20010DB800000000000000000000000B +40 20010DB800000000000000000000000C +41 20010DB800000000000000000000000D +42 20010DB800000000000000000000000E +43 20010DB800000000000000000000000F +44 20010DB8000000000000000000000010 +45 20010DB8000000000000000000000011 +46 20010DB8000000000000000000000012 +47 20010DB8000000000000000000000013 +48 20010DB8000000000000000000000014 +49 20010DB8000000000000000000000015 +50 20010DB8000000000000000000000016 +51 20010DB8000000000000000000000017 +52 20010DB8000000000000000000000018 +53 20010DB8000000000000000000000019 +54 20010DB800000000000000000000001A +55 20010DB800000000000000000000001B +56 20010DB800000000000000000000001C +57 20010DB800000000000000000000001D +58 20010DB800000000000000000000001E +59 20010DB800000000000000000000001F +60 20010DB8000000000000000000000020 +61 20010DB8000000000000000000000021 +62 20010DB8000000000000000000000022 +63 20010DB8000000000000000000000023 +64 20010DB8000000000000000000000024 +65 20010DB8000000000000000000000025 +66 20010DB8000000000000000000000026 +67 20010DB8000000000000000000000027 +68 20010DB8000000000000000000000028 +69 20010DB8000000000000000000000029 +70 20010DB800000000000000000000002A +71 20010DB800000000000000000000002B +72 20010DB800000000000000000000002C +73 20010DB800000000000000000000002D +74 20010DB800000000000000000000002E +75 20010DB800000000000000000000002F +76 20010DB8000000000000000000000030 +77 20010DB8000000000000000000000031 +78 20010DB8000000000000000000000032 +79 20010DB8000000000000000000000033 +80 20010DB8000000000000000000000034 +81 20010DB8000000000000000000000035 +82 20010DB8000000000000000000000036 +83 20010DB8000000000000000000000037 +84 20010DB8000000000000000000000038 +85 20010DB8000000000000000000000039 +86 20010DB800000000000000000000003A +87 20010DB800000000000000000000003B +88 FF010000000000000000000000000001 +89 FF010000000000000000000000000002 +90 FF010000000000000000000000000003 +91 FF010000000000000000000000000004 +92 FF010000000000000000000000000005 +93 FF010000000000000000000000000006 +94 FF010000000000000000000000000007 +95 FF010000000000000000000000000008 +96 FF010000000000000000000000000009 +97 FF01000000000000000000000000000A +98 FF01000000000000000000000000000B +99 FF01000000000000000000000000000C +100 FF01000000000000000000000000000D + +-- !sql_string2num_or_null_ipv6_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_string2num_or_null_ipv4 -- +1 2130706433 +2 167772160 +3 184549375 +4 2886729728 +5 2887778303 +6 3232235520 +7 3232301055 +8 2851995648 +9 134744072 +10 16843009 +11 3758096384 +12 4026531839 +13 3221225984 +14 3405803776 +15 3325256704 +16 \N +17 4026531840 +18 4294967295 +19 \N +20 0 +21 2147483647 +22 2147483648 +23 3221225471 +24 3221225472 +25 3758096383 +26 3758096384 +27 4026531839 +28 \N +29 1 +30 266 +31 532 +32 798 +33 167772165 +34 167772431 +35 167837721 +36 167903267 +37 167968813 +38 168034359 +39 168099905 +40 168165451 +41 168230997 +42 168296543 +43 168362245 +44 168427791 +45 168493337 +46 168558883 +47 168624429 +48 168689975 +49 168755521 +50 168821067 +51 2147483649 +52 2147549194 +53 2147614740 +54 2147680286 +55 2147745832 +56 2147811378 +57 2147876924 +58 2147942470 +59 2148008016 +60 2148073562 +61 2886729729 +62 2886729733 +63 2886729738 +64 2886729743 +65 2886729748 +66 2886729753 +67 2886729758 +68 2886729763 +69 3221225473 +70 3221225474 +71 3221225475 +72 3221225476 +73 3221225477 +74 3221225478 +75 3221225479 +76 3221225480 +77 3221225481 +78 3221225482 +79 3232235521 +80 3232235522 +81 3232235781 +82 3232235786 +83 3232235791 +84 3232235796 +85 3232236037 +86 3232236042 +87 3232236820 +88 3758096385 +89 3758096386 +90 3758096387 +91 3758096388 +92 3758096389 +93 3758096390 +94 3758096391 +95 3758096392 +96 3758096393 +97 3758096394 +98 3758096395 +99 3758096396 +100 3758096397 + +-- !sql_string2num_or_null_ipv4_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ipv4_compat -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 \N +18 \N +19 false +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv4_compat_str6 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ipv4_compat_str4 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ipv4_mapped -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 \N +18 \N +19 false +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv4_mapped_str6 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ipv4_mapped_str4 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ip_address_in_range_ipv6 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 true +14 true +15 true +16 false +17 \N +18 \N +19 false +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 false +27 false +28 false +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ip_address_in_range_ipv4 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 \N +17 false +18 false +19 \N +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 \N +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ip_address_in_range_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ip_address_in_range_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ip_address_in_range_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ip_address_in_range_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_is_ipv4_string -- +1 true +2 true +3 true +4 true +5 true +6 true +7 true +8 true +9 true +10 true +11 true +12 true +13 true +14 true +15 true +16 \N +17 true +18 true +19 \N +20 true +21 true +22 true +23 true +24 true +25 true +26 true +27 true +28 \N +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 true +89 true +90 true +91 true +92 true +93 true +94 true +95 true +96 true +97 true +98 true +99 true +100 true + +-- !sql_is_ipv4_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv6_string -- +1 true +2 true +3 true +4 true +5 true +6 true +7 true +8 true +9 true +10 true +11 true +12 true +13 true +14 true +15 true +16 true +17 \N +18 \N +19 true +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 true +27 true +28 true +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 true +89 true +90 true +91 true +92 true +93 true +94 true +95 true +96 true +97 true +98 true +99 true +100 true + +-- !sql_is_ipv6_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv6_string -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 \N +17 false +18 false +19 \N +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 \N +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv6_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv4_string -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 \N +18 \N +19 false +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_is_ipv4_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +16 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_to_ipv6_or_default -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +16 ::1 +17 :: +18 :: +19 fd00:: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_to_ipv6_or_default_str -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +16 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_to_ipv4_or_default -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +16 0.0.0.0 +17 240.0.0.0 +18 255.255.255.255 +19 0.0.0.0 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 0.0.0.0 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_to_ipv4_or_default_str -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +16 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_to_ipv6_or_default -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +16 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_to_ipv6_or_default_st -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +16 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_to_ipv4_or_default -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +16 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_to_ipv4_or_default_st -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +16 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_to_ipv6_or_null -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +16 ::1 +17 \N +18 \N +19 fd00:: +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_to_ipv6_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_to_ipv4_or_null -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +16 \N +17 240.0.0.0 +18 255.255.255.255 +19 \N +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 \N +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_to_ipv4_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_to_ipv6_or_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_to_ipv6_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_to_ipv4_or_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_to_ipv4_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null -- +99 + +-- !sql_not_null_cidr_ipv6 -- +1 :: ::ffff:ffff:ffff:ffff +2 fc00:: fc00::ffff:ffff:ffff:ffff +3 fdff:ffff:ffff:ffff:: fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: fc00::ffff:ffff:ffff:ffff +5 febf:ffff:ffff:ffff:: febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: fe80::ffff:ffff:ffff:ffff +7 ffff:ffff:ffff:ffff:: ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: fe80::ffff:ffff:ffff:ffff +9 2001:4860:4860:: 2001:4860:4860:0:ffff:ffff:ffff:ffff +10 2606:4700:4700:: 2606:4700:4700:0:ffff:ffff:ffff:ffff +11 ff01:: ff01::ffff:ffff:ffff:ffff +12 ff02:: ff02::ffff:ffff:ffff:ffff +13 2001:db8:85a3:: 2001:db8:85a3:0:ffff:ffff:ffff:ffff +14 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +15 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +17 :: ::ffff:ffff:ffff:ffff +18 ff00:: ff00::ffff:ffff:ffff:ffff +19 fd00:: fd00::ffff:ffff:ffff:ffff +20 fc00:: fc00::ffff:ffff:ffff:ffff +21 ff01:: ff01::ffff:ffff:ffff:ffff +22 ff09:: ff09::ffff:ffff:ffff:ffff +23 fd00:: fd00::ffff:ffff:ffff:ffff +24 fd00:: fd00::ffff:ffff:ffff:ffff +25 fd06:: fd06::ffff:ffff:ffff:ffff +26 ff01:: ff01::ffff:ffff:ffff:ffff +27 ff02:: ff02::ffff:ffff:ffff:ffff +28 ff00:: ff00::ffff:ffff:ffff:ffff +29 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +30 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +31 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +32 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +33 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +34 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +35 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +36 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +37 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +38 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +39 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +40 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +41 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +42 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +43 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +44 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +45 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +46 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +47 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +48 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +49 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +50 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +51 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +52 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +53 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +54 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +55 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +56 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +57 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +58 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +59 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +60 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +61 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +62 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +63 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +64 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +65 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +66 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +67 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +68 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +69 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +70 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +71 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +72 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +73 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +74 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +75 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +76 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +77 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +78 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +79 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +80 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +81 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +82 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +83 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +84 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +85 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +86 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +87 2001:db8:: 2001:db8::ffff:ffff:ffff:ffff +88 ff01:: ff01::ffff:ffff:ffff:ffff +89 ff01:: ff01::ffff:ffff:ffff:ffff +90 ff01:: ff01::ffff:ffff:ffff:ffff +91 ff01:: ff01::ffff:ffff:ffff:ffff +92 ff01:: ff01::ffff:ffff:ffff:ffff +93 ff01:: ff01::ffff:ffff:ffff:ffff +94 ff01:: ff01::ffff:ffff:ffff:ffff +95 ff01:: ff01::ffff:ffff:ffff:ffff +96 ff01:: ff01::ffff:ffff:ffff:ffff +97 ff01:: ff01::ffff:ffff:ffff:ffff +98 ff01:: ff01::ffff:ffff:ffff:ffff +99 ff01:: ff01::ffff:ffff:ffff:ffff +100 ff01:: ff01::ffff:ffff:ffff:ffff + +-- !sql_not_null_cidr_ipv4 -- +1 127.0.0.1 127.0.0.1 +2 10.0.0.0 10.0.0.0 +3 10.255.255.255 10.255.255.255 +4 172.16.0.0 172.16.0.0 +5 172.31.255.255 172.31.255.255 +6 192.168.0.0 192.168.0.0 +7 192.168.255.255 192.168.255.255 +8 169.254.0.0 169.254.0.0 +9 8.8.8.8 8.8.8.8 +10 1.1.1.1 1.1.1.1 +11 224.0.0.0 224.0.0.0 +12 239.255.255.255 239.255.255.255 +13 192.0.2.0 192.0.2.0 +14 203.0.113.0 203.0.113.0 +15 198.51.100.0 198.51.100.0 +17 240.0.0.0 240.0.0.0 +18 255.255.255.255 255.255.255.255 +19 240.100.100.101 240.100.100.101 +20 0.0.0.0 0.0.0.0 +21 127.255.255.255 127.255.255.255 +22 128.0.0.0 128.0.0.0 +23 191.255.255.255 191.255.255.255 +24 192.0.0.0 192.0.0.0 +25 223.255.255.255 223.255.255.255 +26 224.0.0.0 224.0.0.0 +27 239.255.255.255 239.255.255.255 +28 100.127.127.127 100.127.127.127 +29 0.0.0.1 0.0.0.1 +30 0.0.1.10 0.0.1.10 +31 0.0.2.20 0.0.2.20 +32 0.0.3.30 0.0.3.30 +33 10.0.0.5 10.0.0.5 +34 10.0.1.15 10.0.1.15 +35 10.1.0.25 10.1.0.25 +36 10.2.0.35 10.2.0.35 +37 10.3.0.45 10.3.0.45 +38 10.4.0.55 10.4.0.55 +39 10.5.0.65 10.5.0.65 +40 10.6.0.75 10.6.0.75 +41 10.7.0.85 10.7.0.85 +42 10.8.0.95 10.8.0.95 +43 10.9.1.5 10.9.1.5 +44 10.10.1.15 10.10.1.15 +45 10.11.1.25 10.11.1.25 +46 10.12.1.35 10.12.1.35 +47 10.13.1.45 10.13.1.45 +48 10.14.1.55 10.14.1.55 +49 10.15.1.65 10.15.1.65 +50 10.16.1.75 10.16.1.75 +51 128.0.0.1 128.0.0.1 +52 128.1.0.10 128.1.0.10 +53 128.2.0.20 128.2.0.20 +54 128.3.0.30 128.3.0.30 +55 128.4.0.40 128.4.0.40 +56 128.5.0.50 128.5.0.50 +57 128.6.0.60 128.6.0.60 +58 128.7.0.70 128.7.0.70 +59 128.8.0.80 128.8.0.80 +60 128.9.0.90 128.9.0.90 +61 172.16.0.1 172.16.0.1 +62 172.16.0.5 172.16.0.5 +63 172.16.0.10 172.16.0.10 +64 172.16.0.15 172.16.0.15 +65 172.16.0.20 172.16.0.20 +66 172.16.0.25 172.16.0.25 +67 172.16.0.30 172.16.0.30 +68 172.16.0.35 172.16.0.35 +69 192.0.0.1 192.0.0.1 +70 192.0.0.2 192.0.0.2 +71 192.0.0.3 192.0.0.3 +72 192.0.0.4 192.0.0.4 +73 192.0.0.5 192.0.0.5 +74 192.0.0.6 192.0.0.6 +75 192.0.0.7 192.0.0.7 +76 192.0.0.8 192.0.0.8 +77 192.0.0.9 192.0.0.9 +78 192.0.0.10 192.0.0.10 +79 192.168.0.1 192.168.0.1 +80 192.168.0.2 192.168.0.2 +81 192.168.1.5 192.168.1.5 +82 192.168.1.10 192.168.1.10 +83 192.168.1.15 192.168.1.15 +84 192.168.1.20 192.168.1.20 +85 192.168.2.5 192.168.2.5 +86 192.168.2.10 192.168.2.10 +87 192.168.5.20 192.168.5.20 +88 224.0.0.1 224.0.0.1 +89 224.0.0.2 224.0.0.2 +90 224.0.0.3 224.0.0.3 +91 224.0.0.4 224.0.0.4 +92 224.0.0.5 224.0.0.5 +93 224.0.0.6 224.0.0.6 +94 224.0.0.7 224.0.0.7 +95 224.0.0.8 224.0.0.8 +96 224.0.0.9 224.0.0.9 +97 224.0.0.10 224.0.0.10 +98 224.0.0.11 224.0.0.11 +99 224.0.0.12 224.0.0.12 +100 224.0.0.13 224.0.0.13 + +-- !sql_not_null_cidr_ipv6_all -- +1 {"min":"::", "max":"0:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +2 {"min":"fc00::", "max":"fc00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +3 {"min":"fdff::", "max":"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +4 {"min":"fc00::", "max":"fc00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +5 {"min":"febf::", "max":"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +6 {"min":"fe80::", "max":"fe80:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +7 {"min":"ffff::", "max":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +8 {"min":"fe80::", "max":"fe80:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +9 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +10 {"min":"2606::", "max":"2606:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +11 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +12 {"min":"ff02::", "max":"ff02:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +13 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +14 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +15 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +17 {"min":"::", "max":"0:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +18 {"min":"ff00::", "max":"ff00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +19 {"min":"fd00::", "max":"fd00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +20 {"min":"fc00::", "max":"fc00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +21 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +22 {"min":"ff09::", "max":"ff09:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +23 {"min":"fd00::", "max":"fd00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +24 {"min":"fd00::", "max":"fd00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +25 {"min":"fd06::", "max":"fd06:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +26 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +27 {"min":"ff02::", "max":"ff02:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +28 {"min":"ff00::", "max":"ff00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +29 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +30 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +31 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +32 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +33 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +34 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +35 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +36 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +37 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +38 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +39 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +40 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +41 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +42 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +43 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +44 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +45 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +46 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +47 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +48 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +49 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +50 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +51 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +52 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +53 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +54 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +55 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +56 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +57 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +58 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +59 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +60 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +61 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +62 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +63 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +64 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +65 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +66 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +67 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +68 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +69 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +70 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +71 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +72 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +73 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +74 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +75 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +76 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +77 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +78 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +79 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +80 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +81 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +82 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +83 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +84 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +85 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +86 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +87 {"min":"2001::", "max":"2001:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +88 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +89 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +90 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +91 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +92 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +93 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +94 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +95 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +96 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +97 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +98 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +99 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} +100 {"min":"ff01::", "max":"ff01:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} + +-- !sql_not_null_cidr_ipv4_all -- +1 {"min":"127.0.0.0", "max":"127.0.255.255"} +2 {"min":"10.0.0.0", "max":"10.0.255.255"} +3 {"min":"10.255.0.0", "max":"10.255.255.255"} +4 {"min":"172.16.0.0", "max":"172.16.255.255"} +5 {"min":"172.31.0.0", "max":"172.31.255.255"} +6 {"min":"192.168.0.0", "max":"192.168.255.255"} +7 {"min":"192.168.0.0", "max":"192.168.255.255"} +8 {"min":"169.254.0.0", "max":"169.254.255.255"} +9 {"min":"8.8.0.0", "max":"8.8.255.255"} +10 {"min":"1.1.0.0", "max":"1.1.255.255"} +11 {"min":"224.0.0.0", "max":"224.0.255.255"} +12 {"min":"239.255.0.0", "max":"239.255.255.255"} +13 {"min":"192.0.0.0", "max":"192.0.255.255"} +14 {"min":"203.0.0.0", "max":"203.0.255.255"} +15 {"min":"198.51.0.0", "max":"198.51.255.255"} +17 {"min":"240.0.0.0", "max":"240.0.255.255"} +18 {"min":"255.255.0.0", "max":"255.255.255.255"} +19 {"min":"240.100.0.0", "max":"240.100.255.255"} +20 {"min":"0.0.0.0", "max":"0.0.255.255"} +21 {"min":"127.255.0.0", "max":"127.255.255.255"} +22 {"min":"128.0.0.0", "max":"128.0.255.255"} +23 {"min":"191.255.0.0", "max":"191.255.255.255"} +24 {"min":"192.0.0.0", "max":"192.0.255.255"} +25 {"min":"223.255.0.0", "max":"223.255.255.255"} +26 {"min":"224.0.0.0", "max":"224.0.255.255"} +27 {"min":"239.255.0.0", "max":"239.255.255.255"} +28 {"min":"100.127.0.0", "max":"100.127.255.255"} +29 {"min":"0.0.0.0", "max":"0.0.255.255"} +30 {"min":"0.0.0.0", "max":"0.0.255.255"} +31 {"min":"0.0.0.0", "max":"0.0.255.255"} +32 {"min":"0.0.0.0", "max":"0.0.255.255"} +33 {"min":"10.0.0.0", "max":"10.0.255.255"} +34 {"min":"10.0.0.0", "max":"10.0.255.255"} +35 {"min":"10.1.0.0", "max":"10.1.255.255"} +36 {"min":"10.2.0.0", "max":"10.2.255.255"} +37 {"min":"10.3.0.0", "max":"10.3.255.255"} +38 {"min":"10.4.0.0", "max":"10.4.255.255"} +39 {"min":"10.5.0.0", "max":"10.5.255.255"} +40 {"min":"10.6.0.0", "max":"10.6.255.255"} +41 {"min":"10.7.0.0", "max":"10.7.255.255"} +42 {"min":"10.8.0.0", "max":"10.8.255.255"} +43 {"min":"10.9.0.0", "max":"10.9.255.255"} +44 {"min":"10.10.0.0", "max":"10.10.255.255"} +45 {"min":"10.11.0.0", "max":"10.11.255.255"} +46 {"min":"10.12.0.0", "max":"10.12.255.255"} +47 {"min":"10.13.0.0", "max":"10.13.255.255"} +48 {"min":"10.14.0.0", "max":"10.14.255.255"} +49 {"min":"10.15.0.0", "max":"10.15.255.255"} +50 {"min":"10.16.0.0", "max":"10.16.255.255"} +51 {"min":"128.0.0.0", "max":"128.0.255.255"} +52 {"min":"128.1.0.0", "max":"128.1.255.255"} +53 {"min":"128.2.0.0", "max":"128.2.255.255"} +54 {"min":"128.3.0.0", "max":"128.3.255.255"} +55 {"min":"128.4.0.0", "max":"128.4.255.255"} +56 {"min":"128.5.0.0", "max":"128.5.255.255"} +57 {"min":"128.6.0.0", "max":"128.6.255.255"} +58 {"min":"128.7.0.0", "max":"128.7.255.255"} +59 {"min":"128.8.0.0", "max":"128.8.255.255"} +60 {"min":"128.9.0.0", "max":"128.9.255.255"} +61 {"min":"172.16.0.0", "max":"172.16.255.255"} +62 {"min":"172.16.0.0", "max":"172.16.255.255"} +63 {"min":"172.16.0.0", "max":"172.16.255.255"} +64 {"min":"172.16.0.0", "max":"172.16.255.255"} +65 {"min":"172.16.0.0", "max":"172.16.255.255"} +66 {"min":"172.16.0.0", "max":"172.16.255.255"} +67 {"min":"172.16.0.0", "max":"172.16.255.255"} +68 {"min":"172.16.0.0", "max":"172.16.255.255"} +69 {"min":"192.0.0.0", "max":"192.0.255.255"} +70 {"min":"192.0.0.0", "max":"192.0.255.255"} +71 {"min":"192.0.0.0", "max":"192.0.255.255"} +72 {"min":"192.0.0.0", "max":"192.0.255.255"} +73 {"min":"192.0.0.0", "max":"192.0.255.255"} +74 {"min":"192.0.0.0", "max":"192.0.255.255"} +75 {"min":"192.0.0.0", "max":"192.0.255.255"} +76 {"min":"192.0.0.0", "max":"192.0.255.255"} +77 {"min":"192.0.0.0", "max":"192.0.255.255"} +78 {"min":"192.0.0.0", "max":"192.0.255.255"} +79 {"min":"192.168.0.0", "max":"192.168.255.255"} +80 {"min":"192.168.0.0", "max":"192.168.255.255"} +81 {"min":"192.168.0.0", "max":"192.168.255.255"} +82 {"min":"192.168.0.0", "max":"192.168.255.255"} +83 {"min":"192.168.0.0", "max":"192.168.255.255"} +84 {"min":"192.168.0.0", "max":"192.168.255.255"} +85 {"min":"192.168.0.0", "max":"192.168.255.255"} +86 {"min":"192.168.0.0", "max":"192.168.255.255"} +87 {"min":"192.168.0.0", "max":"192.168.255.255"} +88 {"min":"224.0.0.0", "max":"224.0.255.255"} +89 {"min":"224.0.0.0", "max":"224.0.255.255"} +90 {"min":"224.0.0.0", "max":"224.0.255.255"} +91 {"min":"224.0.0.0", "max":"224.0.255.255"} +92 {"min":"224.0.0.0", "max":"224.0.255.255"} +93 {"min":"224.0.0.0", "max":"224.0.255.255"} +94 {"min":"224.0.0.0", "max":"224.0.255.255"} +95 {"min":"224.0.0.0", "max":"224.0.255.255"} +96 {"min":"224.0.0.0", "max":"224.0.255.255"} +97 {"min":"224.0.0.0", "max":"224.0.255.255"} +98 {"min":"224.0.0.0", "max":"224.0.255.255"} +99 {"min":"224.0.0.0", "max":"224.0.255.255"} +100 {"min":"224.0.0.0", "max":"224.0.255.255"} + +-- !sql_not_null_ipv6_string_to_num -- +1 00000000000000000000000000000001 +2 FC000000000000000000000000000000 +3 FDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +4 FC000000000000000000000000000000 +5 FEBFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +6 FE800000000000000000000000000000 +7 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +8 FE800000000000000000000000000000 +9 20014860486000000000000000008888 +10 26064700470000000000000000001111 +11 FF010000000000000000000000000000 +12 FF020000000000000000000000000001 +13 20010DB885A3000000008A2E03707334 +14 20010DB8000000000000000000000001 +15 20010DB8000000000000000000000002 +17 00000000000000000000000000000001 +18 FF000000000000000000000000000009 +19 FD000000000000000000000000000000 +20 FC00000000000000000000000000BABE +21 FF01000000000000000000000000F0F0 +22 FF090000000000000000000000001234 +23 FD00000000000000000000000000DEAD +24 FD00000000000000000000000000FEED +25 FD06000000000000000000000000DEAD +26 FF010000000000000000000000000000 +27 FF020000000000000000000000000001 +28 FF000000000000000000000000000000 +29 20010DB8000000000000000000000001 +30 20010DB8000000000000000000000002 +31 20010DB8000000000000000000000003 +32 20010DB8000000000000000000000004 +33 20010DB8000000000000000000000005 +34 20010DB8000000000000000000000006 +35 20010DB8000000000000000000000007 +36 20010DB8000000000000000000000008 +37 20010DB8000000000000000000000009 +38 20010DB800000000000000000000000A +39 20010DB800000000000000000000000B +40 20010DB800000000000000000000000C +41 20010DB800000000000000000000000D +42 20010DB800000000000000000000000E +43 20010DB800000000000000000000000F +44 20010DB8000000000000000000000010 +45 20010DB8000000000000000000000011 +46 20010DB8000000000000000000000012 +47 20010DB8000000000000000000000013 +48 20010DB8000000000000000000000014 +49 20010DB8000000000000000000000015 +50 20010DB8000000000000000000000016 +51 20010DB8000000000000000000000017 +52 20010DB8000000000000000000000018 +53 20010DB8000000000000000000000019 +54 20010DB800000000000000000000001A +55 20010DB800000000000000000000001B +56 20010DB800000000000000000000001C +57 20010DB800000000000000000000001D +58 20010DB800000000000000000000001E +59 20010DB800000000000000000000001F +60 20010DB8000000000000000000000020 +61 20010DB8000000000000000000000021 +62 20010DB8000000000000000000000022 +63 20010DB8000000000000000000000023 +64 20010DB8000000000000000000000024 +65 20010DB8000000000000000000000025 +66 20010DB8000000000000000000000026 +67 20010DB8000000000000000000000027 +68 20010DB8000000000000000000000028 +69 20010DB8000000000000000000000029 +70 20010DB800000000000000000000002A +71 20010DB800000000000000000000002B +72 20010DB800000000000000000000002C +73 20010DB800000000000000000000002D +74 20010DB800000000000000000000002E +75 20010DB800000000000000000000002F +76 20010DB8000000000000000000000030 +77 20010DB8000000000000000000000031 +78 20010DB8000000000000000000000032 +79 20010DB8000000000000000000000033 +80 20010DB8000000000000000000000034 +81 20010DB8000000000000000000000035 +82 20010DB8000000000000000000000036 +83 20010DB8000000000000000000000037 +84 20010DB8000000000000000000000038 +85 20010DB8000000000000000000000039 +86 20010DB800000000000000000000003A +87 20010DB800000000000000000000003B +88 FF010000000000000000000000000001 +89 FF010000000000000000000000000002 +90 FF010000000000000000000000000003 +91 FF010000000000000000000000000004 +92 FF010000000000000000000000000005 +93 FF010000000000000000000000000006 +94 FF010000000000000000000000000007 +95 FF010000000000000000000000000008 +96 FF010000000000000000000000000009 +97 FF01000000000000000000000000000A +98 FF01000000000000000000000000000B +99 FF01000000000000000000000000000C +100 FF01000000000000000000000000000D + +-- !sql_not_null_ipv4_string_to_num -- +1 2130706433 +2 167772160 +3 184549375 +4 2886729728 +5 2887778303 +6 3232235520 +7 3232301055 +8 2851995648 +9 134744072 +10 16843009 +11 3758096384 +12 4026531839 +13 3221225984 +14 3405803776 +15 3325256704 +17 4026531840 +18 4294967295 +19 4033111141 +20 0 +21 2147483647 +22 2147483648 +23 3221225471 +24 3221225472 +25 3758096383 +26 3758096384 +27 4026531839 +28 1686077311 +29 1 +30 266 +31 532 +32 798 +33 167772165 +34 167772431 +35 167837721 +36 167903267 +37 167968813 +38 168034359 +39 168099905 +40 168165451 +41 168230997 +42 168296543 +43 168362245 +44 168427791 +45 168493337 +46 168558883 +47 168624429 +48 168689975 +49 168755521 +50 168821067 +51 2147483649 +52 2147549194 +53 2147614740 +54 2147680286 +55 2147745832 +56 2147811378 +57 2147876924 +58 2147942470 +59 2148008016 +60 2148073562 +61 2886729729 +62 2886729733 +63 2886729738 +64 2886729743 +65 2886729748 +66 2886729753 +67 2886729758 +68 2886729763 +69 3221225473 +70 3221225474 +71 3221225475 +72 3221225476 +73 3221225477 +74 3221225478 +75 3221225479 +76 3221225480 +77 3221225481 +78 3221225482 +79 3232235521 +80 3232235522 +81 3232235781 +82 3232235786 +83 3232235791 +84 3232235796 +85 3232236037 +86 3232236042 +87 3232236820 +88 3758096385 +89 3758096386 +90 3758096387 +91 3758096388 +92 3758096389 +93 3758096390 +94 3758096391 +95 3758096392 +96 3758096393 +97 3758096394 +98 3758096395 +99 3758096396 +100 3758096397 + +-- !sql_not_null_num2string_ipv6 -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +17 ::1 +18 ff00::9 +19 fd00:: +20 fc00::babe +21 ff01::f0f0 +22 ff09::1234 +23 fd00::dead +24 fd00::feed +25 fd06::dead +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_not_null_num2string_ipv6_str -- +1 223a:3a31:2200:: +2 2266:6330:303a:3a22:: +3 \N +4 2266:6330:303a:3a22:: +5 \N +6 2266:6538:303a:3a22:: +7 \N +8 2266:6538:303a:3a22:: +9 \N +10 \N +11 2266:6630:313a:3a22:: +12 2266:6630:323a:3a31:2200:: +13 \N +14 2232:3030:313a:6462:383a:3a31:2200:0 +15 2232:3030:313a:6462:383a:3a32:2200:0 +17 223a:3a31:2200:: +18 2266:6630:303a:3a39:2200:: +19 2266:6430:303a:3a22:: +20 2266:6330:303a:3a62:6162:6522:: +21 2266:6630:313a:3a66:3066:3022:: +22 2266:6630:393a:3a31:3233:3422:: +23 2266:6430:303a:3a64:6561:6422:: +24 2266:6430:303a:3a66:6565:6422:: +25 2266:6430:363a:3a64:6561:6422:: +26 2266:6630:313a:3a22:: +27 2266:6630:323a:3a31:2200:: +28 2266:6630:303a:3a22:: +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 2766:6630:313a:3a31:2700:: +89 2766:6630:313a:3a32:2700:: +90 2766:6630:313a:3a33:2700:: +91 2766:6630:313a:3a34:2700:: +92 2766:6630:313a:3a35:2700:: +93 2766:6630:313a:3a36:2700:: +94 2766:6630:313a:3a37:2700:: +95 2766:6630:313a:3a38:2700:: +96 2766:6630:313a:3a39:2700:: +97 2766:6630:313a:3a61:2700:: +98 2766:6630:313a:3a62:2700:: +99 2766:6630:313a:3a63:2700:: +100 2766:6630:313a:3a64:2700:: + +-- !sql_not_null_num2string_ipv4 -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +17 240.0.0.0 +18 255.255.255.255 +19 240.100.100.101 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 100.127.127.127 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_not_null_num2string_ipv4_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_inet6_ntoa -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +17 ::1 +18 ff00::9 +19 fd00:: +20 fc00::babe +21 ff01::f0f0 +22 ff09::1234 +23 fd00::dead +24 fd00::feed +25 fd06::dead +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_not_null_inet6_ntoa_str -- +1 223a:3a31:2200:: +2 2266:6330:303a:3a22:: +3 \N +4 2266:6330:303a:3a22:: +5 \N +6 2266:6538:303a:3a22:: +7 \N +8 2266:6538:303a:3a22:: +9 \N +10 \N +11 2266:6630:313a:3a22:: +12 2266:6630:323a:3a31:2200:: +13 \N +14 2232:3030:313a:6462:383a:3a31:2200:0 +15 2232:3030:313a:6462:383a:3a32:2200:0 +17 223a:3a31:2200:: +18 2266:6630:303a:3a39:2200:: +19 2266:6430:303a:3a22:: +20 2266:6330:303a:3a62:6162:6522:: +21 2266:6630:313a:3a66:3066:3022:: +22 2266:6630:393a:3a31:3233:3422:: +23 2266:6430:303a:3a64:6561:6422:: +24 2266:6430:303a:3a66:6565:6422:: +25 2266:6430:363a:3a64:6561:6422:: +26 2266:6630:313a:3a22:: +27 2266:6630:323a:3a31:2200:: +28 2266:6630:303a:3a22:: +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 2766:6630:313a:3a31:2700:: +89 2766:6630:313a:3a32:2700:: +90 2766:6630:313a:3a33:2700:: +91 2766:6630:313a:3a34:2700:: +92 2766:6630:313a:3a35:2700:: +93 2766:6630:313a:3a36:2700:: +94 2766:6630:313a:3a37:2700:: +95 2766:6630:313a:3a38:2700:: +96 2766:6630:313a:3a39:2700:: +97 2766:6630:313a:3a61:2700:: +98 2766:6630:313a:3a62:2700:: +99 2766:6630:313a:3a63:2700:: +100 2766:6630:313a:3a64:2700:: + +-- !sql_not_null_inet_ntoa -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +17 240.0.0.0 +18 255.255.255.255 +19 240.100.100.101 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 100.127.127.127 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_not_null_inet_ntoa_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_string2num_or_default_ipv6 -- +1 00000000000000000000000000000001 +2 FC000000000000000000000000000000 +3 FDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +4 FC000000000000000000000000000000 +5 FEBFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +6 FE800000000000000000000000000000 +7 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +8 FE800000000000000000000000000000 +9 20014860486000000000000000008888 +10 26064700470000000000000000001111 +11 FF010000000000000000000000000000 +12 FF020000000000000000000000000001 +13 20010DB885A3000000008A2E03707334 +14 20010DB8000000000000000000000001 +15 20010DB8000000000000000000000002 +17 00000000000000000000000000000001 +18 FF000000000000000000000000000009 +19 FD000000000000000000000000000000 +20 FC00000000000000000000000000BABE +21 FF01000000000000000000000000F0F0 +22 FF090000000000000000000000001234 +23 FD00000000000000000000000000DEAD +24 FD00000000000000000000000000FEED +25 FD06000000000000000000000000DEAD +26 FF010000000000000000000000000000 +27 FF020000000000000000000000000001 +28 FF000000000000000000000000000000 +29 20010DB8000000000000000000000001 +30 20010DB8000000000000000000000002 +31 20010DB8000000000000000000000003 +32 20010DB8000000000000000000000004 +33 20010DB8000000000000000000000005 +34 20010DB8000000000000000000000006 +35 20010DB8000000000000000000000007 +36 20010DB8000000000000000000000008 +37 20010DB8000000000000000000000009 +38 20010DB800000000000000000000000A +39 20010DB800000000000000000000000B +40 20010DB800000000000000000000000C +41 20010DB800000000000000000000000D +42 20010DB800000000000000000000000E +43 20010DB800000000000000000000000F +44 20010DB8000000000000000000000010 +45 20010DB8000000000000000000000011 +46 20010DB8000000000000000000000012 +47 20010DB8000000000000000000000013 +48 20010DB8000000000000000000000014 +49 20010DB8000000000000000000000015 +50 20010DB8000000000000000000000016 +51 20010DB8000000000000000000000017 +52 20010DB8000000000000000000000018 +53 20010DB8000000000000000000000019 +54 20010DB800000000000000000000001A +55 20010DB800000000000000000000001B +56 20010DB800000000000000000000001C +57 20010DB800000000000000000000001D +58 20010DB800000000000000000000001E +59 20010DB800000000000000000000001F +60 20010DB8000000000000000000000020 +61 20010DB8000000000000000000000021 +62 20010DB8000000000000000000000022 +63 20010DB8000000000000000000000023 +64 20010DB8000000000000000000000024 +65 20010DB8000000000000000000000025 +66 20010DB8000000000000000000000026 +67 20010DB8000000000000000000000027 +68 20010DB8000000000000000000000028 +69 20010DB8000000000000000000000029 +70 20010DB800000000000000000000002A +71 20010DB800000000000000000000002B +72 20010DB800000000000000000000002C +73 20010DB800000000000000000000002D +74 20010DB800000000000000000000002E +75 20010DB800000000000000000000002F +76 20010DB8000000000000000000000030 +77 20010DB8000000000000000000000031 +78 20010DB8000000000000000000000032 +79 20010DB8000000000000000000000033 +80 20010DB8000000000000000000000034 +81 20010DB8000000000000000000000035 +82 20010DB8000000000000000000000036 +83 20010DB8000000000000000000000037 +84 20010DB8000000000000000000000038 +85 20010DB8000000000000000000000039 +86 20010DB800000000000000000000003A +87 20010DB800000000000000000000003B +88 FF010000000000000000000000000001 +89 FF010000000000000000000000000002 +90 FF010000000000000000000000000003 +91 FF010000000000000000000000000004 +92 FF010000000000000000000000000005 +93 FF010000000000000000000000000006 +94 FF010000000000000000000000000007 +95 FF010000000000000000000000000008 +96 FF010000000000000000000000000009 +97 FF01000000000000000000000000000A +98 FF01000000000000000000000000000B +99 FF01000000000000000000000000000C +100 FF01000000000000000000000000000D + +-- !sql_not_null_string2num_or_default_ipv6_str -- +1 00000000000000000000000000000000 +2 00000000000000000000000000000000 +3 00000000000000000000000000000000 +4 00000000000000000000000000000000 +5 00000000000000000000000000000000 +6 00000000000000000000000000000000 +7 00000000000000000000000000000000 +8 00000000000000000000000000000000 +9 00000000000000000000000000000000 +10 00000000000000000000000000000000 +11 00000000000000000000000000000000 +12 00000000000000000000000000000000 +13 00000000000000000000000000000000 +14 00000000000000000000000000000000 +15 00000000000000000000000000000000 +17 00000000000000000000000000000000 +18 00000000000000000000000000000000 +19 00000000000000000000000000000000 +20 00000000000000000000000000000000 +21 00000000000000000000000000000000 +22 00000000000000000000000000000000 +23 00000000000000000000000000000000 +24 00000000000000000000000000000000 +25 00000000000000000000000000000000 +26 00000000000000000000000000000000 +27 00000000000000000000000000000000 +28 00000000000000000000000000000000 +29 00000000000000000000000000000000 +30 00000000000000000000000000000000 +31 00000000000000000000000000000000 +32 00000000000000000000000000000000 +33 00000000000000000000000000000000 +34 00000000000000000000000000000000 +35 00000000000000000000000000000000 +36 00000000000000000000000000000000 +37 00000000000000000000000000000000 +38 00000000000000000000000000000000 +39 00000000000000000000000000000000 +40 00000000000000000000000000000000 +41 00000000000000000000000000000000 +42 00000000000000000000000000000000 +43 00000000000000000000000000000000 +44 00000000000000000000000000000000 +45 00000000000000000000000000000000 +46 00000000000000000000000000000000 +47 00000000000000000000000000000000 +48 00000000000000000000000000000000 +49 00000000000000000000000000000000 +50 00000000000000000000000000000000 +51 00000000000000000000000000000000 +52 00000000000000000000000000000000 +53 00000000000000000000000000000000 +54 00000000000000000000000000000000 +55 00000000000000000000000000000000 +56 00000000000000000000000000000000 +57 00000000000000000000000000000000 +58 00000000000000000000000000000000 +59 00000000000000000000000000000000 +60 00000000000000000000000000000000 +61 00000000000000000000000000000000 +62 00000000000000000000000000000000 +63 00000000000000000000000000000000 +64 00000000000000000000000000000000 +65 00000000000000000000000000000000 +66 00000000000000000000000000000000 +67 00000000000000000000000000000000 +68 00000000000000000000000000000000 +69 00000000000000000000000000000000 +70 00000000000000000000000000000000 +71 00000000000000000000000000000000 +72 00000000000000000000000000000000 +73 00000000000000000000000000000000 +74 00000000000000000000000000000000 +75 00000000000000000000000000000000 +76 00000000000000000000000000000000 +77 00000000000000000000000000000000 +78 00000000000000000000000000000000 +79 00000000000000000000000000000000 +80 00000000000000000000000000000000 +81 00000000000000000000000000000000 +82 00000000000000000000000000000000 +83 00000000000000000000000000000000 +84 00000000000000000000000000000000 +85 00000000000000000000000000000000 +86 00000000000000000000000000000000 +87 00000000000000000000000000000000 +88 00000000000000000000000000000000 +89 00000000000000000000000000000000 +90 00000000000000000000000000000000 +91 00000000000000000000000000000000 +92 00000000000000000000000000000000 +93 00000000000000000000000000000000 +94 00000000000000000000000000000000 +95 00000000000000000000000000000000 +96 00000000000000000000000000000000 +97 00000000000000000000000000000000 +98 00000000000000000000000000000000 +99 00000000000000000000000000000000 +100 00000000000000000000000000000000 + +-- !sql_not_null_string2num_or_default_ipv4 -- +1 2130706433 +2 167772160 +3 184549375 +4 2886729728 +5 2887778303 +6 3232235520 +7 3232301055 +8 2851995648 +9 134744072 +10 16843009 +11 3758096384 +12 4026531839 +13 3221225984 +14 3405803776 +15 3325256704 +17 4026531840 +18 4294967295 +19 4033111141 +20 0 +21 2147483647 +22 2147483648 +23 3221225471 +24 3221225472 +25 3758096383 +26 3758096384 +27 4026531839 +28 1686077311 +29 1 +30 266 +31 532 +32 798 +33 167772165 +34 167772431 +35 167837721 +36 167903267 +37 167968813 +38 168034359 +39 168099905 +40 168165451 +41 168230997 +42 168296543 +43 168362245 +44 168427791 +45 168493337 +46 168558883 +47 168624429 +48 168689975 +49 168755521 +50 168821067 +51 2147483649 +52 2147549194 +53 2147614740 +54 2147680286 +55 2147745832 +56 2147811378 +57 2147876924 +58 2147942470 +59 2148008016 +60 2148073562 +61 2886729729 +62 2886729733 +63 2886729738 +64 2886729743 +65 2886729748 +66 2886729753 +67 2886729758 +68 2886729763 +69 3221225473 +70 3221225474 +71 3221225475 +72 3221225476 +73 3221225477 +74 3221225478 +75 3221225479 +76 3221225480 +77 3221225481 +78 3221225482 +79 3232235521 +80 3232235522 +81 3232235781 +82 3232235786 +83 3232235791 +84 3232235796 +85 3232236037 +86 3232236042 +87 3232236820 +88 3758096385 +89 3758096386 +90 3758096387 +91 3758096388 +92 3758096389 +93 3758096390 +94 3758096391 +95 3758096392 +96 3758096393 +97 3758096394 +98 3758096395 +99 3758096396 +100 3758096397 + +-- !sql_not_null_string2num_or_default_ipv4_str -- +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 +10 0 +11 0 +12 0 +13 0 +14 0 +15 0 +17 0 +18 0 +19 0 +20 0 +21 0 +22 0 +23 0 +24 0 +25 0 +26 0 +27 0 +28 0 +29 0 +30 0 +31 0 +32 0 +33 0 +34 0 +35 0 +36 0 +37 0 +38 0 +39 0 +40 0 +41 0 +42 0 +43 0 +44 0 +45 0 +46 0 +47 0 +48 0 +49 0 +50 0 +51 0 +52 0 +53 0 +54 0 +55 0 +56 0 +57 0 +58 0 +59 0 +60 0 +61 0 +62 0 +63 0 +64 0 +65 0 +66 0 +67 0 +68 0 +69 0 +70 0 +71 0 +72 0 +73 0 +74 0 +75 0 +76 0 +77 0 +78 0 +79 0 +80 0 +81 0 +82 0 +83 0 +84 0 +85 0 +86 0 +87 0 +88 0 +89 0 +90 0 +91 0 +92 0 +93 0 +94 0 +95 0 +96 0 +97 0 +98 0 +99 0 +100 0 + +-- !sql_not_null_string2num_or_null_ipv6 -- +1 00000000000000000000000000000001 +2 FC000000000000000000000000000000 +3 FDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +4 FC000000000000000000000000000000 +5 FEBFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +6 FE800000000000000000000000000000 +7 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +8 FE800000000000000000000000000000 +9 20014860486000000000000000008888 +10 26064700470000000000000000001111 +11 FF010000000000000000000000000000 +12 FF020000000000000000000000000001 +13 20010DB885A3000000008A2E03707334 +14 20010DB8000000000000000000000001 +15 20010DB8000000000000000000000002 +17 00000000000000000000000000000001 +18 FF000000000000000000000000000009 +19 FD000000000000000000000000000000 +20 FC00000000000000000000000000BABE +21 FF01000000000000000000000000F0F0 +22 FF090000000000000000000000001234 +23 FD00000000000000000000000000DEAD +24 FD00000000000000000000000000FEED +25 FD06000000000000000000000000DEAD +26 FF010000000000000000000000000000 +27 FF020000000000000000000000000001 +28 FF000000000000000000000000000000 +29 20010DB8000000000000000000000001 +30 20010DB8000000000000000000000002 +31 20010DB8000000000000000000000003 +32 20010DB8000000000000000000000004 +33 20010DB8000000000000000000000005 +34 20010DB8000000000000000000000006 +35 20010DB8000000000000000000000007 +36 20010DB8000000000000000000000008 +37 20010DB8000000000000000000000009 +38 20010DB800000000000000000000000A +39 20010DB800000000000000000000000B +40 20010DB800000000000000000000000C +41 20010DB800000000000000000000000D +42 20010DB800000000000000000000000E +43 20010DB800000000000000000000000F +44 20010DB8000000000000000000000010 +45 20010DB8000000000000000000000011 +46 20010DB8000000000000000000000012 +47 20010DB8000000000000000000000013 +48 20010DB8000000000000000000000014 +49 20010DB8000000000000000000000015 +50 20010DB8000000000000000000000016 +51 20010DB8000000000000000000000017 +52 20010DB8000000000000000000000018 +53 20010DB8000000000000000000000019 +54 20010DB800000000000000000000001A +55 20010DB800000000000000000000001B +56 20010DB800000000000000000000001C +57 20010DB800000000000000000000001D +58 20010DB800000000000000000000001E +59 20010DB800000000000000000000001F +60 20010DB8000000000000000000000020 +61 20010DB8000000000000000000000021 +62 20010DB8000000000000000000000022 +63 20010DB8000000000000000000000023 +64 20010DB8000000000000000000000024 +65 20010DB8000000000000000000000025 +66 20010DB8000000000000000000000026 +67 20010DB8000000000000000000000027 +68 20010DB8000000000000000000000028 +69 20010DB8000000000000000000000029 +70 20010DB800000000000000000000002A +71 20010DB800000000000000000000002B +72 20010DB800000000000000000000002C +73 20010DB800000000000000000000002D +74 20010DB800000000000000000000002E +75 20010DB800000000000000000000002F +76 20010DB8000000000000000000000030 +77 20010DB8000000000000000000000031 +78 20010DB8000000000000000000000032 +79 20010DB8000000000000000000000033 +80 20010DB8000000000000000000000034 +81 20010DB8000000000000000000000035 +82 20010DB8000000000000000000000036 +83 20010DB8000000000000000000000037 +84 20010DB8000000000000000000000038 +85 20010DB8000000000000000000000039 +86 20010DB800000000000000000000003A +87 20010DB800000000000000000000003B +88 FF010000000000000000000000000001 +89 FF010000000000000000000000000002 +90 FF010000000000000000000000000003 +91 FF010000000000000000000000000004 +92 FF010000000000000000000000000005 +93 FF010000000000000000000000000006 +94 FF010000000000000000000000000007 +95 FF010000000000000000000000000008 +96 FF010000000000000000000000000009 +97 FF01000000000000000000000000000A +98 FF01000000000000000000000000000B +99 FF01000000000000000000000000000C +100 FF01000000000000000000000000000D + +-- !sql_not_null_string2num_or_null_ipv6_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_string2num_or_null_ipv4 -- +1 2130706433 +2 167772160 +3 184549375 +4 2886729728 +5 2887778303 +6 3232235520 +7 3232301055 +8 2851995648 +9 134744072 +10 16843009 +11 3758096384 +12 4026531839 +13 3221225984 +14 3405803776 +15 3325256704 +17 4026531840 +18 4294967295 +19 4033111141 +20 0 +21 2147483647 +22 2147483648 +23 3221225471 +24 3221225472 +25 3758096383 +26 3758096384 +27 4026531839 +28 1686077311 +29 1 +30 266 +31 532 +32 798 +33 167772165 +34 167772431 +35 167837721 +36 167903267 +37 167968813 +38 168034359 +39 168099905 +40 168165451 +41 168230997 +42 168296543 +43 168362245 +44 168427791 +45 168493337 +46 168558883 +47 168624429 +48 168689975 +49 168755521 +50 168821067 +51 2147483649 +52 2147549194 +53 2147614740 +54 2147680286 +55 2147745832 +56 2147811378 +57 2147876924 +58 2147942470 +59 2148008016 +60 2148073562 +61 2886729729 +62 2886729733 +63 2886729738 +64 2886729743 +65 2886729748 +66 2886729753 +67 2886729758 +68 2886729763 +69 3221225473 +70 3221225474 +71 3221225475 +72 3221225476 +73 3221225477 +74 3221225478 +75 3221225479 +76 3221225480 +77 3221225481 +78 3221225482 +79 3232235521 +80 3232235522 +81 3232235781 +82 3232235786 +83 3232235791 +84 3232235796 +85 3232236037 +86 3232236042 +87 3232236820 +88 3758096385 +89 3758096386 +90 3758096387 +91 3758096388 +92 3758096389 +93 3758096390 +94 3758096391 +95 3758096392 +96 3758096393 +97 3758096394 +98 3758096395 +99 3758096396 +100 3758096397 + +-- !sql_not_null_string2num_or_null_ipv4_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ipv4_compat -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv4_compat_str6 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ipv4_compat_str4 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ipv4_mapped -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv4_mapped_str6 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ipv4_mapped_str4 -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ip_address_in_range_ipv6 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 true +14 true +15 true +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ip_address_in_range_ipv4 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ip_address_in_range_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ip_address_in_range_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ip_address_in_range_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ip_address_in_range_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_is_ipv4_string -- +1 true +2 true +3 true +4 true +5 true +6 true +7 true +8 true +9 true +10 true +11 true +12 true +13 true +14 true +15 true +17 true +18 true +19 true +20 true +21 true +22 true +23 true +24 true +25 true +26 true +27 true +28 true +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 true +89 true +90 true +91 true +92 true +93 true +94 true +95 true +96 true +97 true +98 true +99 true +100 true + +-- !sql_not_null_is_ipv4_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv6_string -- +1 true +2 true +3 true +4 true +5 true +6 true +7 true +8 true +9 true +10 true +11 true +12 true +13 true +14 true +15 true +17 true +18 true +19 true +20 true +21 true +22 true +23 true +24 true +25 true +26 true +27 true +28 true +29 true +30 true +31 true +32 true +33 true +34 true +35 true +36 true +37 true +38 true +39 true +40 true +41 true +42 true +43 true +44 true +45 true +46 true +47 true +48 true +49 true +50 true +51 true +52 true +53 true +54 true +55 true +56 true +57 true +58 true +59 true +60 true +61 true +62 true +63 true +64 true +65 true +66 true +67 true +68 true +69 true +70 true +71 true +72 true +73 true +74 true +75 true +76 true +77 true +78 true +79 true +80 true +81 true +82 true +83 true +84 true +85 true +86 true +87 true +88 true +89 true +90 true +91 true +92 true +93 true +94 true +95 true +96 true +97 true +98 true +99 true +100 true + +-- !sql_not_null_is_ipv6_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv6_string -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv6_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv4_string -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_is_ipv4_string1 -- +1 false +2 false +3 false +4 false +5 false +6 false +7 false +8 false +9 false +10 false +11 false +12 false +13 false +14 false +15 false +17 false +18 false +19 false +20 false +21 false +22 false +23 false +24 false +25 false +26 false +27 false +28 false +29 false +30 false +31 false +32 false +33 false +34 false +35 false +36 false +37 false +38 false +39 false +40 false +41 false +42 false +43 false +44 false +45 false +46 false +47 false +48 false +49 false +50 false +51 false +52 false +53 false +54 false +55 false +56 false +57 false +58 false +59 false +60 false +61 false +62 false +63 false +64 false +65 false +66 false +67 false +68 false +69 false +70 false +71 false +72 false +73 false +74 false +75 false +76 false +77 false +78 false +79 false +80 false +81 false +82 false +83 false +84 false +85 false +86 false +87 false +88 false +89 false +90 false +91 false +92 false +93 false +94 false +95 false +96 false +97 false +98 false +99 false +100 false + +-- !sql_not_null_to_ipv4 -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +17 240.0.0.0 +18 255.255.255.255 +19 240.100.100.101 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 100.127.127.127 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_not_null_to_ipv6 -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +17 ::1 +18 ff00::9 +19 fd00:: +20 fc00::babe +21 ff01::f0f0 +22 ff09::1234 +23 fd00::dead +24 fd00::feed +25 fd06::dead +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_not_null_to_ipv6_or_default -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +17 ::1 +18 ff00::9 +19 fd00:: +20 fc00::babe +21 ff01::f0f0 +22 ff09::1234 +23 fd00::dead +24 fd00::feed +25 fd06::dead +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_not_null_to_ipv6_or_default_str -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_not_null_to_ipv4_or_default -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +17 240.0.0.0 +18 255.255.255.255 +19 240.100.100.101 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 100.127.127.127 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_not_null_to_ipv4_or_default_str -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_not_null_to_ipv6_or_default -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_not_null_to_ipv6_or_default_st -- +1 :: +2 :: +3 :: +4 :: +5 :: +6 :: +7 :: +8 :: +9 :: +10 :: +11 :: +12 :: +13 :: +14 :: +15 :: +17 :: +18 :: +19 :: +20 :: +21 :: +22 :: +23 :: +24 :: +25 :: +26 :: +27 :: +28 :: +29 :: +30 :: +31 :: +32 :: +33 :: +34 :: +35 :: +36 :: +37 :: +38 :: +39 :: +40 :: +41 :: +42 :: +43 :: +44 :: +45 :: +46 :: +47 :: +48 :: +49 :: +50 :: +51 :: +52 :: +53 :: +54 :: +55 :: +56 :: +57 :: +58 :: +59 :: +60 :: +61 :: +62 :: +63 :: +64 :: +65 :: +66 :: +67 :: +68 :: +69 :: +70 :: +71 :: +72 :: +73 :: +74 :: +75 :: +76 :: +77 :: +78 :: +79 :: +80 :: +81 :: +82 :: +83 :: +84 :: +85 :: +86 :: +87 :: +88 :: +89 :: +90 :: +91 :: +92 :: +93 :: +94 :: +95 :: +96 :: +97 :: +98 :: +99 :: +100 :: + +-- !sql_not_null_to_ipv4_or_default -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_not_null_to_ipv4_or_default_st -- +1 0.0.0.0 +2 0.0.0.0 +3 0.0.0.0 +4 0.0.0.0 +5 0.0.0.0 +6 0.0.0.0 +7 0.0.0.0 +8 0.0.0.0 +9 0.0.0.0 +10 0.0.0.0 +11 0.0.0.0 +12 0.0.0.0 +13 0.0.0.0 +14 0.0.0.0 +15 0.0.0.0 +17 0.0.0.0 +18 0.0.0.0 +19 0.0.0.0 +20 0.0.0.0 +21 0.0.0.0 +22 0.0.0.0 +23 0.0.0.0 +24 0.0.0.0 +25 0.0.0.0 +26 0.0.0.0 +27 0.0.0.0 +28 0.0.0.0 +29 0.0.0.0 +30 0.0.0.0 +31 0.0.0.0 +32 0.0.0.0 +33 0.0.0.0 +34 0.0.0.0 +35 0.0.0.0 +36 0.0.0.0 +37 0.0.0.0 +38 0.0.0.0 +39 0.0.0.0 +40 0.0.0.0 +41 0.0.0.0 +42 0.0.0.0 +43 0.0.0.0 +44 0.0.0.0 +45 0.0.0.0 +46 0.0.0.0 +47 0.0.0.0 +48 0.0.0.0 +49 0.0.0.0 +50 0.0.0.0 +51 0.0.0.0 +52 0.0.0.0 +53 0.0.0.0 +54 0.0.0.0 +55 0.0.0.0 +56 0.0.0.0 +57 0.0.0.0 +58 0.0.0.0 +59 0.0.0.0 +60 0.0.0.0 +61 0.0.0.0 +62 0.0.0.0 +63 0.0.0.0 +64 0.0.0.0 +65 0.0.0.0 +66 0.0.0.0 +67 0.0.0.0 +68 0.0.0.0 +69 0.0.0.0 +70 0.0.0.0 +71 0.0.0.0 +72 0.0.0.0 +73 0.0.0.0 +74 0.0.0.0 +75 0.0.0.0 +76 0.0.0.0 +77 0.0.0.0 +78 0.0.0.0 +79 0.0.0.0 +80 0.0.0.0 +81 0.0.0.0 +82 0.0.0.0 +83 0.0.0.0 +84 0.0.0.0 +85 0.0.0.0 +86 0.0.0.0 +87 0.0.0.0 +88 0.0.0.0 +89 0.0.0.0 +90 0.0.0.0 +91 0.0.0.0 +92 0.0.0.0 +93 0.0.0.0 +94 0.0.0.0 +95 0.0.0.0 +96 0.0.0.0 +97 0.0.0.0 +98 0.0.0.0 +99 0.0.0.0 +100 0.0.0.0 + +-- !sql_not_null_to_ipv6_or_null -- +1 ::1 +2 fc00:: +3 fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4 fc00:: +5 febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff +6 fe80:: +7 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8 fe80:: +9 2001:4860:4860::8888 +10 2606:4700:4700::1111 +11 ff01:: +12 ff02::1 +13 2001:db8:85a3::8a2e:370:7334 +14 2001:db8::1 +15 2001:db8::2 +17 ::1 +18 ff00::9 +19 fd00:: +20 fc00::babe +21 ff01::f0f0 +22 ff09::1234 +23 fd00::dead +24 fd00::feed +25 fd06::dead +26 ff01:: +27 ff02::1 +28 ff00:: +29 2001:db8::1 +30 2001:db8::2 +31 2001:db8::3 +32 2001:db8::4 +33 2001:db8::5 +34 2001:db8::6 +35 2001:db8::7 +36 2001:db8::8 +37 2001:db8::9 +38 2001:db8::a +39 2001:db8::b +40 2001:db8::c +41 2001:db8::d +42 2001:db8::e +43 2001:db8::f +44 2001:db8::10 +45 2001:db8::11 +46 2001:db8::12 +47 2001:db8::13 +48 2001:db8::14 +49 2001:db8::15 +50 2001:db8::16 +51 2001:db8::17 +52 2001:db8::18 +53 2001:db8::19 +54 2001:db8::1a +55 2001:db8::1b +56 2001:db8::1c +57 2001:db8::1d +58 2001:db8::1e +59 2001:db8::1f +60 2001:db8::20 +61 2001:db8::21 +62 2001:db8::22 +63 2001:db8::23 +64 2001:db8::24 +65 2001:db8::25 +66 2001:db8::26 +67 2001:db8::27 +68 2001:db8::28 +69 2001:db8::29 +70 2001:db8::2a +71 2001:db8::2b +72 2001:db8::2c +73 2001:db8::2d +74 2001:db8::2e +75 2001:db8::2f +76 2001:db8::30 +77 2001:db8::31 +78 2001:db8::32 +79 2001:db8::33 +80 2001:db8::34 +81 2001:db8::35 +82 2001:db8::36 +83 2001:db8::37 +84 2001:db8::38 +85 2001:db8::39 +86 2001:db8::3a +87 2001:db8::3b +88 ff01::1 +89 ff01::2 +90 ff01::3 +91 ff01::4 +92 ff01::5 +93 ff01::6 +94 ff01::7 +95 ff01::8 +96 ff01::9 +97 ff01::a +98 ff01::b +99 ff01::c +100 ff01::d + +-- !sql_not_null_to_ipv6_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_to_ipv4_or_null -- +1 127.0.0.1 +2 10.0.0.0 +3 10.255.255.255 +4 172.16.0.0 +5 172.31.255.255 +6 192.168.0.0 +7 192.168.255.255 +8 169.254.0.0 +9 8.8.8.8 +10 1.1.1.1 +11 224.0.0.0 +12 239.255.255.255 +13 192.0.2.0 +14 203.0.113.0 +15 198.51.100.0 +17 240.0.0.0 +18 255.255.255.255 +19 240.100.100.101 +20 0.0.0.0 +21 127.255.255.255 +22 128.0.0.0 +23 191.255.255.255 +24 192.0.0.0 +25 223.255.255.255 +26 224.0.0.0 +27 239.255.255.255 +28 100.127.127.127 +29 0.0.0.1 +30 0.0.1.10 +31 0.0.2.20 +32 0.0.3.30 +33 10.0.0.5 +34 10.0.1.15 +35 10.1.0.25 +36 10.2.0.35 +37 10.3.0.45 +38 10.4.0.55 +39 10.5.0.65 +40 10.6.0.75 +41 10.7.0.85 +42 10.8.0.95 +43 10.9.1.5 +44 10.10.1.15 +45 10.11.1.25 +46 10.12.1.35 +47 10.13.1.45 +48 10.14.1.55 +49 10.15.1.65 +50 10.16.1.75 +51 128.0.0.1 +52 128.1.0.10 +53 128.2.0.20 +54 128.3.0.30 +55 128.4.0.40 +56 128.5.0.50 +57 128.6.0.60 +58 128.7.0.70 +59 128.8.0.80 +60 128.9.0.90 +61 172.16.0.1 +62 172.16.0.5 +63 172.16.0.10 +64 172.16.0.15 +65 172.16.0.20 +66 172.16.0.25 +67 172.16.0.30 +68 172.16.0.35 +69 192.0.0.1 +70 192.0.0.2 +71 192.0.0.3 +72 192.0.0.4 +73 192.0.0.5 +74 192.0.0.6 +75 192.0.0.7 +76 192.0.0.8 +77 192.0.0.9 +78 192.0.0.10 +79 192.168.0.1 +80 192.168.0.2 +81 192.168.1.5 +82 192.168.1.10 +83 192.168.1.15 +84 192.168.1.20 +85 192.168.2.5 +86 192.168.2.10 +87 192.168.5.20 +88 224.0.0.1 +89 224.0.0.2 +90 224.0.0.3 +91 224.0.0.4 +92 224.0.0.5 +93 224.0.0.6 +94 224.0.0.7 +95 224.0.0.8 +96 224.0.0.9 +97 224.0.0.10 +98 224.0.0.11 +99 224.0.0.12 +100 224.0.0.13 + +-- !sql_not_null_to_ipv4_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_to_ipv6_or_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_to_ipv6_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_to_ipv4_or_null -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + +-- !sql_not_null_to_ipv4_or_null_str -- +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +14 \N +15 \N +17 \N +18 \N +19 \N +20 \N +21 \N +22 \N +23 \N +24 \N +25 \N +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N +51 \N +52 \N +53 \N +54 \N +55 \N +56 \N +57 \N +58 \N +59 \N +60 \N +61 \N +62 \N +63 \N +64 \N +65 \N +66 \N +67 \N +68 \N +69 \N +70 \N +71 \N +72 \N +73 \N +74 \N +75 \N +76 \N +77 \N +78 \N +79 \N +80 \N +81 \N +82 \N +83 \N +84 \N +85 \N +86 \N +87 \N +88 \N +89 \N +90 \N +91 \N +92 \N +93 \N +94 \N +95 \N +96 \N +97 \N +98 \N +99 \N +100 \N + diff --git a/regression-test/data/nereids_function_p0/test_ip.csv b/regression-test/data/nereids_function_p0/test_ip.csv new file mode 100644 index 00000000000000..00977ede70fdf1 --- /dev/null +++ b/regression-test/data/nereids_function_p0/test_ip.csv @@ -0,0 +1,100 @@ +1;127.0.0.1;::1 +2;10.0.0.0;fc00:: +3;10.255.255.255;fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +4;172.16.0.0;fe80:: +5;172.31.255.255;febf::ffff:ffff:ffff:ffff +6;192.168.0.0;ff00:: +7;192.168.255.255;ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff +8;224.0.0.0;ff01:: +9;239.255.255.255;ff02::1 +10;255.255.255.255;ff05::1 +11;169.254.0.0;ff02::2 +12;169.254.255.255;2001:db8::1 +13;192.0.2.0;2002::1 +14;198.51.100.0;ff02::5 +15;203.0.113.0;ff05::5 +16;240.0.0.0;ff00::2 +17;240.255.255.255;ff02::6 +18;100.64.0.0;2001::1 +19;100.127.255.255;2001::2 +20;192.0.0.9;ff02::1:ff00:1 +21;1.0.0.0;2001:0db8:85a3:0000:0000:8a2e:0370:7334 +22;126.255.255.255;fd00::1234:abcd +23;128.0.0.0;fd12:3456:789a:1::1 +24;191.255.255.255;ff05::1:3 +25;192.0.0.0;fe80::1234:5678:9abc:def0 +26;223.255.255.255;2001:0db8:abcd:0012:0000:0000:0000:0001 +27;240.0.0.1;ff02::7 +28;240.255.255.1;ff03::1 +29;192.88.99.1;ff04::1 +30;192.0.0.170;ff05::2 +31;192.0.0.171;ff06::1 +32;198.18.0.0;2001:0db8:85a3::8a2e:0370:7334 +33;198.19.255.255;2001:db8::abcd:ef12 +34;233.252.0.0;ff08::1 +35;233.252.0.255;ff0e::1 +36;240.0.0.10;ff0f::1 +37;172.20.10.5;fe80::1234:abcd +38;172.31.254.254;fc00::1234 +39;10.100.10.10;fd00::cafe:babe +40;10.200.200.200;fdff::feed:beef +41;100.100.100.100;fe80::5678:1234 +42;198.51.100.99;ff01::cafe +43;203.0.113.123;ff02::babe +44;8.8.8.8;2001:4860:4860::8888 +45;8.8.4.4;2001:4860:4860::8844 +46;1.1.1.1;2606:4700:4700::1111 +47;9.9.9.9;2620:fe::fe +48;3.3.3.3;2600::3 +49;4.4.4.4;2602::4 +50;255.0.0.0;ff0f::8 +51;240.0.0.8;ff0e::b +52;192.168.1.1;ff0d::1234 +53;10.10.10.10;fd12::abcd +54;192.168.2.254;ff01::5678 +55;172.16.10.10;fc00::babe +56;198.51.100.200;ff02::1234 +57;203.0.113.200;ff02::abcd +58;100.100.100.101;fd00::5678:beef +59;169.254.1.1;fe80::cafe +60;192.168.100.100;ff04::9 +61;172.20.20.20;ff0f::babe +62;10.50.50.50;fd00::50:50 +63;192.0.0.200;ff02::4321 +64;198.51.100.150;ff05::cafe +65;203.0.113.50;ff08::abcd +66;100.127.0.1;2001:db8::5678:abcd +67;10.1.1.1;fd00::a1:b1 +68;192.168.0.254;ff09::1 +69;172.16.1.1;fc01::1 +70;198.18.1.1;2001:db8::f0f0 +71;10.10.20.20;fd00::1234:5678 +72;192.168.200.200;fe80::abcd:1234 +73;100.100.10.10;ff02::9 +74;8.8.8.8;2001:4860:4860::8888 +75;1.0.0.1;2606:4700:4700::1001 +76;198.18.255.255;2001:db8::dead:beef +77;172.31.1.1;fe80::dead:beef +78;169.254.100.100;ff01::dead +79;203.0.113.255;ff05::dead +80;192.88.99.99;ff0e::feed +81;240.255.255.255;ff03::dead +82;8.8.4.4;2001:4860:4860::8844 +83;9.9.9.10;2620:fe::fe10 +84;3.4.5.6;2600::4 +85;255.255.255.255;ff00::9 +86;10.0.0.1;fd12::cafe +87;192.168.3.1;fc00::babe +88;100.100.100.102;ff05::5678 +89;169.254.2.2;fe80::feed:beef +90;198.18.200.200;ff02::1111 +91;203.0.113.123;fd00::dead:beef +92;192.88.99.10;ff0f::abcd +93;240.100.100.100;fd12::dead +94;172.20.30.40;ff01::f0f0 +95;10.100.100.100;ff09::1234 +96;198.51.100.250;fd00::dead +97;203.0.113.60;ff02::feed +98;169.254.255.254;fe80::feed +99;192.0.0.100;ff03::babe +100;100.127.127.127;ff05::4321 \ No newline at end of file diff --git a/regression-test/data/nereids_p0/ddl/show_variables/show_variables_command.out b/regression-test/data/nereids_p0/ddl/show_variables/show_variables_command.out new file mode 100644 index 00000000000000..dc009a38e77678 --- /dev/null +++ b/regression-test/data/nereids_p0/ddl/show_variables/show_variables_command.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cmd -- +license Apache License, Version 2.0 Apache License, Version 2.0 0 + +-- !cmd -- +license Apache License, Version 2.0 + +-- !cmd -- +license Apache License, Version 2.0 + diff --git a/regression-test/data/nereids_p0/ddl/show_view/show_view_command.out b/regression-test/data/nereids_p0/ddl/show_view/show_view_command.out new file mode 100644 index 00000000000000..33e1f8315bfe1a --- /dev/null +++ b/regression-test/data/nereids_p0/ddl/show_view/show_view_command.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cmd -- +ttv1 CREATE VIEW `ttv1` AS select `internal`.`test_show_view_db2`.`t1`.`c1`, `internal`.`test_show_view_db2`.`t2`.`c2` from `internal`.`test_show_view_db2`.`t1` join `internal`.`test_show_view_db2`.`t2` on `internal`.`test_show_view_db2`.`t1`.`c2` = `internal`.`test_show_view_db2`.`t2`.`c2`; +ttv2 CREATE VIEW `ttv2` AS select `internal`.`test_show_view_db2`.`t1`.`c1` from `internal`.`test_show_view_db2`.`t1` where `internal`.`test_show_view_db2`.`t1`.`c2` = 1; + +-- !cmd -- +tv1 CREATE VIEW `tv1` AS select `internal`.`test_show_view_db1`.`t1`.`c1`, `internal`.`test_show_view_db1`.`t2`.`c2` from `internal`.`test_show_view_db1`.`t1` join `internal`.`test_show_view_db1`.`t2` on `internal`.`test_show_view_db1`.`t1`.`c2` = `internal`.`test_show_view_db1`.`t2`.`c2`; +tv2 CREATE VIEW `tv2` AS select `internal`.`test_show_view_db1`.`t1`.`c1` from `internal`.`test_show_view_db1`.`t1` where `internal`.`test_show_view_db1`.`t1`.`c2` = 1; + +-- !cmd -- +ttv1 CREATE VIEW `ttv1` AS select `internal`.`test_show_view_db2`.`t1`.`c1`, `internal`.`test_show_view_db2`.`t2`.`c2` from `internal`.`test_show_view_db2`.`t1` join `internal`.`test_show_view_db2`.`t2` on `internal`.`test_show_view_db2`.`t1`.`c2` = `internal`.`test_show_view_db2`.`t2`.`c2`; +ttv2 CREATE VIEW `ttv2` AS select `internal`.`test_show_view_db2`.`t1`.`c1` from `internal`.`test_show_view_db2`.`t1` where `internal`.`test_show_view_db2`.`t1`.`c2` = 1; + +-- !cmd -- +ttv1 CREATE VIEW `ttv1` AS select `internal`.`test_show_view_db2`.`t1`.`c1`, `internal`.`test_show_view_db2`.`t2`.`c2` from `internal`.`test_show_view_db2`.`t1` join `internal`.`test_show_view_db2`.`t2` on `internal`.`test_show_view_db2`.`t1`.`c2` = `internal`.`test_show_view_db2`.`t2`.`c2`; +ttv2 CREATE VIEW `ttv2` AS select `internal`.`test_show_view_db2`.`t1`.`c1` from `internal`.`test_show_view_db2`.`t1` where `internal`.`test_show_view_db2`.`t1`.`c2` = 1; + diff --git a/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out b/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out new file mode 100644 index 00000000000000..393e13a2b546a2 --- /dev/null +++ b/regression-test/data/nereids_p0/sql_functions/table_function/posexplode.out @@ -0,0 +1,166 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 zhangsan ["Chinese", "Math", "English"] +1 lisi ["null"] +2 wangwu ["88a", "90b", "96c"] +3 lisi2 [null] +4 amory \N + +-- !explode_sql -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 amory \N \N \N + +-- !explode_sql_multi -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 1 Math +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 2 English +0 zhangsan ["Chinese", "Math", "English"] 1 Math 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math 1 Math +0 zhangsan ["Chinese", "Math", "English"] 1 Math 2 English +0 zhangsan ["Chinese", "Math", "English"] 2 English 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 2 English 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English 2 English +1 lisi ["null"] 0 null 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a 0 88a +2 wangwu ["88a", "90b", "96c"] 0 88a 1 90b +2 wangwu ["88a", "90b", "96c"] 0 88a 2 96c +2 wangwu ["88a", "90b", "96c"] 1 90b 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b 1 90b +2 wangwu ["88a", "90b", "96c"] 1 90b 2 96c +2 wangwu ["88a", "90b", "96c"] 2 96c 0 88a +2 wangwu ["88a", "90b", "96c"] 2 96c 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c 2 96c +3 lisi2 [null] 0 \N 0 \N + +-- !explode_sql_alias -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql_alias -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 amory \N \N \N + +-- !explode_sql_alias_multi -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 1 Math +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese 2 English +0 zhangsan ["Chinese", "Math", "English"] 1 Math 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math 1 Math +0 zhangsan ["Chinese", "Math", "English"] 1 Math 2 English +0 zhangsan ["Chinese", "Math", "English"] 2 English 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 2 English 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English 2 English +1 lisi ["null"] 0 null 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a 0 88a +2 wangwu ["88a", "90b", "96c"] 0 88a 1 90b +2 wangwu ["88a", "90b", "96c"] 0 88a 2 96c +2 wangwu ["88a", "90b", "96c"] 1 90b 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b 1 90b +2 wangwu ["88a", "90b", "96c"] 1 90b 2 96c +2 wangwu ["88a", "90b", "96c"] 2 96c 0 88a +2 wangwu ["88a", "90b", "96c"] 2 96c 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c 2 96c +3 lisi2 [null] 0 \N 0 \N + +-- !sql -- +0 zhangsan ["Chinese", "Math", "English"] +1 lisi ["null"] +2 wangwu ["88a", "90b", "96c"] +3 lisi2 [null] +4 liuba [] + +-- !explode_sql_not -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N + +-- !explode_outer_sql_not -- +0 zhangsan ["Chinese", "Math", "English"] 0 Chinese +0 zhangsan ["Chinese", "Math", "English"] 1 Math +0 zhangsan ["Chinese", "Math", "English"] 2 English +1 lisi ["null"] 0 null +2 wangwu ["88a", "90b", "96c"] 0 88a +2 wangwu ["88a", "90b", "96c"] 1 90b +2 wangwu ["88a", "90b", "96c"] 2 96c +3 lisi2 [null] 0 \N +4 liuba [] \N \N + +-- !explode_sql_alias_multi2 -- +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":2, "col":"English"} +1 lisi ["null"] {"pos":0, "col":"null"} {"pos":0, "col":"null"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":2, "col":"96c"} +3 lisi2 [null] {"pos":0, "col":null} {"pos":0, "col":null} + +-- !explode_sql_alias_multi3 -- +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":0, "col":"Chinese"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":1, "col":"Math"} {"pos":2, "col":"English"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":0, "col":"Chinese"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":1, "col":"Math"} +0 zhangsan ["Chinese", "Math", "English"] {"pos":2, "col":"English"} {"pos":2, "col":"English"} +1 lisi ["null"] {"pos":0, "col":"null"} {"pos":0, "col":"null"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":0, "col":"88a"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":1, "col":"90b"} {"pos":2, "col":"96c"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":0, "col":"88a"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":1, "col":"90b"} +2 wangwu ["88a", "90b", "96c"] {"pos":2, "col":"96c"} {"pos":2, "col":"96c"} +3 lisi2 [null] {"pos":0, "col":null} {"pos":0, "col":null} + diff --git a/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.out b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.out new file mode 100644 index 00000000000000..885c9ae71ccf2e --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.out @@ -0,0 +1,65 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- + +-- !query1_0_after -- + +-- !query1_4_before -- + +-- !query1_4_after -- + +-- !query1_5_before -- + +-- !query1_5_after -- + +-- !query1_1_before -- + +-- !query1_1_after -- + +-- !query1_2_before -- + +-- !query1_2_after -- + +-- !query1_3_before -- + +-- !query1_3_after -- + +-- !query2_0_before -- + +-- !query2_0_after -- + +-- !query2_1_before -- + +-- !query2_1_after -- + +-- !query2_2_before -- + +-- !query2_2_after -- + +-- !query2_3_before -- + +-- !query2_3_after -- + +-- !query2_4_before -- + +-- !query2_4_after -- + +-- !query2_5_before -- + +-- !query2_5_after -- + +-- !query3_0_before -- +4 +4 + +-- !query3_0_after -- +4 +4 + +-- !query4_0_before -- +4 +4 + +-- !query4_0_after -- +4 +4 + diff --git a/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.out b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.out new file mode 100644 index 00000000000000..a177e9422f1ea1 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.out @@ -0,0 +1,1489 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_0_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_6_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_6_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_7_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_7_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_1_before -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_1_after -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_2_before -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_2_after -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query1_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query1_4_before -- +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_4_after -- +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-07 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-08 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-08 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-09 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-10 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-10 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-11 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-12 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_5_before -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_5_after -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query2_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query2_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query2_4_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_4_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_5_before -- +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_5_after -- +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_6_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_6_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_6_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_6_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_7_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_7_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_8_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_8_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query3_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query3_4_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_4_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_5_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_5_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + diff --git a/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.out b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.out new file mode 100644 index 00000000000000..17bedeaa33fdea --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.out @@ -0,0 +1,2545 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_0_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_4_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_4_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_5_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_5_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_1_before -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-13 +4 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_1_after -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-13 +4 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query1_2_before -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_2_after -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query1_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +4 \N +4 \N +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query1_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +4 \N +4 \N +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query2_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query2_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query2_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_3_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-09 yy 2 \N +2023-12-08 2023-12-09 yy 2 \N +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-11 mm 4 \N +2023-12-10 2023-12-11 mm 4 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-11 mm 4 \N +2023-12-13 2023-12-11 mm 4 \N +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_3_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-09 yy 2 \N +2023-12-08 2023-12-09 yy 2 \N +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-11 mm 4 \N +2023-12-10 2023-12-11 mm 4 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-11 mm 4 \N +2023-12-13 2023-12-11 mm 4 \N +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_4_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_4_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query3_5_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_5_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_6_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query3_6_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query4_0_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_0_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_1_before -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_1_after -- +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_2_before -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query4_2_after -- +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query4_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_4_before -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-10 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query4_4_after -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-10 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query5_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query5_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query5_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query5_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query_5_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query5_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query5_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query5_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query6_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query6_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query6_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query6_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query6_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query6_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query6_3_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query6_3_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query6_4_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query6_4_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query7_0_before -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-10 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query7_0_after -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-10 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-11 +4 2023-12-11 +4 2023-12-13 +4 2023-12-13 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query7_7_before -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-08 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-09 +\N 2023-12-10 +\N 2023-12-10 +\N 2023-12-11 +\N 2023-12-12 +\N 2023-12-12 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-13 +4 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query7_7_after -- +\N 2023-12-07 +\N 2023-12-08 +\N 2023-12-08 +\N 2023-12-08 +\N 2023-12-09 +\N 2023-12-09 +\N 2023-12-10 +\N 2023-12-10 +\N 2023-12-11 +\N 2023-12-12 +\N 2023-12-12 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +1 2023-12-09 +2 2023-12-11 +2 2023-12-11 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-12 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +3 2023-12-13 +4 2023-12-13 +4 2023-12-13 +5 2023-12-14 +5 2023-12-14 +5 2023-12-16 +5 2023-12-16 + +-- !query7_2_before -- +\N \N 2023-12-07 +\N \N 2023-12-08 +\N \N 2023-12-08 +\N \N 2023-12-08 +\N \N 2023-12-09 +\N \N 2023-12-09 +\N \N 2023-12-10 +\N \N 2023-12-10 +\N \N 2023-12-11 +\N \N 2023-12-12 +\N \N 2023-12-12 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query7_2_after -- +\N \N 2023-12-07 +\N \N 2023-12-08 +\N \N 2023-12-08 +\N \N 2023-12-08 +\N \N 2023-12-09 +\N \N 2023-12-09 +\N \N 2023-12-10 +\N \N 2023-12-10 +\N \N 2023-12-11 +\N \N 2023-12-12 +\N \N 2023-12-12 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +1 2023-12-08 2023-12-09 +2 2023-12-09 2023-12-11 +2 2023-12-09 2023-12-11 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-12 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +3 2023-12-10 2023-12-13 +4 2023-12-11 2023-12-13 +4 2023-12-11 2023-12-13 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-14 +5 2023-12-12 2023-12-16 +5 2023-12-12 2023-12-16 + +-- !query8_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query8_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query8_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query8_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query8_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query8_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_0_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_0_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_1_before -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_1_after -- +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_2_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_2_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_3_before -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query9_3_after -- +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +1 2023-12-08 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +2 2023-12-09 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +3 2023-12-10 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 +5 2023-12-12 + +-- !query9_4_before -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-09 yy 2 \N +2023-12-08 2023-12-09 yy 2 \N +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-11 mm 4 \N +2023-12-10 2023-12-11 mm 4 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-11 mm 4 \N +2023-12-13 2023-12-11 mm 4 \N +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_4_after -- +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-07 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-08 yy 1 2 +2023-12-08 2023-12-09 yy 2 \N +2023-12-08 2023-12-09 yy 2 \N +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-09 yy 2 \N +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-09 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-10 yy 3 2 +2023-12-10 2023-12-11 mm 4 \N +2023-12-10 2023-12-11 mm 4 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-09 yy 2 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-11 2023-12-11 mm 4 \N +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-12 2023-12-12 mi 5 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-11 mm 4 \N +2023-12-13 2023-12-11 mm 4 \N +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_5_before -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_5_after -- +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-09 2023-12-08 yy 1 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-12 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-13 2023-12-10 yy 3 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-14 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 +2023-12-16 2023-12-12 mi 5 2 + +-- !query9_6_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_6_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_7_before -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + +-- !query9_7_after -- +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-08 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-09 1 yy 1 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-10 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 1 yy 3 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-12 2 mi 5 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-13 1 yy 3 2 +2023-12-14 2 mi 5 2 +2023-12-14 2 mi 5 2 +2023-12-16 2 mi 5 2 +2023-12-16 2 mi 5 2 + diff --git a/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.out b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.out new file mode 100644 index 00000000000000..b3111bed5f74a5 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.out @@ -0,0 +1,281 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +1 +1 +2 +2 +3 +3 +5 +5 + +-- !query1_0_after -- +1 +1 +2 +2 +3 +3 +5 +5 + +-- !query2_0_before -- +1 +1 +2 +2 +3 +3 +5 +5 + +-- !query2_0_after -- +1 +1 +2 +2 +3 +3 +5 +5 + +-- !query3_0_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_0_after -- +1 +1 +3 +3 +5 +5 + +-- !query3_4_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_4_after -- +1 +1 +3 +3 +5 +5 + +-- !query3_5_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_5_after -- +1 +1 +3 +3 +5 +5 + +-- !query3_1_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_1_after -- +1 +1 +3 +3 +5 +5 + +-- !query3_2_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_2_after -- +1 +1 +3 +3 +5 +5 + +-- !query3_3_before -- +1 +1 +3 +3 +5 +5 + +-- !query3_3_after -- +1 +1 +3 +3 +5 +5 + +-- !query4_0_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_0_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_1_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_1_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_2_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_2_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_3_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_3_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_4_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_4_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_5_before -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + +-- !query4_5_after -- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 + diff --git a/regression-test/data/nereids_rules_p0/mv/tpch/mv_tpch_test.out b/regression-test/data/nereids_rules_p0/mv/tpch/mv_tpch_test.out index 9ee82d64d28285..9bc3d0a99baf68 100644 --- a/regression-test/data/nereids_rules_p0/mv/tpch/mv_tpch_test.out +++ b/regression-test/data/nereids_rules_p0/mv/tpch/mv_tpch_test.out @@ -127,6 +127,30 @@ R F 3785523.00 5337950526.47 5071818532.9420 5274405503.049367 25.5259 35994.029 573861 351238.2770 1995-03-09 0 584291 354494.7318 1995-02-21 0 +-- !query3_1_before -- +108514 314967.0754 1995-02-20 0 +121604 318576.4154 1995-03-07 0 +178727 309728.9306 1995-02-25 0 +223140 355369.0698 1995-03-14 0 +405063 353125.4577 1995-03-03 0 +462502 312604.5420 1995-03-08 0 +506021 321075.5810 1995-03-10 0 +554757 349181.7426 1995-03-14 0 +573861 351238.2770 1995-03-09 0 +584291 354494.7318 1995-02-21 0 + +-- !query3_1_after -- +108514 314967.0754 1995-02-20 0 +121604 318576.4154 1995-03-07 0 +178727 309728.9306 1995-02-25 0 +223140 355369.0698 1995-03-14 0 +405063 353125.4577 1995-03-03 0 +462502 312604.5420 1995-03-08 0 +506021 321075.5810 1995-03-10 0 +554757 349181.7426 1995-03-14 0 +573861 351238.2770 1995-03-09 0 +584291 354494.7318 1995-02-21 0 + -- !query4_before -- 1-URGENT 999 2-HIGH 997 @@ -579,6 +603,50 @@ VIETNAM 1998 1924313.4862 9151 Customer#000009151 396562.0295 5691.95 IRAQ 7gIdRdaxB91EVdyx8DyPjShpMD 21-834-147-4906 ajole fluffily. furiously regular accounts are special, silent account 961 Customer#000000961 401198.1737 6963.68 JAPAN 5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn 22-989-463-6089 e final requests: busily final accounts believe a +-- !query10_1_before -- +11026 Customer#000011026 417913.4142 7738.76 ALGERIA XorIktoJOAEJkpNNMx 10-184-163-4632 ly even dolphins eat along the blithely even instructions. express attainments cajole slyly. busy dolphins in +11032 Customer#000011032 512500.9641 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again +12106 Customer#000012106 479414.2133 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas. +12595 Customer#000012595 401402.2391 -6.92 INDIA LmeaX5cR,w9NqKugl yRm98 18-186-132-3352 o the busy accounts. blithely special gifts maintain a +13478 Customer#000013478 395513.1358 -778.11 KENYA 9VIsvIeZrJpC6OOdYheMC2vdtq8Ai0Rt 24-983-202-8240 r theodolites. slyly unusual pinto beans sleep fluffily against the asymptotes. quickly r +13984 Customer#000013984 446316.5104 3482.28 IRAN qZXwuapCHvxbX 20-981-264-2952 y unusual courts could wake furiously +14299 Customer#000014299 400968.3751 6595.97 RUSSIA 7lFczTya0iM1bhEWT 32-156-618-1224 carefully regular requests. quickly ironic accounts against the ru +14398 Customer#000014398 408575.3600 -602.24 UNITED STATES GWRCgIPHajtU21vICVvbJJerFu2cUk 34-814-111-5424 s. blithely even accounts cajole blithely. even foxes doubt-- +1465 Customer#000001465 405055.3457 9365.93 INDIA tDRaTC7UgFbBX7VF6cVXYQA0 18-807-487-1074 s lose blithely ironic, regular packages. regular, final foxes haggle c +14819 Customer#000014819 396271.1036 7308.39 FRANCE w8StIbymUXmLCcUag6sx6LUIp8E3pA,Ux 16-769-398-7926 ss, final asymptotes use furiously slyly ironic dependencies. special, express dugouts according to the dep +1565 Customer#000001565 412506.0062 1820.03 BRAZIL EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW 12-402-178-2007 ously regular accounts wake slyly ironic idea +1966 Customer#000001966 444059.0382 1937.72 ALGERIA jPv1 UHra5JLALR5Isci5u0636RoAu7t vH 10-973-269-8886 the blithely even accounts. final deposits cajole around the blithely final packages. +2455 Customer#000002455 481592.4053 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro +623 Customer#000000623 399883.4257 7887.60 INDONESIA HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6 19-113-202-7085 requests. dolphins above the busily regular dependencies cajole after +7714 Customer#000007714 557400.3053 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re +8242 Customer#000008242 622786.7297 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i +8501 Customer#000008501 412797.5100 6906.70 ARGENTINA 776af4rOa mZ66hczs 11-317-552-5840 y final deposits after the fluffily even accounts are slyly final, regular +8530 Customer#000008530 457855.9467 9734.95 MOROCCO GMQyte94oDM7eD7exnkj 4hH9yq3 25-736-932-5850 slyly asymptotes. quickly final deposits in +9151 Customer#000009151 396562.0295 5691.95 IRAQ 7gIdRdaxB91EVdyx8DyPjShpMD 21-834-147-4906 ajole fluffily. furiously regular accounts are special, silent account +961 Customer#000000961 401198.1737 6963.68 JAPAN 5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn 22-989-463-6089 e final requests: busily final accounts believe a + +-- !query10_1_after -- +11026 Customer#000011026 417913.4142 7738.76 ALGERIA XorIktoJOAEJkpNNMx 10-184-163-4632 ly even dolphins eat along the blithely even instructions. express attainments cajole slyly. busy dolphins in +11032 Customer#000011032 512500.9641 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again +12106 Customer#000012106 479414.2133 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas. +12595 Customer#000012595 401402.2391 -6.92 INDIA LmeaX5cR,w9NqKugl yRm98 18-186-132-3352 o the busy accounts. blithely special gifts maintain a +13478 Customer#000013478 395513.1358 -778.11 KENYA 9VIsvIeZrJpC6OOdYheMC2vdtq8Ai0Rt 24-983-202-8240 r theodolites. slyly unusual pinto beans sleep fluffily against the asymptotes. quickly r +13984 Customer#000013984 446316.5104 3482.28 IRAN qZXwuapCHvxbX 20-981-264-2952 y unusual courts could wake furiously +14299 Customer#000014299 400968.3751 6595.97 RUSSIA 7lFczTya0iM1bhEWT 32-156-618-1224 carefully regular requests. quickly ironic accounts against the ru +14398 Customer#000014398 408575.3600 -602.24 UNITED STATES GWRCgIPHajtU21vICVvbJJerFu2cUk 34-814-111-5424 s. blithely even accounts cajole blithely. even foxes doubt-- +1465 Customer#000001465 405055.3457 9365.93 INDIA tDRaTC7UgFbBX7VF6cVXYQA0 18-807-487-1074 s lose blithely ironic, regular packages. regular, final foxes haggle c +14819 Customer#000014819 396271.1036 7308.39 FRANCE w8StIbymUXmLCcUag6sx6LUIp8E3pA,Ux 16-769-398-7926 ss, final asymptotes use furiously slyly ironic dependencies. special, express dugouts according to the dep +1565 Customer#000001565 412506.0062 1820.03 BRAZIL EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW 12-402-178-2007 ously regular accounts wake slyly ironic idea +1966 Customer#000001966 444059.0382 1937.72 ALGERIA jPv1 UHra5JLALR5Isci5u0636RoAu7t vH 10-973-269-8886 the blithely even accounts. final deposits cajole around the blithely final packages. +2455 Customer#000002455 481592.4053 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro +623 Customer#000000623 399883.4257 7887.60 INDONESIA HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6 19-113-202-7085 requests. dolphins above the busily regular dependencies cajole after +7714 Customer#000007714 557400.3053 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re +8242 Customer#000008242 622786.7297 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i +8501 Customer#000008501 412797.5100 6906.70 ARGENTINA 776af4rOa mZ66hczs 11-317-552-5840 y final deposits after the fluffily even accounts are slyly final, regular +8530 Customer#000008530 457855.9467 9734.95 MOROCCO GMQyte94oDM7eD7exnkj 4hH9yq3 25-736-932-5850 slyly asymptotes. quickly final deposits in +9151 Customer#000009151 396562.0295 5691.95 IRAQ 7gIdRdaxB91EVdyx8DyPjShpMD 21-834-147-4906 ajole fluffily. furiously regular accounts are special, silent account +961 Customer#000000961 401198.1737 6963.68 JAPAN 5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn 22-989-463-6089 e final requests: busily final accounts believe a + -- !query11_before -- 10012 4223841.21 1002 1323418.65 @@ -11305,6 +11373,20 @@ Customer#000006655 6655 29158 1995-10-21 452805.02 305.00 Customer#000011459 11459 551136 1993-05-19 386812.74 308.00 Customer#000014110 14110 565574 1995-09-24 425099.85 301.00 +-- !query18_1_before -- +Customer#000001639 1639 502886 1994-04-12 456423.88 312.00 +Customer#000001775 1775 6882 1997-04-09 408368.10 303.00 +Customer#000006655 6655 29158 1995-10-21 452805.02 305.00 +Customer#000011459 11459 551136 1993-05-19 386812.74 308.00 +Customer#000014110 14110 565574 1995-09-24 425099.85 301.00 + +-- !query18_1_after -- +Customer#000001639 1639 502886 1994-04-12 456423.88 312.00 +Customer#000001775 1775 6882 1997-04-09 408368.10 303.00 +Customer#000006655 6655 29158 1995-10-21 452805.02 305.00 +Customer#000011459 11459 551136 1993-05-19 386812.74 308.00 +Customer#000014110 14110 565574 1995-09-24 425099.85 301.00 + -- !query19_before -- 168597.2860 @@ -11431,6 +11513,104 @@ Supplier#000000920 4 Supplier#000000929 11 Supplier#000000977 13 +-- !query21_1_before -- +Supplier#000000074 8 +Supplier#000000114 6 +Supplier#000000144 12 +Supplier#000000167 9 +Supplier#000000188 12 +Supplier#000000262 10 +Supplier#000000357 14 +Supplier#000000379 6 +Supplier#000000380 11 +Supplier#000000399 14 +Supplier#000000415 12 +Supplier#000000427 7 +Supplier#000000436 6 +Supplier#000000445 16 +Supplier#000000460 10 +Supplier#000000472 12 +Supplier#000000486 10 +Supplier#000000496 14 +Supplier#000000500 6 +Supplier#000000503 7 +Supplier#000000565 8 +Supplier#000000578 9 +Supplier#000000602 11 +Supplier#000000610 7 +Supplier#000000633 12 +Supplier#000000648 8 +Supplier#000000659 11 +Supplier#000000660 6 +Supplier#000000669 10 +Supplier#000000670 7 +Supplier#000000673 9 +Supplier#000000687 9 +Supplier#000000708 12 +Supplier#000000709 15 +Supplier#000000718 10 +Supplier#000000762 15 +Supplier#000000778 10 +Supplier#000000788 6 +Supplier#000000811 7 +Supplier#000000821 11 +Supplier#000000825 16 +Supplier#000000846 6 +Supplier#000000889 12 +Supplier#000000918 8 +Supplier#000000920 4 +Supplier#000000929 11 +Supplier#000000977 13 + +-- !query21_1_after -- +Supplier#000000074 8 +Supplier#000000114 6 +Supplier#000000144 12 +Supplier#000000167 9 +Supplier#000000188 12 +Supplier#000000262 10 +Supplier#000000357 14 +Supplier#000000379 6 +Supplier#000000380 11 +Supplier#000000399 14 +Supplier#000000415 12 +Supplier#000000427 7 +Supplier#000000436 6 +Supplier#000000445 16 +Supplier#000000460 10 +Supplier#000000472 12 +Supplier#000000486 10 +Supplier#000000496 14 +Supplier#000000500 6 +Supplier#000000503 7 +Supplier#000000565 8 +Supplier#000000578 9 +Supplier#000000602 11 +Supplier#000000610 7 +Supplier#000000633 12 +Supplier#000000648 8 +Supplier#000000659 11 +Supplier#000000660 6 +Supplier#000000669 10 +Supplier#000000670 7 +Supplier#000000673 9 +Supplier#000000687 9 +Supplier#000000708 12 +Supplier#000000709 15 +Supplier#000000718 10 +Supplier#000000762 15 +Supplier#000000778 10 +Supplier#000000788 6 +Supplier#000000811 7 +Supplier#000000821 11 +Supplier#000000825 16 +Supplier#000000846 6 +Supplier#000000889 12 +Supplier#000000918 8 +Supplier#000000920 4 +Supplier#000000929 11 +Supplier#000000977 13 + -- !query22_before -- 13 94 714035.05 17 96 722560.15 diff --git a/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out b/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out new file mode 100644 index 00000000000000..b034ebe1bbdc69 --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_scan_rowsets.out @@ -0,0 +1,34 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_rowsets -- +BACKEND_ID bigint Yes false \N +ROWSET_ID varchar(64) Yes false \N +TABLET_ID bigint Yes false \N +ROWSET_NUM_ROWS bigint Yes false \N +TXN_ID bigint Yes false \N +NUM_SEGMENTS bigint Yes false \N +START_VERSION bigint Yes false \N +END_VERSION bigint Yes false \N +INDEX_DISK_SIZE bigint Yes false \N +DATA_DISK_SIZE bigint Yes false \N +CREATION_TIME datetime Yes false \N +NEWEST_WRITE_TIMESTAMP datetime Yes false \N +SCHEMA_VERSION int Yes false \N + +-- !rowsets1 -- + +-- !rowsets2 -- +0 1 +2 2 + +-- !rowsets3 -- +0 1 +2 2 +3 3 +4 4 + +-- !rowsets4 -- +2 2 +3 3 +4 4 +5 5 + diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index 276ca4daeb5f7a..812b7d2d4b7945 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_check_1 -- -102 +105 -- !select_check_2 -- internal test_table_properties_db duplicate_table _auto_bucket false @@ -29,6 +29,7 @@ internal test_table_properties_db duplicate_table row_store_page_size 16384 internal test_table_properties_db duplicate_table skip_write_index_on_load false internal test_table_properties_db duplicate_table storage_format V2 internal test_table_properties_db duplicate_table storage_medium HDD +internal test_table_properties_db duplicate_table storage_page_size 65536 internal test_table_properties_db duplicate_table store_row_column false internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000 @@ -62,6 +63,7 @@ internal test_table_properties_db listtable row_store_page_size 16384 internal test_table_properties_db listtable skip_write_index_on_load false internal test_table_properties_db listtable storage_format V2 internal test_table_properties_db listtable storage_medium HDD +internal test_table_properties_db listtable storage_page_size 65536 internal test_table_properties_db listtable store_row_column false internal test_table_properties_db listtable time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db listtable time_series_compaction_file_count_threshold 2000 @@ -95,6 +97,7 @@ internal test_table_properties_db unique_table row_store_page_size 16384 internal test_table_properties_db unique_table skip_write_index_on_load false internal test_table_properties_db unique_table storage_format V2 internal test_table_properties_db unique_table storage_medium HDD +internal test_table_properties_db unique_table storage_page_size 65536 internal test_table_properties_db unique_table store_row_column false internal test_table_properties_db unique_table time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 2000 @@ -130,6 +133,7 @@ internal test_table_properties_db duplicate_table row_store_page_size 16384 internal test_table_properties_db duplicate_table skip_write_index_on_load false internal test_table_properties_db duplicate_table storage_format V2 internal test_table_properties_db duplicate_table storage_medium HDD +internal test_table_properties_db duplicate_table storage_page_size 65536 internal test_table_properties_db duplicate_table store_row_column false internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000 @@ -163,6 +167,7 @@ internal test_table_properties_db unique_table row_store_page_size 16384 internal test_table_properties_db unique_table skip_write_index_on_load false internal test_table_properties_db unique_table storage_format V2 internal test_table_properties_db unique_table storage_medium HDD +internal test_table_properties_db unique_table storage_page_size 65536 internal test_table_properties_db unique_table store_row_column false internal test_table_properties_db unique_table time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 2000 @@ -200,6 +205,7 @@ internal test_table_properties_db duplicate_table row_store_page_size 16384 internal test_table_properties_db duplicate_table skip_write_index_on_load false internal test_table_properties_db duplicate_table storage_format V2 internal test_table_properties_db duplicate_table storage_medium HDD +internal test_table_properties_db duplicate_table storage_page_size 65536 internal test_table_properties_db duplicate_table store_row_column false internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5 internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000 diff --git a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update.out b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update.out index 8ed2e6fd900e21..e5efd74100465a 100644 --- a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update.out +++ b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update.out @@ -37,7 +37,7 @@ 1 -- !select_timestamp2 -- -11 +1 -- !select_date -- 1 @@ -86,7 +86,7 @@ B 1 -- !select_timestamp2 -- -11 +1 -- !select_date -- 1 diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update.out index 70c57c2555e58e..6c611c00030e9e 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update.out @@ -39,7 +39,7 @@ 1 -- !select_timestamp2 -- -11 +1 -- !select_date -- 1 @@ -90,7 +90,7 @@ B 1 -- !select_timestamp2 -- -11 +1 -- !select_date -- 1 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index f9532a19f8fc1e..d20517f15d5ff0 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -475,7 +475,7 @@ class Suite implements GroovyInterceptable { return res; } - String getMasterIp(Connection conn) { + String getMasterIp(Connection conn = null) { def result = sql_return_maparray_impl("select Host, QueryPort, IsMaster from frontends();", conn) logger.info("get master fe: ${result}") @@ -493,6 +493,28 @@ class Suite implements GroovyInterceptable { return masterHost; } + int getMasterPort(String type = "http") { + def result = sql_return_maparray_impl("select EditLogPort,HttpPort,QueryPort,RpcPort,ArrowFlightSqlPort from frontends() where IsMaster = 'true';") + if (result.size() != 1) { + throw new RuntimeException("could not find Master in this Cluster") + } + type = type.toLowerCase() + switch (type) { + case "editlog": + return result[0].EditLogPort as int + case "http": + return result[0].HttpPort as int + case ["query", "jdbc", "mysql"]: + return result[0].QueryPort as int + case ["rpc", "thrift"]: + return result[0].RpcPort as int + case ["arrow", "arrowflight"]: + return result[0].ArrowFlightSqlPort as int + default: + throw new RuntimeException("Unknown type: '${type}', you should select one of this type:[editlog, http, mysql, thrift, arrowflight]") + } + } + def jdbc_sql_return_maparray(String sqlStr) { return sql_return_maparray_impl(sqlStr, context.getConnection()) } @@ -1753,6 +1775,17 @@ class Suite implements GroovyInterceptable { return result.values().toList() } + // enable_sync_mv_cost_based_rewrite is true or not + boolean enable_sync_mv_cost_based_rewrite () { + def showVariable = "show variables like 'enable_sync_mv_cost_based_rewrite';" + List> result = sql(showVariable) + logger.info("enable_sync_mv_cost_based_rewrite = " + result) + if (result.isEmpty()) { + return false; + } + return Boolean.parseBoolean(result.get(0).get(1)); + } + // Given tables to decide whether the table partition row count statistic is ready or not boolean is_partition_statistics_ready(db, tables) { boolean isReady = true; @@ -1832,12 +1865,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_success = { query_sql, mv_name, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_success(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite) + mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1856,12 +1891,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_all_success = { query_sql, mv_names, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_all_success( query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1892,12 +1929,14 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite // is_partition_statistics_ready is the bool value which identifying if partition row count is valid or not // if true, check if chosen by cbo or doesn't check - def mv_rewrite_any_success = { query_sql, mv_names, sync_cbo_rewrite = true, is_partition_statistics_ready = true -> + void mv_rewrite_any_success(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite(), + is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen - return mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) + return } if (!sync_cbo_rewrite) { explain { @@ -1926,7 +1965,8 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, all rewrte success without check if chosen by cbo // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_all_success_without_check_chosen = { query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()){ logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -1934,7 +1974,9 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - success = success && result.contains("(${mv_name})") + def splitResult = result.split("MaterializedViewRewriteFail") + def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false + success = success && (result.contains("(${mv_name})") || each_result) } Assert.assertEquals(true, success) } @@ -1956,7 +1998,8 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, any of them rewrte success without check if chosen by cbo or not // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_any_success_without_check_chosen = { query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -1964,7 +2007,9 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = false; for (String mv_name : mv_names) { - success = success || result.contains("(${mv_name})") + def splitResult = result.split("MaterializedViewRewriteFail") + def each_result = splitResult.length == 2 ? splitResult[0].contains(mv_name) : false + success = success || (result.contains("(${mv_name})") || each_result) } Assert.assertEquals(true, success) } @@ -1973,7 +2018,7 @@ class Suite implements GroovyInterceptable { } explain { sql(" memo plan ${query_sql}") - check {result -> + check { result -> boolean success = false for (String mv_name : mv_names) { success = success || result.contains("${mv_name} chose") || result.contains("${mv_name} not chose") @@ -1985,12 +2030,16 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, rewrte success without check if chosen by cbo or not // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_success_without_check_chosen = { query_sql, mv_name, sync_cbo_rewrite = true -> + void mv_rewrite_success_without_check_chosen(query_sql, mv_name, + sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) if (!sync_cbo_rewrite) { explain { sql("${query_sql}") - contains("(${mv_name})") + check { result -> + def splitResult = result.split("MaterializedViewRewriteFail") + result.contains("(${mv_name})") || (splitResult.length == 2 ? splitResult[0].contains(mv_name) : false) + } } return } @@ -2004,12 +2053,12 @@ class Suite implements GroovyInterceptable { // single mv part in rewrite process, rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_fail = { query_sql, mv_name, sync_cbo_rewrite = true -> + void mv_rewrite_fail(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) if (!sync_cbo_rewrite) { explain { sql("${query_sql}") - nonContains("(${mv_name})") + notContains("(${mv_name})") } return } @@ -2021,7 +2070,7 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, all rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_all_fail = {query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_all_fail(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -2052,7 +2101,7 @@ class Suite implements GroovyInterceptable { // multi mv part in rewrite process, any rewrte fail // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite - def mv_rewrite_any_fail = {query_sql, mv_names, sync_cbo_rewrite = true -> + void mv_rewrite_any_fail (query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) if (!sync_cbo_rewrite) { explain { @@ -2091,7 +2140,7 @@ class Suite implements GroovyInterceptable { """ def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_success(query_sql, mv_name) + mv_rewrite_success(query_sql, mv_name, true) } def async_mv_rewrite_success_without_check_chosen = { db, mv_sql, query_sql, mv_name -> @@ -2107,7 +2156,7 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_success_without_check_chosen(query_sql, mv_name) + mv_rewrite_success_without_check_chosen(query_sql, mv_name, true) } @@ -2124,7 +2173,7 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) - mv_rewrite_fail(query_sql, mv_name) + mv_rewrite_fail(query_sql, mv_name, true) } def token = context.config.metaServiceToken diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IntLoadTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IntLoadTest.java new file mode 100644 index 00000000000000..7e48719e6d3a23 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/IntLoadTest.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +public class IntLoadTest { + public Integer evaluate(Integer value) { + return value == null? null: value + 1000; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringLoadTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringLoadTest.java new file mode 100644 index 00000000000000..008acb720a7898 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringLoadTest.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +public class StringLoadTest { + public String evaluate(String str) { + if (str == null) { + return null; + } + return str + "doris udf load"; + } +} diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index 625012f9a65478..c31db3d902676d 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -83,3 +83,5 @@ enable_advance_next_id = true # enable deadlock detection enable_deadlock_detection = true max_lock_hold_threshold_seconds = 10 + +force_olap_table_replication_allocation=tag.location.default:1 diff --git a/regression-test/suites/audit/test_audit_log_behavior.groovy b/regression-test/suites/audit/test_audit_log_behavior.groovy index c895abd2c3f6ce..3d24eb2e26dc9e 100644 --- a/regression-test/suites/audit/test_audit_log_behavior.groovy +++ b/regression-test/suites/audit/test_audit_log_behavior.groovy @@ -82,10 +82,13 @@ suite("test_audit_log_behavior") { sql tuple2[0] } - // make sure audit event is created. - // see WorkloadRuntimeStatusMgr.getQueryNeedAudit() - Thread.sleep(6000) - sql """call flush_audit_log()""" + if (on == true) { + // only new planner supports call flush_audit_log + // make sure audit event is created. + // see WorkloadRuntimeStatusMgr.getQueryNeedAudit() + Thread.sleep(6000) + sql """call flush_audit_log()""" + } // check result for (int i = 0; i < cnt; i++) { def tuple2 = sqls.get(i) diff --git a/regression-test/suites/auth_up_down_hive_p0/load.groovy b/regression-test/suites/auth_up_down_hive_p0/load.groovy new file mode 100644 index 00000000000000..c972d6905ce6fb --- /dev/null +++ b/regression-test/suites/auth_up_down_hive_p0/load.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_up_down_hive_prepare_auth","p0,auth,restart_fe,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "auth_up_down_hive" + String hivePrefix = "hive2"; + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "${hivePrefix}_${suiteName}_catalog" + String userName = "${hivePrefix}_${suiteName}_user" + String pwd = 'C123_567p' + + try_sql("DROP USER ${userName}") + sql """CREATE USER '${userName}' IDENTIFIED BY '${pwd}'""" + + sql """drop catalog if exists ${catalogName}""" + sql """create catalog if not exists ${catalogName} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """grant select_priv on ${catalogName}.tpch1_parquet.customer to ${userName}""" + + def res = sql """show grants for ${userName}""" + logger.info("res: " + res.toString()) + assertTrue(res.toString().contains("${catalogName}.tpch1_parquet.customer")) +} diff --git a/regression-test/suites/auth_up_down_hive_p0/test_up_down_hive_auth.groovy b/regression-test/suites/auth_up_down_hive_p0/test_up_down_hive_auth.groovy new file mode 100644 index 00000000000000..8a305bcc87d025 --- /dev/null +++ b/regression-test/suites/auth_up_down_hive_p0/test_up_down_hive_auth.groovy @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_up_down_hive_auth","p0,mtmv,restart_fe,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "auth_up_down_hive" + String hivePrefix = "hive2"; + String catalogName = "${hivePrefix}_${suiteName}_catalog" + String userName = "${hivePrefix}_${suiteName}_user" + + def res = sql """show grants for ${userName}""" + logger.info("res: " + res.toString()) + assertTrue(res.toString().contains("${catalogName}.tpch1_parquet.customer")) +} diff --git a/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy b/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy new file mode 100644 index 00000000000000..acd33a22f0c40e --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_list_cache_file") { + sql """ use @regression_cluster_name1 """ + String[][] backends = sql """ show backends """ + String backendId; + def backendIdToBackendIP = [:] + def backendIdToBackendHttpPort = [:] + def backendIdToBackendBrpcPort = [:] + for (String[] backend in backends) { + if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + backendIdToBackendIP.put(backend[0], backend[1]) + backendIdToBackendHttpPort.put(backend[0], backend[4]) + backendIdToBackendBrpcPort.put(backend[0], backend[5]) + } + } + assertEquals(backendIdToBackendIP.size(), 1) + + backendId = backendIdToBackendIP.keySet()[0] + def socket = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + + sql "drop table IF EXISTS `user`" + + sql """ + CREATE TABLE IF NOT EXISTS `user` ( + `id` int NULL, + `name` string NULL + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "file_cache_ttl_seconds" = "2884" + ) + """ + + sql "insert into user select number, cast(rand() as varchar(32)) from numbers(\"number\"=\"1000000\")" + + def get_tablets = { String tbl_name -> + def res = sql "show tablets from ${tbl_name}" + List tablets = new ArrayList<>() + for (final def line in res) { + tablets.add(Integer.valueOf(line[0].toString())) + } + return tablets + } + + def get_rowsets = { int tablet_id -> + var ret = [] + httpTest { + endpoint "" + uri socket + "/api/compaction/show?tablet_id=" + tablet_id + op "get" + check {respCode, body -> + assertEquals(respCode, 200) + var map = parseJson(body) + for (final def line in map.get("rowsets")) { + var tokens = line.toString().split(" ") + ret.add(tokens[4]) + } + } + } + return ret + } + + var tablets = get_tablets("user") + var rowsets = get_rowsets(tablets.get(0)) + var segment_file = rowsets[rowsets.size() - 1] + "_0.dat" + + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=list_cache&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200) + var arr = parseJson(body) + assertTrue(arr.size() > 0, "There shouldn't be no cache file at all, maybe you need to check disk capacity and modify file_cache_enter_disk_resource_limit_mode_percent in be.conf") + } + } + + // clear single segment file cache + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=clear&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200, "clear local cache fail, maybe you can find something in respond: " + parseJson(body)) + } + } + + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=list_cache&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200) + var arr = parseJson(body) + assertTrue(arr.size() == 0, "local cache files should not greater than 0, because it has already clear") + } + } +} diff --git a/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy b/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy new file mode 100644 index 00000000000000..4c7f6406a2abdd --- /dev/null +++ b/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.jupiter.api.Assertions; + +suite("docs/admin-manual/data-admin/backup.md", "p0,nonConcurrent") { + try { + multi_sql """ + CREATE DATABASE IF NOT EXISTS example_db; + USE example_db; + DROP TABLE IF EXISTS example_tbl; + CREATE TABLE IF NOT EXISTS example_db.example_tbl( + a INT + ) PARTITION BY RANGE(a) ( + PARTITION p1 VALUES LESS THAN (1), + PARTITION p2 VALUES LESS THAN (2) + ) DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + INSERT INTO example_tbl VALUES (1); + DROP TABLE IF EXISTS example_tbl2; + CREATE TABLE example_tbl2 LIKE example_tbl; + INSERT INTO example_tbl2 SELECT * FROM example_tbl; + """ + + def uuid = UUID.randomUUID().hashCode().abs() + def syncer = getSyncer() + /* + CREATE REPOSITORY `example_repo` + WITH HDFS + ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/" + PROPERTIES + ( + "fs.defaultFS"="hdfs://hdfs_host:port", + "hadoop.username" = "hadoop" + ); + */ + syncer.createHdfsRepository("example_repo") + + /* + CREATE REPOSITORY `s3_repo` + WITH S3 + ON LOCATION "s3://bucket_name/test" + PROPERTIES + ( + "AWS_ENDPOINT" = "http://xxxx.xxxx.com", + "AWS_ACCESS_KEY" = "xxxx", + "AWS_SECRET_KEY"="xxx", + "AWS_REGION" = "xxx" + ); + */ + syncer.createS3Repository("s3_repo") + sql """ + BACKUP SNAPSHOT example_db.snapshot_label1${uuid} + TO example_repo + ON (example_tbl) + PROPERTIES ("type" = "full"); + """ + syncer.waitSnapshotFinish("example_db") + + sql """ + BACKUP SNAPSHOT example_db.snapshot_label2${uuid} + TO example_repo + ON + ( + example_tbl PARTITION (p1,p2), + example_tbl2 + ); + """ + syncer.waitSnapshotFinish("example_db") + + sql """show BACKUP""" + sql """SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "snapshot_label1${uuid}";""" + + } catch (Throwable t) { + Assertions.fail("examples in docs/admin-manual/data-admin/backup.md failed to exec, please fix it", t) + } +} diff --git a/regression-test/suites/doc/admin-manual/data-admin/recyclebin.md.groovy b/regression-test/suites/doc/admin-manual/data-admin/recyclebin.md.groovy new file mode 100644 index 00000000000000..529584f207f8df --- /dev/null +++ b/regression-test/suites/doc/admin-manual/data-admin/recyclebin.md.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.jupiter.api.Assertions; + +suite("docs/admin-manual/data-admin/recyclebin.md", "p0,nonConcurrent") { + try { + multi_sql """ + CREATE DATABASE IF NOT EXISTS example_db; + USE example_db; + DROP TABLE IF EXISTS example_tbl; + CREATE TABLE IF NOT EXISTS example_db.example_tbl( + a INT + ) PARTITION BY RANGE(a) ( + PARTITION p1 VALUES LESS THAN (1), + PARTITION p2 VALUES LESS THAN (2) + ) DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + INSERT INTO example_tbl VALUES (1); + + ALTER TABLE example_tbl DROP PARTITION p1; + DROP TABLE example_tbl; + DROP DATABASE example_db; + """ + + sql """RECOVER DATABASE example_db;""" + sql """RECOVER TABLE example_db.example_tbl;""" + sql """RECOVER PARTITION p1 FROM example_tbl;""" + + } catch (Throwable t) { + Assertions.fail("examples in docs/admin-manual/data-admin/recyclebin.md failed to exec, please fix it", t) + } +} diff --git a/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy b/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy new file mode 100644 index 00000000000000..f36d225854bae6 --- /dev/null +++ b/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.jupiter.api.Assertions; + +suite("docs/admin-manual/data-admin/restore.md", "p0,nonConcurrent") { + try { + multi_sql """ + CREATE DATABASE IF NOT EXISTS example_db1; + USE example_db1; + DROP TABLE IF EXISTS backup_tbl; + CREATE TABLE IF NOT EXISTS example_db1.backup_tbl( + a INT + ) PARTITION BY RANGE(a) ( + PARTITION p1 VALUES LESS THAN (1), + PARTITION p2 VALUES LESS THAN (2) + ) DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + INSERT INTO backup_tbl VALUES (1); + DROP TABLE IF EXISTS backup_tbl2; + CREATE TABLE backup_tbl2 LIKE backup_tbl; + INSERT INTO backup_tbl2 SELECT * FROM backup_tbl; + """ + + def uuid = UUID.randomUUID().hashCode().abs() + def syncer = getSyncer() + syncer.createS3Repository("example_repo") + sql """ + BACKUP SNAPSHOT example_db1.snapshot_1${uuid} + TO example_repo + ON (backup_tbl) + PROPERTIES ("type" = "full"); + """ + syncer.waitSnapshotFinish("example_db1") + sql """ + BACKUP SNAPSHOT example_db1.snapshot_2${uuid} + TO example_repo + ON (backup_tbl, backup_tbl2) + PROPERTIES ("type" = "full"); + """ + syncer.waitSnapshotFinish("example_db1") + + multi_sql """ + truncate table backup_tbl; + truncate table backup_tbl2; + """ + + var timestamp = syncer.getSnapshotTimestamp("example_repo", "snapshot_1${uuid}") + sql """ + RESTORE SNAPSHOT example_db1.`snapshot_1${uuid}` + FROM `example_repo` + ON ( `backup_tbl` ) + PROPERTIES + ( + "backup_timestamp"="${timestamp}", + "replication_num" = "1" + ); + """ + syncer.waitAllRestoreFinish("example_db1") + + var timestamp2 = syncer.getSnapshotTimestamp("example_repo", "snapshot_2${uuid}") + sql """ + RESTORE SNAPSHOT example_db1.`snapshot_2${uuid}` + FROM `example_repo` + ON + ( + `backup_tbl` PARTITION (`p1`, `p2`), + `backup_tbl2` AS `new_tbl` + ) + PROPERTIES + ( + "backup_timestamp"="${timestamp2}", + "replication_num" = "1" + ); + """ + syncer.waitAllRestoreFinish("example_db1") + + sql """SHOW RESTORE""" + + } catch (Throwable t) { + Assertions.fail("examples in docs/admin-manual/data-admin/restore.md failed to exec, please fix it", t) + } +} diff --git a/regression-test/suites/doc/data-operate/import/import-way/client_local.csv b/regression-test/suites/doc/data-operate/import/import-way/client_local.csv new file mode 100644 index 00000000000000..f674458778eed2 --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/client_local.csv @@ -0,0 +1,6 @@ +1,10 +2,20 +3,30 +4,40 +5,50 +6,60 diff --git a/regression-test/suites/doc/data-operate/import/import-way/data.csv b/regression-test/suites/doc/data-operate/import/import-way/data.csv new file mode 100644 index 00000000000000..9ce46086f69e5b --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/data.csv @@ -0,0 +1,2 @@ +6,Amy,60 +7,Ross,98 diff --git a/regression-test/suites/doc/data-operate/import/import-way/group-commit-manual.md.groovy b/regression-test/suites/doc/data-operate/import/import-way/group-commit-manual.md.groovy new file mode 100644 index 00000000000000..2f082e12f88c35 --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/group-commit-manual.md.groovy @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +import org.junit.jupiter.api.Assertions + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.Statement + +suite("docs/data-operate/import/import-way/group-commit-manual.md", "p0,nonConcurrent") { + try { + sql "CREATE DATABASE IF NOT EXISTS `db`; use `db`;" + sql "DROP TABLE IF EXISTS `dt`;" + sql """ + CREATE TABLE `dt` ( + `id` int(11) NOT NULL, + `name` varchar(50) NULL, + `score` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + Demo.HOST = getMasterIp() + Demo.PORT = getMasterPort("mysql") + Demo.USER = context.config.jdbcUser + Demo.PASSWD = context.config.jdbcPassword + + Demo.groupCommitInsert() + Demo.groupCommitInsertBatch() + + + sql "truncate table dt;" + multi_sql """ + set group_commit = async_mode; + insert into dt values(1, 'Bob', 90), (2, 'Alice', 99); + insert into dt(id, name) values(3, 'John'); + select * from dt; + select * from dt; + """ + + multi_sql """ + set group_commit = sync_mode; + insert into dt values(4, 'Bob', 90), (5, 'Alice', 99); + select * from dt; + """ + + sql "set group_commit = off_mode;" + cmd """curl --location-trusted -u ${context.config.jdbcUser}:${context.config.jdbcPassword} -T ${context.file.parent}/data.csv -H "group_commit:async_mode" -H "column_separator:," http://${context.config.feHttpAddress}/api/db/dt/_stream_load""" + cmd """curl --location-trusted -u ${context.config.jdbcUser}:${context.config.jdbcPassword} -T ${context.file.parent}/data.csv -H "group_commit:sync_mode" -H "column_separator:," http://${context.config.feHttpAddress}/api/db/dt/_stream_load""" + cmd """curl --location-trusted -u ${context.config.jdbcUser}:${context.config.jdbcPassword} -T ${context.file.parent}/data.csv -H "group_commit:async_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')" http://${context.config.feHttpAddress}/api/_http_stream""" + cmd """curl --location-trusted -u ${context.config.jdbcUser}:${context.config.jdbcPassword} -T ${context.file.parent}/data.csv -H "group_commit:sync_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')" http://${context.config.feHttpAddress}/api/_http_stream""" + + sql """ALTER TABLE dt SET ("group_commit_interval_ms" = "2000");""" + sql """ALTER TABLE dt SET ("group_commit_data_bytes" = "134217728");""" + } catch (Throwable t) { + Assertions.fail("examples in docs/data-operate/import/import-way/group-commit-manual.md failed to exec, please fix it", t) + } +} + +class Demo { + static String JDBC_DRIVER = "com.mysql.jdbc.Driver"; + static String URL_PATTERN = "jdbc:mysql://%s:%d/%s?useServerPrepStmts=true"; + static String HOST = "127.0.0.1"; + static int PORT = 9030; + static String DB = "db"; + static String TBL = "dt"; + static String USER = "root"; + static String PASSWD = ""; + static int INSERT_BATCH_SIZE = 10; + + static final void groupCommitInsert() throws Exception { + Class.forName(JDBC_DRIVER); + Connection conn = DriverManager.getConnection(String.format(URL_PATTERN, HOST, PORT, DB), USER, PASSWD) + try { + // set session variable 'group_commit' + Statement statement = conn.createStatement() + try { + statement.execute("SET group_commit = async_mode;"); + } finally { + statement.close() + } + + String query = "insert into " + TBL + " values(?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(query) + try { + for (int i = 0; i < INSERT_BATCH_SIZE; i++) { + stmt.setInt(1, i); + stmt.setString(2, "name" + i); + stmt.setInt(3, i + 10); + int result = stmt.executeUpdate(); + System.out.println("rows: " + result); + } + } finally { + stmt.close() + } + } catch (Exception e) { + e.printStackTrace(); + throw e + } finally { + conn.close() + } + } + + static final void groupCommitInsertBatch() throws Exception { + Class.forName(JDBC_DRIVER); + // add rewriteBatchedStatements=true and cachePrepStmts=true in JDBC url + // set session variables by sessionVariables=group_commit=async_mode in JDBC url + Connection conn = DriverManager.getConnection( + String.format(URL_PATTERN + "&rewriteBatchedStatements=true&cachePrepStmts=true&sessionVariables=group_commit=async_mode", HOST, PORT, DB), USER, PASSWD) + try { + String query = "insert into " + TBL + " values(?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(query) + try { + for (int j = 0; j < 5; j++) { + // 10 rows per insert + for (int i = 0; i < INSERT_BATCH_SIZE; i++) { + stmt.setInt(1, i); + stmt.setString(2, "name" + i); + stmt.setInt(3, i + 10); + stmt.addBatch(); + } + int[] result = stmt.executeBatch(); + } + } finally { + stmt.close() + } + } catch (Exception e) { + e.printStackTrace(); + throw e + } finally { + conn.close() + } + } +} diff --git a/regression-test/suites/doc/data-operate/import/import-way/insert-into-manual.md.groovy b/regression-test/suites/doc/data-operate/import/import-way/insert-into-manual.md.groovy new file mode 100644 index 00000000000000..625d66f4682c8e --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/insert-into-manual.md.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.jupiter.api.Assertions; + +suite("docs/data-operate/import/import-way/insert-into-manual.md") { + try { + multi_sql """ + CREATE DATABASE IF NOT EXISTS testdb; + DROP TABLE IF EXISTS testdb.test_table; + """ + sql """ + CREATE TABLE testdb.test_table( + user_id BIGINT NOT NULL COMMENT "用户 ID", + name VARCHAR(20) COMMENT "用户姓名", + age INT COMMENT "用户年龄" + ) + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + """ + sql """ + INSERT INTO testdb.test_table (user_id, name, age) + VALUES (1, "Emily", 25), + (2, "Benjamin", 35), + (3, "Olivia", 28), + (4, "Alexander", 60), + (5, "Ava", 17); + """ + qt_sql "SELECT COUNT(*) FROM testdb.test_table;" + + sql "DROP TABLE IF EXISTS testdb.test_table2;" + sql "CREATE TABLE testdb.test_table2 LIKE testdb.test_table;" + sql """ + INSERT INTO testdb.test_table2 + SELECT * FROM testdb.test_table WHERE age < 30; + """ + qt_sql "SELECT COUNT(*) FROM testdb.test_table2;" + sql "SHOW LOAD FROM testdb;" + + + multi_sql """ + CREATE TABLE IF NOT EXISTS empty_tbl (k1 VARCHAR(32)) PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl1 LIKE empty_tbl; + CREATE TABLE IF NOT EXISTS tbl2 LIKE empty_tbl; + """ + sql "INSERT INTO tbl1 SELECT * FROM empty_tbl;" + sql "CLEAN LABEL FROM ${curDbName}" + multi_sql """ + INSERT INTO tbl1 SELECT * FROM tbl2; + INSERT INTO tbl1 WITH LABEL my_label1 SELECT * FROM tbl2; + INSERT INTO tbl1 SELECT * FROM tbl2; + INSERT INTO tbl1 SELECT * FROM tbl2; + """ + sql """SHOW LOAD WHERE label="xxx";""" + try { + """SHOW TRANSACTION WHERE id=4005;""" + } catch (Exception e) { + if (!e.getMessage().contains("transaction with id 4005 does not exist")) { + logger.error("this sql should not throw other error") + throw e + } + } + sql """INSERT INTO tbl1 SELECT * FROM tbl2 WHERE k1 = "a";""" + if (!isCloudMode()) { + // skip this case if this is a cloud cluster + try { + sql "INSERT INTO tbl1 SELECT LPAD('foo', 100, 'bar');" + Assertions.fail("this sql should fail, because we want get err url ") + } catch (Exception e) { + var msg = e.getMessage() + var err_url = msg.substring(msg.lastIndexOf("http")) + sql """SHOW LOAD WARNINGS ON "${err_url}";""" + } + } + } catch (Throwable t) { + Assertions.fail("examples in docs/data-operate/import/import-way/insert-into-manual.md failed to exec, please fix it", t) + } +} diff --git a/regression-test/suites/doc/data-operate/import/import-way/mysql-load-manual.md.groovy b/regression-test/suites/doc/data-operate/import/import-way/mysql-load-manual.md.groovy new file mode 100644 index 00000000000000..9b350402aafeee --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/mysql-load-manual.md.groovy @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.jupiter.api.Assertions + +suite("docs/data-operate/import/import-way/mysql-load-manual.md") { + def is_linux = System.getProperty("os.name").toLowerCase().contains("linux") + def run_cmd = { String cmdText -> + try { + println(cmd cmdText) + return true + } catch (Exception ignored) { + return false + } + } + if (!is_linux) { + logger.warn("don't run this case if not in Linux") + return + } + logger.info("check if has installed mysql cmd") + if (run_cmd("mysql --help 2>&1 >/dev/null") || run_cmd("yum -y install mysql")) { + logger.info("mysql cmd can work properly, go continue") + } else { + logger.warn("could not install mysql cmd client, skip this case") + return + } + def writeToFile = {String path, String data -> + OutputStreamWriter w = null + try { + w = new OutputStreamWriter(new FileOutputStream(path)) + w.write(data) + w.flush() + w.close() + } finally { + if (w != null) w.close() + } + } + def load_local = {String sql -> + if (is_linux) { + var output = cmd """ + cd ${context.file.parent} && \\ + cat << EOF | mysql --local-infile -vvv -h ${getMasterIp()} -P ${getMasterPort("mysql")} -u ${context.config.jdbcUser} ${context.config.jdbcPassword.isEmpty() ? "" : "-p ${context.config.jdbcPassword}"} -D testdb +${sql} +EOF + """ + println(output) + } + } + + try { + multi_sql """ + CREATE DATABASE IF NOT EXISTS testdb; + USE testdb; + DROP TABLE IF EXISTS t1; + """ + sql """ + CREATE TABLE testdb.t1 ( + pk INT, + v1 INT SUM + ) AGGREGATE KEY (pk) + DISTRIBUTED BY hash (pk) + PROPERTIES ("replication_num" = "1"); + """ + load_local """ + LOAD DATA LOCAL + INFILE 'client_local.csv' + INTO TABLE testdb.t1 + COLUMNS TERMINATED BY ',' + LINES TERMINATED BY '\\n'; + """ + + try { + sql """show load warnings where label='b612907c-ccf4-4ac2-82fe-107ece655f0f';""" + } catch (Exception e) { + if (!e.getMessage().contains("job is not exist")) { + logger.error("occurring other error is not in expected") + throw e + } + } + + sql "CREATE DATABASE IF NOT EXISTS testDb" + sql "DROP TABLE IF EXISTS testDb.testTbl" + sql """ + CREATE TABLE testDb.testTbl ( + k1 INT, + k2 INT, + v1 INT SUM + ) AGGREGATE KEY (k1,k2) + PARTITION BY RANGE(k1) ( + PARTITION p1 VALUES LESS THAN (1), + PARTITION p2 VALUES LESS THAN (2), + PARTITION p3 VALUES LESS THAN (3) + ) + DISTRIBUTED BY hash (k1) + PROPERTIES ("replication_num" = "1"); + """ + writeToFile("${context.file.parent}/testData", "1\t2\t3\n") + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + PROPERTIES ("timeout"="100"); + """ + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + PROPERTIES ("max_filter_ratio"="0.2"); + """ + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + (k2, k1, v1); + """ + writeToFile("${context.file.parent}/testData", "1,2,3\n") + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + COLUMNS TERMINATED BY ',' + LINES TERMINATED BY '\\n'; + """ + writeToFile("${context.file.parent}/testData", "1\t2\t3\n") + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + PARTITION (p1, p2); + """ + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + PROPERTIES ("timezone"="Africa/Abidjan"); + """ + load_local """ + LOAD DATA LOCAL + INFILE 'testData' + INTO TABLE testDb.testTbl + PROPERTIES ("exec_mem_limit"="10737418240"); + """ + } catch (Throwable t) { + Assertions.fail("examples in docs/data-operate/import/import-way/mysql-load-manual.md failed to exec, please fix it", t) + } +} diff --git a/regression-test/suites/doc/data-operate/import/import-way/routine-load-manual.md.groovy b/regression-test/suites/doc/data-operate/import/import-way/routine-load-manual.md.groovy new file mode 100644 index 00000000000000..b4327a4cb8346e --- /dev/null +++ b/regression-test/suites/doc/data-operate/import/import-way/routine-load-manual.md.groovy @@ -0,0 +1,1019 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_routine_load_doc_case","p0") { + def kafkaCsvTopics = [ + "test_rl_csv", + "test_rl_max_filter_ratio", + "test_rl_partition", + "test_rl_delete", + "test_rl_column_mapping", + "test_rl_hll" + ] + + def kafkaJsonTopics = [ + "test_rl_json", + "test_rl_json_path", + "test_rl_json_root", + "test_rl_array", + "test_rl_map", + "test_rl_bitmap" + ] + + def jsonpaths = [ + '[\"$.id\",\"$.name\",\"$.age\"]', + '[\"$.name\",\"$.id\",\"$.num\",\"$.age\"]', + ] + + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // define kafka + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // Create kafka producer + def producer = new KafkaProducer<>(props) + + for (String kafkaCsvTopic in kafkaCsvTopics) { + def txt = new File("""${context.config.dataPath}/doc/data-operate/import/import-way/${kafkaCsvTopic}.csv""").text + def lines = txt.readLines() + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(kafkaCsvTopic, null, line) + producer.send(record) + } + } + for (String kafkaJsonTopic in kafkaJsonTopics) { + def kafkaJson = new File("""${context.config.dataPath}/doc/data-operate/import/import-way/${kafkaJsonTopic}.json""").text + def lines = kafkaJson.readLines() + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(kafkaJsonTopic, null, line) + producer.send(record) + } + } + + // case1: load csv + def tableName = "test_routine_load_doc_case" + def jobName = "test_routine_load_doc_case_job" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + user_id BIGINT NOT NULL COMMENT "用户 ID", + name VARCHAR(20) COMMENT "用户姓名", + age INT COMMENT "用户年龄" + ) + UNIQUE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(user_id, name, age) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql1 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case2: load json + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS(user_id,name,age) + PROPERTIES( + "format"="json", + "jsonpaths"='${jsonpaths[0]}' + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case3: alter routine load + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(user_id, name, age) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "pause routine load for ${jobName}" + sql """ + ALTER ROUTINE LOAD FOR ${jobName} + PROPERTIES( + "desired_concurrent_number" = "3" + ) + FROM KAFKA( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "test-topic" + ); + """ + sql "stop routine load for ${jobName}" + + //case4: max_filter_ratio + def tableName1 = "test_routine_load_doc_case1" + sql """ DROP TABLE IF EXISTS ${tableName1} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName1} ( + id INT NOT NULL COMMENT "User ID", + name VARCHAR(30) NOT NULL COMMENT "Name", + age INT COMMENT "Age" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName1} + COLUMNS TERMINATED BY "," + PROPERTIES + ( + "max_filter_ratio"="0.5", + "max_error_number" = "100", + "strict_mode" = "true" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[1]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName1}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + log.info("url: ${state[0][18].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql4 "select * from ${tableName1} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName1} """ + } + + //case5: kafka_offsets + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "kafka_partitions" = "0", + "kafka_offsets" = "3" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql5 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case6: group.id and client.id + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.group.id" = "kafka_job03", + "property.client.id" = "kafka_client_03", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql6 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case7: filter + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + WHERE user_id >= 3 + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql7 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case8: Loading specified partition data + def tableName2 = "test_routine_load_doc_case2" + sql """ DROP TABLE IF EXISTS ${tableName2} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName2} ( + id INT NOT NULL COMMENT "User ID", + name VARCHAR(30) NOT NULL COMMENT "Name", + age INT COMMENT "Age", + date DATETIME COMMENT "Date" + ) + DUPLICATE KEY(`id`) + PARTITION BY RANGE(`id`) + (PARTITION partition_a VALUES [("0"), ("1")), + PARTITION partition_b VALUES [("1"), ("2")), + PARTITION partition_c VALUES [("2"), ("3"))) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName2} + COLUMNS TERMINATED BY ",", + PARTITION(partition_b) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[2]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName2}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql8 "select * from ${tableName2} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName2} """ + } + + // case9: timezone + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName2} + COLUMNS TERMINATED BY "," + PROPERTIES + ( + "timezone" = "Asia/Shanghai" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[2]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName2}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql9 "select * from ${tableName2} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName2} """ + } + + // case10: merge delete + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(user_id, name, age) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + sql "stop routine load for ${jobName}" + sql "sync" + + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + WITH DELETE + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[3]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql10 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case11: delete on + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + WITH MERGE + COLUMNS TERMINATED BY ",", + DELETE ON user_id = 2 + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql11 "select * from ${tableName} order by user_id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case12: Load with column mapping and derived column calculation + tableName = "test_routine_load_doc_case3" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age", + num INT COMMENT "number" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(id, name, age, num=age*10) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[4]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql12 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case13: test json + tableName = "routine_test12" + jobName = "kafka_job12" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + PROPERTIES + ( + "format" = "json" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql13 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case14: json path + tableName = "routine_test13" + jobName = "kafka_job13" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age", + num INT COMMENT "num" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS(name, id, num, age) + PROPERTIES + ( + "format" = "json", + "jsonpaths"='${jsonpaths[1]}' + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[1]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql14 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case15: json root + tableName = "routine_test14" + jobName = "kafka_job14" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + PROPERTIES + ( + "format" = "json", + "json_root" = "\$.source" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[2]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql15 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case16: array + tableName = "routine_test16" + jobName = "kafka_job16" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age", + array ARRAY NULL COMMENT "test array column" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + PROPERTIES + ( + "format" = "json" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[3]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql16 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case17: map + tableName = "routine_test17" + jobName = "kafka_job17" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age", + map Map NULL COMMENT "test column" + ) + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + PROPERTIES + ( + "format" = "json" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[4]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql17 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case18: bitmap + tableName = "routine_test18" + jobName = "kafka_job18" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id INT NOT NULL COMMENT "id", + name VARCHAR(30) NOT NULL COMMENT "name", + age INT COMMENT "age", + bitmap_id INT COMMENT "test", + device_id BITMAP BITMAP_UNION COMMENT "test column" + ) + AGGREGATE KEY (`id`,`name`,`age`,`bitmap_id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS(id, name, age, bitmap_id, device_id=to_bitmap(bitmap_id)) + PROPERTIES + ( + "format" = "json" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaJsonTopics[4]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql18 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + // case19: hll + tableName = "routine_test19" + jobName = "kafka_job19" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + dt DATE, + id INT, + name VARCHAR(10), + province VARCHAR(10), + os VARCHAR(10), + pv hll hll_union + ) + Aggregate KEY (dt,id,name,province,os) + distributed by hash(id) buckets 10 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(dt, id, name, province, os, pv=hll_hash(id)) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[5]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + log.info("url: ${state[0][18].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql19 "select * from ${tableName} order by id" + } finally { + sql "stop routine load for ${jobName}" + sql """ truncate table ${tableName} """ + } + + //case 19: Single-task Loading to Multiple Tables + try { + sql """ + CREATE ROUTINE LOAD ${jobName} + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[5]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + } finally { + sql "stop routine load for ${jobName}" + } + + //case20: strict mode + try { + sql """ + CREATE ROUTINE LOAD ${jobName} + PROPERTIES + ( + "strict_mode" = "true" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTopics[5]}" + ); + """ + } finally { + sql "stop routine load for ${jobName}" + } + } +} \ No newline at end of file diff --git a/regression-test/suites/doc/table-design/data-model/unique.md.groovy b/regression-test/suites/doc/table-design/data-model/unique.md.groovy index 7c5e7783b065e9..bceb684454b347 100644 --- a/regression-test/suites/doc/table-design/data-model/unique.md.groovy +++ b/regression-test/suites/doc/table-design/data-model/unique.md.groovy @@ -32,14 +32,15 @@ suite("docs/table-design/data-model/unique.md") { `register_time` DATETIME COMMENT "用户注册时间" ) UNIQUE KEY(`user_id`, `username`) - DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 + DISTRIBUTED BY HASH(`user_id`) BUCKETS 10 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 3" ); """ multi_sql """ - CREATE TABLE IF NOT EXISTS example_tbl_unique_merge_on_write + DROP TABLE example_tbl_unique; + CREATE TABLE IF NOT EXISTS example_tbl_unique ( `user_id` LARGEINT NOT NULL COMMENT "用户id", `username` VARCHAR(50) NOT NULL COMMENT "用户昵称", @@ -51,10 +52,10 @@ suite("docs/table-design/data-model/unique.md") { `register_time` DATETIME COMMENT "用户注册时间" ) UNIQUE KEY(`user_id`, `username`) - DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 + DISTRIBUTED BY HASH(`user_id`) BUCKETS 10 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "enable_unique_key_merge_on_write" = "true" + "replication_allocation" = "tag.location.default: 3", + "enable_unique_key_merge_on_write" = "false" ); """ } catch (Throwable t) { diff --git a/regression-test/suites/export_p0/test_outfile.groovy b/regression-test/suites/export_p0/test_outfile.groovy index 8b60803e185818..56ede909514332 100644 --- a/regression-test/suites/export_p0/test_outfile.groovy +++ b/regression-test/suites/export_p0/test_outfile.groovy @@ -201,17 +201,22 @@ suite("test_outfile") { `name` varchar(30) ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 2 + DISTRIBUTED BY HASH(`id`) BUCKETS 16 PROPERTIES ( "replication_allocation" = "tag.location.default: 1" );""" sql """insert into select_into_file values(1, "b"),(2, "z"),(3, "a"), (4, "c"), (5, "睿"), (6, "多"), (7, "丝"), (8, "test"), - (100, "aa"), (111, "bb"), (123, "cc"), (222, "dd");""" + (100, "aa"), (111, "bb"), (123, "cc"), (222, "dd"),(1, "b"),(2, "z"),(3, "a"), + (44, "c"), (55, "睿"), (66, "多"), (77, "丝"), (88, "test"), + (1000, "aa"), (1111, "bb"), (1234, "cc"), (2222, "dd");""" sql "set enable_parallel_outfile = true;" - sql """select * from select_into_file into outfile "file://${outFilePath}/";""" - - sql """select * from select_into_file into outfile "file://${outFilePath}/" properties("success_file_name" = "SUCCESS");""" + sql "set parallel_pipeline_task_num=4;" + def result = sql """select * from select_into_file into outfile "file://${outFilePath}/";""" + assertEquals(4, result.size()) + sql "set parallel_pipeline_task_num=8;" + result = sql """select * from select_into_file into outfile "file://${outFilePath}/" properties("success_file_name" = "SUCCESS");""" + assertEquals(8, result.size()) } finally { try_sql("DROP TABLE IF EXISTS select_into_file") File path = new File(outFilePath) diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index d2478aeadd262a..a73df1ed5e1096 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -214,6 +214,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_5_26 """select test6 from test2;""" order_qt_sql_5_27 """select * from composite_type_array order by name;""" order_qt_sql_5_28 """select * from test3_20231005;""" + order_qt_sql_5_29 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_5_30 """select test1, test2 from test1 where test2 like 'text%';""" sql """switch test_es_query_es6""" // order_qt_sql_6_01 """show tables""" @@ -244,6 +246,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_6_26 """select test6 from test2;""" order_qt_sql_6_27 """select * from composite_type_array order by name;""" order_qt_sql_6_28 """select * from test3_20231005;""" + order_qt_sql_6_29 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_6_30 """select test1, test2 from test1 where test2 like 'text%';""" List> tables6N = sql """show tables""" boolean notContainHide = true @@ -299,6 +303,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_7_32 """select test6 from test1;""" order_qt_sql_7_33 """select test6 from test2;""" order_qt_sql_7_34 """select * from composite_type_array order by name;""" + order_qt_sql_7_35 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_7_36 """select test1, test2 from test1 where test2 like 'text%';""" List> tables7N = sql """show tables""" boolean notContainHide7 = true @@ -354,7 +360,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_8_30 """select test6 from test1;""" order_qt_sql_8_31 """select test6 from test2;""" order_qt_sql_8_32 """select * from composite_type_array order by name;""" - + order_qt_sql_8_33 """select test1, test2 from test1 where test1 like 'string%';""" + order_qt_sql_8_34 """select test1, test2 from test1 where test2 like 'text%';""" + } sql """set enable_es_parallel_scroll=true""" diff --git a/regression-test/suites/external_table_p0/hive/test_orc_tiny_stripes.groovy b/regression-test/suites/external_table_p0/hive/test_orc_tiny_stripes.groovy new file mode 100644 index 00000000000000..bc585340fccb54 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_orc_tiny_stripes.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_orc_tiny_stripes", "p0,external,hive,external_docker,external_docker_hive") { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + + } + + for (String hivePrefix : ["hive2"]) { + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_orc_tiny_stripes" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """use `${catalog_name}`.`default`""" + + + + + def orc_configs = [ + [0,0,0], + [0,10230,1024], + [1,1,1], + [201,130,0], + [1024,1024,0], + [1024,1024,1024], + [4096,1024,0], + [1024,4096,0], + [1,10240,10000000], + [1000000,888888888,0], + [1000000000000,1000000000000,100000000000] + ] + def li = [ "set enable_orc_lazy_materialization=true;","set enable_orc_lazy_materialization=false;"] + + + li.each { it1 -> + sql it1 + + orc_configs.each { it2 -> + def value1 = it2[0].toString() + def value2 = it2[1].toString() + def value3 = it2[2].toString() + + sql "set orc_tiny_stripe_threshold_bytes = " + value1 + ";" + sql "set orc_once_max_read_bytes = " + value2 + ";" + sql "set orc_max_merge_distance_bytes = " + value3 + ";" + + + qt_test_1 """ select count(*) from orc_tiny_stripes; """ //372 + +/* +*/ + + qt_test_2 """ select * from orc_tiny_stripes where col1 = 1 order by col1,col2,col3; """ +/* +1 str_1 10000000001 +1 str_1 10000000001 +*/ + qt_test_3 """ select * from orc_tiny_stripes where col1%100 = 0 order by col1,col2,col3 ; """ +/* +0 str_0 10000000000 +0 str_0 10000000000 +100 9DPJaFc00euBteqiW1f1 10000000027 +100 str_100 10000000100 +2200 tQ7BRFEuf8h56kahqsLPa1vu 10000000034 +4800 TaWGgh4iZ 10000000115 +5700 SwOaGJj9fVbk5j0Np 10000000050 +*/ + + qt_test_4 """ select * from orc_tiny_stripes where col2 = "str_4" order by col1,col2,col3; """ +/* +4 str_4 10000000004 +4 str_4 10000000004 +*/ + qt_test_5 """ select count(*) from orc_tiny_stripes where col3 > 10000000005; """ //348 + qt_test_6 """ select * from orc_tiny_stripes where col3 in ( 10000000005,10000000053,10000000146) order by col1,col2,col3 ; """ +/* +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 +*/ + + qt_test_7 """ select * from orc_tiny_stripes where col3 in ( 10000000005,10000000053,10000000146) order by col1,col2,col3 ; """ +/* +5 str_5 10000000005 +5 str_5 10000000005 +53 str_53 10000000053 +146 str_146 10000000146 +3961 hMgIY4oui0MHYgaIFg4zz5Ti3p 10000000053 +4129 qwPIwtkTZb 10000000005 +4942 vAdLpLUN3VkGNmTjvuPv 10000000053 +5349 koTeYPr2Qaqqnlk07X 10000000146 +5745 1cx1jZ6QGRWAkskiOgURj6dscYxDOl 10000000005 +7573 e3lIPwNnbG6DPmog 10000000005 +8614 TtyopDvRptLB5 10000000005 +*/ + + qt_test_8 """ select col3 from orc_tiny_stripes where col3 in ( 10000000005,10000000053,10000000146) order by col3 ; """ +/* +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000005 +10000000053 +10000000053 +10000000053 +10000000146 +10000000146 +*/ + + qt_test_9 """ select col1 from orc_tiny_stripes where col1 in (10,1000) order by col1 ; """ // 10 + qt_test_10 """ select col2 from orc_tiny_stripes where length(col2) > 29 order by col2 ; """ +/* +1cx1jZ6QGRWAkskiOgURj6dscYxDOl +Asn3tnIg1xYm8Lbgey8baqw3EmooFm +MSBtFURjtMu3LyDTLYx9FBM23UQdZ1 +e8e7xgwaSI2JKI65FEThzSQBVmKeAZ +w3xAirHLO1tvjon2jgr7y9tBtrGfMS +zABBLCkowUIqfONQOAjir8YPkFqfDW +*/ + qt_test_11 """ select * from orc_tiny_stripes where col1 < 10 order by col1,col2,col3; """ +/* +0 str_0 10000000000 +0 str_0 10000000000 +1 str_1 10000000001 +1 str_1 10000000001 +2 str_2 10000000002 +2 str_2 10000000002 +3 str_3 10000000003 +3 str_3 10000000003 +4 str_4 10000000004 +4 str_4 10000000004 +5 str_5 10000000005 +5 str_5 10000000005 +6 str_6 10000000006 +7 str_7 10000000007 +8 str_8 10000000008 +9 str_9 10000000009 +*/ + + qt_test_12 """ select col1 from orc_tiny_stripes where col1 in(0,6 ) order by col1; """ +/* +0 +0 +6 +*/ + + qt_test_13 """ select col1 from orc_tiny_stripes where col1 in(20,60 ) order by col1; """ + /* +20 +60 +*/ + + qt_test_14 """ select col1 from orc_tiny_stripes where col1 in(40,0 ) order by col1; """ +/* +0 +0 +40 +*/ + + + } + } + + sql """drop catalog if exists ${catalog_name}""" + } finally { + } + } +} + diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy index 65a55c70f15e2c..fba50c3341e8a8 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy @@ -88,10 +88,12 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts""" } order_qt_func_push2 """select * from ts where ts <= unix_timestamp(from_unixtime(ts,'yyyyMMdd'));""" + sql "set enable_jdbc_cast_predicate_push_down = true;" explain { sql("select * from ts where ts <= unix_timestamp(from_unixtime(ts,'yyyy-MM-dd'));") contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts" WHERE (("ts" <= toUnixTimestamp(FROM_UNIXTIME("ts", '%Y-%m-%d'))))""" } + sql "set enable_jdbc_cast_predicate_push_down = false;" order_qt_dt_with_tz """ select * from dt_with_tz order by id; """ diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index a88ba550eea87f..c97c74ec28eb0f 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -643,6 +643,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc // ctas logic is different between new and old planner. // so need to test both. sql """drop catalog if exists mysql_conjuncts;""" + sql """set enable_nereids_planner=true""" } } diff --git a/regression-test/suites/inverted_index_p0/test_match_without_index.groovy b/regression-test/suites/fault_injection_p0/test_match_without_index_fault_injection.groovy similarity index 96% rename from regression-test/suites/inverted_index_p0/test_match_without_index.groovy rename to regression-test/suites/fault_injection_p0/test_match_without_index_fault_injection.groovy index a008cf703d865b..db03b75bee6976 100644 --- a/regression-test/suites/inverted_index_p0/test_match_without_index.groovy +++ b/regression-test/suites/fault_injection_p0/test_match_without_index_fault_injection.groovy @@ -16,9 +16,9 @@ // under the License. -suite("test_match_without_index", "p0") { +suite("test_match_without_index_fault_injection", "nonConcurrent") { - def testTable = "test_match_without_index" + def testTable = "test_match_without_index_fault_injection" sql "DROP TABLE IF EXISTS ${testTable}" sql """ CREATE TABLE ${testTable} ( diff --git a/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy new file mode 100644 index 00000000000000..76b666ecba736e --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_storage_page_size_fault.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_storage_page_size_fault", "nonConcurrent") { + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + boolean disableAutoCompaction = false + + def set_be_config = { key, value -> + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def dbName = "regression_test_fault_injection_p0" + def tableName = "test_storage_page_size_fault" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `@timestamp` int(11) NULL COMMENT "", + `clientip` varchar(20) NULL COMMENT "", + `request` text NULL COMMENT "", + `status` int(11) NULL COMMENT "", + `size` int(11) NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT "OLAP" + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "storage_page_size" = "65537" + ); + """ + + def tableId = getTableId(dbName, tableName) + if (tableId == null) { + throw new IllegalStateException("Table ID not found for table: ${tableName}") + } + logger.info("tableId: " + tableId) + + try { + GetDebugPoint().enableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size", ["table_id": tableId, "storage_page_size": 69632]) + sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """ + + set_be_config.call("enable_vertical_segment_writer", "false") + sql """ INSERT INTO ${tableName} VALUES (893964617, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """ + set_be_config.call("enable_vertical_segment_writer", "true") + + } finally { + GetDebugPoint().disableDebugPointForAllBEs("VerticalSegmentWriter._create_column_writer.storage_page_size") + } +} \ No newline at end of file diff --git a/regression-test/suites/insert_p0/test_insert_nan.groovy b/regression-test/suites/insert_p0/test_insert_nan.groovy new file mode 100644 index 00000000000000..8650c5d0cf1ef3 --- /dev/null +++ b/regression-test/suites/insert_p0/test_insert_nan.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_insert_nan") { + sql """drop table if exists `nan_table`;""" + + sql """ + CREATE TABLE `nan_table` ( + `id` int NOT NULL , + `val` double, + ) ENGINE=OLAP + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + + sql """insert into nan_table values ('1', 'nan');""" + + qt_select """select * from nan_table;""" +} diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy index 6c8b8c4e0d8fbc..2d92c560cf1a9d 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy @@ -103,7 +103,7 @@ suite("test_index_change_on_renamed_column") { assertEquals(show_result[0][2], "idx_s") qt_select2 """ SELECT * FROM ${tableName} order by id; """ - qt_select3 """ SELECT * FROM ${tableName} where s1 match 'welcome'; """ + qt_select3 """ SELECT /*+ SET_VAR(enable_inverted_index_query = true) */ * FROM ${tableName} where s1 match 'welcome'; """ def tablets = sql_return_maparray """ show tablets from ${tableName}; """ String tablet_id = tablets[0].TabletId diff --git a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy index 09663b5c10a2f3..e3255beb8a898c 100644 --- a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy @@ -267,6 +267,10 @@ suite("test_array_with_inverted_index_all_type"){ assertEquals(101, json.NumberLoadedRows) } } + + // insert into null array + sql """insert into ${tableName} (k1) values(101);""" + qt_sql_null """ select * from ${tableName} where k1 = 101; """ } // stream load with table diff --git a/regression-test/suites/load_p0/routine_load/data/test_routine_load_udf.csv b/regression-test/suites/load_p0/routine_load/data/test_routine_load_udf.csv new file mode 100644 index 00000000000000..b226b99ee4e0e0 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/data/test_routine_load_udf.csv @@ -0,0 +1 @@ +1,eab,2023-07-15,def,2023-07-20:05:48:31,"ghi" \ No newline at end of file diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy new file mode 100644 index 00000000000000..11a562ba9d4660 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_routine_load_with_udf","p0") { + def kafkaCsvTpoics = [ + "test_routine_load_udf", + ] + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // define kafka + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // Create kafka producer + def producer = new KafkaProducer<>(props) + + for (String kafkaCsvTopic in kafkaCsvTpoics) { + def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text + def lines = txt.readLines() + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(kafkaCsvTopic, null, line) + producer.send(record) + } + } + } + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + def tableName = "test_routine_load_with_udf" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + def jarPath = """${context.file.parent}/../../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" + scp_udf_file_to_all_be(jarPath) + log.info("Jar path: ${jarPath}".toString()) + + sql """ ADMIN SET FRONTEND CONFIG ("enable_udf_in_load" = "true"); """ + try_sql("DROP GLOBAL FUNCTION IF EXISTS java_udf_string_load_global(string);") + sql """ CREATE GLOBAL FUNCTION java_udf_string_load_global(string) RETURNS string PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.StringLoadTest", + "type"="JAVA_UDF" + ); """ + + try { + sql """ + CREATE ROUTINE LOAD test_udf_load ON ${tableName} + COLUMNS TERMINATED BY ",", + COLUMNS(k1, k2, v1, v2, v3, tmp, v4=java_udf_string_load_global(v2)) + PROPERTIES + ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTpoics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + + String db = context.config.getDbNameByFile(context.file) + log.info("reason of state changed: ${db}".toString()) + + def count = 0 + while (true) { + res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for test_udf_load" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + qt_sql_topic_udf "select * from ${tableName} order by k1" + } finally { + sql "stop routine load for test_udf_load" + } + } +} diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_with_udf.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_with_udf.groovy new file mode 100644 index 00000000000000..657a298991742c --- /dev/null +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_with_udf.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_stream_load_with_udf", "p0") { + def tableName = "test_stream_load_with_udf" + sql """ set enable_fallback_to_original_planner=false;""" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + id int, + v1 string + ) ENGINE=OLAP + duplicate key (`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + def jarPath = """${context.file.parent}/../../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar""" + scp_udf_file_to_all_be(jarPath) + log.info("Jar path: ${jarPath}".toString()) + + sql """ ADMIN SET FRONTEND CONFIG ("enable_udf_in_load" = "true"); """ + try_sql("DROP GLOBAL FUNCTION IF EXISTS java_udf_int_load_global(int);") + sql """ CREATE GLOBAL FUNCTION java_udf_int_load_global(int) RETURNS int PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.IntLoadTest", + "type"="JAVA_UDF" + ); """ + + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', """ tmp,v1, id=java_udf_int_load_global(tmp) """ + file 'test_stream_load_udf.csv' + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(3, json.NumberLoadedRows) + } + } + + sql """sync""" + + qt_sql """select * from ${tableName} order by id;""" + +} diff --git a/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy b/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy new file mode 100644 index 00000000000000..693df86e014844 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_outfile.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("test_mtmv_outfile","mtmv") { + + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_mtmv_outfile" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 INT, + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into ${tableName} values (1,1),(1,2); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k2) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + + def jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinished(jobName) + + // use to outfile to s3 + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName"); + + def outfile_to_S3_directly = { + // select ... into outfile ... + def s3_outfile_path = "${bucket}/outfile/csv/test-mtmv-outfile" + def uri = "s3://${s3_outfile_path}/exp_" + + def res = sql """ + SELECT * FROM ${mvName} t + INTO OUTFILE "${uri}" + FORMAT AS csv + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + logger.info("outfile to s3 success path: " + res[0][3]); + } + + outfile_to_S3_directly() + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_mtmv_property.groovy b/regression-test/suites/mtmv_p0/test_mtmv_property.groovy new file mode 100644 index 00000000000000..e0ffbca8dbf63c --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_property.groovy @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_mtmv_property","mtmv") { + if (isCloudMode()) { + return + } + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_mtmv_property" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 INT, + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k2) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'store_row_column' = 'true', + 'skip_write_index_on_load' = 'false', + "binlog.enable" = "false", + "binlog.ttl_seconds" = "86400", + "binlog.max_bytes" = "9223372036854775807", + "binlog.max_history_nums" = "9223372036854775807", + "enable_duplicate_without_keys_by_default" = "true" + ) + AS + SELECT * from ${tableName}; + """ + + def showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('tag.location.default: 1')) + assertTrue(showCreateTableResult.toString().contains('"min_load_replica_num" = "-1"')) + assertTrue(showCreateTableResult.toString().contains('"storage_medium" = "hdd"')) + assertTrue(showCreateTableResult.toString().contains('"store_row_column" = "true"')) + assertTrue(showCreateTableResult.toString().contains('"enable_duplicate_without_keys_by_default" = "true"')) + + sql """ + insert into ${tableName} values (2,1),(2,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + def jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinished(jobName) + sql """select * from ${mvName}""" + + // replication_num + sql """ + ALTER TABLE ${mvName} set ("replication_num" = "1"); + """ + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('tag.location.default: 1')) + + // replication_allocation + sql """ + ALTER TABLE ${mvName} set ("replication_allocation" = "tag.location.default: 1"); + """ + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('tag.location.default: 1')) + + // min_load_replica_num + sql """ + ALTER TABLE ${mvName} set ("min_load_replica_num" = "1"); + """ + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('"min_load_replica_num" = "1"')) + + // type + sql """ + insert into ${tableName} values (3,1),(3,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinished(jobName) + def tablets_res = sql """show tablets from ${mvName}""" + assertTrue(tablets_res.size() > 0) + sql """ + admin check tablet (${tablets_res[0][0].toString()}) properties("type"="consistency"); + """ + + // storage_policy + def resource_name = "test_mtmv_remote_s3_resource" + def policy_name= "test_mtmv_storage_policy" + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name} + PROPERTIES( + "storage_resource" = "${resource_name}", + "cooldown_ttl" = "300" + ) + """ + + sql """ALTER TABLE ${mvName} SET ("storage_policy" = "${policy_name}");""" + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('"storage_policy" = "test_mtmv_storage_policy"')) + + // store_row_column + try { + sql """ALTER TABLE ${mvName} SET ("store_row_column" = "false");""" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Can not alter store_row_column")) + } + + // is_being_synced + sql """ + ALTER TABLE ${mvName} set ("is_being_synced" = "true"); + """ + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('"is_being_synced" = "true"')) + + // skip_write_index_on_load + def desc_res = sql """desc ${mvName}""" + assertTrue(desc_res.size() == 2) + assertTrue(desc_res[0][3] == "false") + assertTrue(desc_res[1][3] == "false") + + // enable_unique_key_merge_on_write + try { + sql """ + CREATE MATERIALIZED VIEW mv_unique_key_merge_on_write + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k2) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + "enable_unique_key_merge_on_write" = "true" + ) + AS + SELECT * from ${tableName}; + """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Unknown properties")) + } + + // group_commit_interval_ms + sql """ALTER TABLE ${mvName} SET ("group_commit_interval_ms" = "2000");""" + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('"group_commit_interval_ms" = "2000"')) + + // group_commit_data_bytes + sql """ALTER TABLE ${mvName} SET ("group_commit_data_bytes" = "234217728");""" + showCreateTableResult = sql """show create materialized view ${mvName}""" + logger.info("showCreateTableResult: " + showCreateTableResult.toString()) + assertTrue(showCreateTableResult.toString().contains('"group_commit_data_bytes" = "234217728"')) + + // check mv can update normally + sql """ + insert into ${tableName} values (1,1),(1,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinished(jobName) + sql """select * from ${mvName}""" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy b/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy new file mode 100644 index 00000000000000..02191f0d4cd7d4 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_mtmv_sql_cache_and_profile", "mtmv") { + + sql """ADMIN SET FRONTEND CONFIG ('cache_enable_sql_mode' = 'true')""" + + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_mtmv_sql_cache_and_profile" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + sql """use ${dbName};""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 INT, + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into ${tableName} values (1,1),(1,2); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k2) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + + def jobName = getJobName(dbName, mvName) + waitingMTMVTaskFinished(jobName) + + sql """set enable_sql_cache=true;""" + + long startTime = System.currentTimeMillis() + long timeoutTimestamp = startTime + 5 * 60 * 1000 + def explain_res = "" + while (System.currentTimeMillis() < timeoutTimestamp) { + sleep(5 * 1000) + sql """select k2 from ${mvName} group by k2;""" + try { + explain_res = sql """explain plan select k2 from ${mvName} group by k2;""" + } catch (Exception e) { + logger.info(e.getMessage()) + } + logger.info("explain_res: " + explain_res) + if (explain_res.toString().indexOf("LogicalSqlCache") != -1 || explain_res.toString().indexOf("PhysicalSqlCache") != -1) { + break + } + } + assertTrue(explain_res.toString().indexOf("LogicalSqlCache") != -1 || explain_res.toString().indexOf("PhysicalSqlCache") != -1) + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/nereids_function_p0/load.groovy b/regression-test/suites/nereids_function_p0/load.groovy index 105767843bdc37..d4aeea530412cb 100644 --- a/regression-test/suites/nereids_function_p0/load.groovy +++ b/regression-test/suites/nereids_function_p0/load.groovy @@ -27,6 +27,311 @@ suite("load") { DROP TABLE IF EXISTS `fn_test_bitmap` """ + // test ipv4/ipv6 + sql """ drop table if exists fn_test_ip_nullable """ + sql """ CREATE TABLE IF NOT EXISTS fn_test_ip_nullable (id int, ip4 ipv4, ip6 ipv6, ip4_str string, ip6_str string) engine=olap + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + properties("replication_num" = "1") """ + + sql """ drop table if exists fn_test_ip_not_nullable """ + sql """ CREATE TABLE IF NOT EXISTS fn_test_ip_not_nullable (id int, ip4 ipv4 not null, ip6 ipv6 not null, ip4_str string, ip6_str string) engine=olap + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + properties("replication_num" = "1") """ + + // test ip with rowstore + sql """ drop table if exists fn_test_ip_nullable_rowstore """ + sql """ CREATE TABLE IF NOT EXISTS fn_test_ip_nullable_rowstore (id int, ip4 ipv4, ip6 ipv6, ip4_str string, ip6_str string) engine=olap + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + properties("replication_num" = "1", "store_row_column" = "true") """ + sql """ drop table if exists fn_test_ip_not_nullable_rowstore """ + sql """ CREATE TABLE IF NOT EXISTS fn_test_ip_not_nullable_rowstore (id int, ip4 ipv4 not null, ip6 ipv6 not null, ip4_str string, ip6_str string) engine=olap + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 + properties("replication_num" = "1", "store_row_column" = "true") """ + // make some special ip address + /*** + 回环地址 + 1;127.0.0.1;::1 + // 私有地址 + - 网络地址 (最小地址) + 2;10.0.0.0;fc00:: + - 广播地址 (最大地址) + 3;10.255.255.255;fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff + - 网络地址 (最小地址) + 4;172.16.0.0;fc00:: + - 广播地址 (最大地址) + 5;172.31.255.255;febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff + - 网络地址 (最小地址) + 6;192.168.0.0;fe80:: + - 广播地址 (最大地址) + 7;192.168.255.255;ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff + // 链路本地地址 + 8;169.254.0.0;fe80:: + // 公有地址 + 9;8.8.8.8;2001:4860:4860::8888 // Google Public DNS + 10;1.1.1.1;2606:4700:4700::1111 // Cloudflare DNS + // 组播地址 + 11;224.0.0.0;ff01:: // 所有主机 + 12;239.255.255.255;ff02::1 // 所有路由器 + // 仅用于文档示例的地址 + 13;192.0.2.0;2001:0db8:85a3::8a2e:0370:7334 + 14;203.0.113.0;2001:db8::1 + 15;198.51.100.0;2001:db8::2 + // 本地回环地址 + 16;localhost;::1 + // IPv4 特殊地址 + 17;240.0.0.0;null // 保留地址 + 18;255.255.255.255;null // 广播地址 + // 唯一本地地址 + 19;null;fd00:: // 唯一本地地址 (ULA) + // A 类地址 + - 网络地址 (最小地址) + 20;0.0.0.0;null + - 最大地址 + 21;127.255.255.255;null + // B 类地址 + - 网络地址 (最小地址) + 22;128.0.0.0;null + - 最大地址 + 23;191.255.255.255;null + // C 类地址 + - 网络地址 (最小地址) + 24;192.0.0.0;null + - 最大地址 + 25;223.255.255.255;null + // D 类地址 + - 组播地址 (最小地址) + 26;224.0.0.0;ff01:: + - 最大地址 + 27;239.255.255.255;ff02::1 + // 无效的多播地址 + 28;null;ff00:: // 保留地址 + ***/ + + streamLoad { + table "fn_test_ip_nullable" + db "regression_test_nereids_function_p0" + file "fn_test_ip_special.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(28, json.NumberTotalRows) + assertEquals(28, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_nullable_rowstore" + db "regression_test_nereids_function_p0" + file "fn_test_ip_special.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(28, json.NumberTotalRows) + assertEquals(28, json.NumberLoadedRows) + } + } + + // rowstore table to checkout with not rowstore table + def sql_res = sql "select * from fn_test_ip_nullable order by id;" + def sql_res_rowstore = sql "select * from fn_test_ip_nullable_rowstore order by id;" + assertEquals(sql_res.size(), sql_res_rowstore.size()) + for (int i = 0; i < sql_res.size(); i++) { + for (int j = 0; j < sql_res[i].size(); j++) { + assertEquals(sql_res[i][j], sql_res_rowstore[i][j]) + } + } + + // test fn_test_ip_nullable_rowstore table with update action + sql "update fn_test_ip_nullable_rowstore set ip4 = '' where id = 1;" + sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;" + log.info("sql_res: ${sql_res[0]}".toString()) + assertEquals(sql_res[0].toString(), '[1, null, ::1, "127.0.0.1", "::1"]') + sql "update fn_test_ip_nullable_rowstore set ip6 = '' where id = 1;" + sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;" + assertEquals(sql_res[0].toString(), '[1, null, null, "127.0.0.1", "::1"]') + sql "update fn_test_ip_nullable_rowstore set ip4 = '127.0.0.1' where id = 1;" + sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;" + assertEquals(sql_res[0].toString(), '[1, 127.0.0.1, null, "127.0.0.1", "::1"]') + sql "update fn_test_ip_nullable_rowstore set ip6 = '::1' where id = 1;" + sql_res = sql "select * from fn_test_ip_nullable_rowstore where id = 1;" + assertEquals(sql_res[0].toString(), '[1, 127.0.0.1, ::1, "127.0.0.1", "::1"]') + + streamLoad { + table "fn_test_ip_not_nullable" + db "regression_test_nereids_function_p0" + file "fn_test_ip_special_no_null.csv" + set 'column_separator', ';' + set "max_filter_ratio", "0.1" + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(28, json.NumberTotalRows) + assertEquals(27, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_not_nullable_rowstore" + db "regression_test_nereids_function_p0" + file "fn_test_ip_special_no_null.csv" + set 'column_separator', ';' + set "max_filter_ratio", "0.1" + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(28, json.NumberTotalRows) + assertEquals(27, json.NumberLoadedRows) + } + } + + // rowstore table to checkout with not rowstore table + def sql_res_not_null = sql "select * from fn_test_ip_not_nullable_rowstore order by id;" + def sql_res_not_null_rowstore = sql "select * from fn_test_ip_not_nullable_rowstore order by id;" + assertEquals(sql_res_not_null.size(), sql_res_not_null_rowstore.size()) + for (int i = 0; i < sql_res_not_null.size(); i++) { + for (int j = 0; j < sql_res_not_null[i].size(); j++) { + assertEquals(sql_res_not_null[i][j], sql_res_not_null_rowstore[i][j]) + } + } + + // test fn_test_ip_not_nullable_rowstore table with update action + // not null will throw exception if we has data in table + test { + sql "update fn_test_ip_not_nullable_rowstore set ip4 = '' where id = 1;" + exception("Insert has filtered data in strict mode") + } + + test { + sql "update fn_test_ip_not_nullable_rowstore set ip6 = '' where id = 1;" + exception("Insert has filtered data in strict mode") + } + + sql "update fn_test_ip_not_nullable_rowstore set ip4 = '192.10.10.1' where id = 1;" + def sql_res1 = sql "select * from fn_test_ip_not_nullable_rowstore where id = 1;" + log.info("sql_res: ${sql_res1[0]}".toString()) + assertEquals(sql_res1[0].toString(), '[1, 192.10.10.1, ::1, "127.0.0.1", "::1"]') + sql "update fn_test_ip_not_nullable_rowstore set ip6 = '::2' where id = 1;" + sql_res1 = sql "select * from fn_test_ip_not_nullable_rowstore where id = 1;" + assertEquals(sql_res1[0].toString(), '[1, 192.10.10.1, ::2, "127.0.0.1", "::1"]') + + + // make some normal ipv4/ipv6 data for sql function , which is increased one by one + // 29-50 A 类地址 ; 51-68 B 类地址 ; 69-87 C 类地址 ; 88-100 D 类地址 + streamLoad { + table "fn_test_ip_nullable" + db "regression_test_nereids_function_p0" + file "fn_test_ip_normal.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(72, json.NumberTotalRows) + assertEquals(72, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_nullable_rowstore" + db "regression_test_nereids_function_p0" + file "fn_test_ip_normal.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(72, json.NumberTotalRows) + assertEquals(72, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_not_nullable" + db "regression_test_nereids_function_p0" + file "fn_test_ip_normal.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(72, json.NumberTotalRows) + assertEquals(72, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_not_nullable_rowstore" + db "regression_test_nereids_function_p0" + file "fn_test_ip_normal.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(72, json.NumberTotalRows) + assertEquals(72, json.NumberLoadedRows) + } + } + + streamLoad { + table "fn_test_ip_not_nullable" + db "regression_test_nereids_function_p0" + file "fn_test_ip_invalid.csv" + set 'column_separator', ';' + time 60000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals(31, json.NumberTotalRows) + assertEquals(0, json.NumberLoadedRows) + } + } + + sql """ CREATE TABLE IF NOT EXISTS `fn_test` ( `id` int null, diff --git a/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy b/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy new file mode 100644 index 00000000000000..9cbdccbe78840b --- /dev/null +++ b/regression-test/suites/nereids_function_p0/scalar_function/IP.groovy @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nereids_scalar_fn_IP") { + sql 'use regression_test_nereids_function_p0' + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + def cidr_v6 = 64 + def cidr_v4 = 32 + // for table fn_test_ip_nullable + qt_sql """ select count() from fn_test_ip_nullable; """ + // test_ip_cidr_to_range_function + qt_sql_cidr_ipv6 "select id, struct_element(ipv6_cidr_to_range(ip6,$cidr_v6), 'min') as min_range, struct_element(ipv6_cidr_to_range(ip6, $cidr_v6), 'max') as max_range from fn_test_ip_nullable order by id" + qt_sql_cidr_ipv4 "select id, struct_element(ipv4_cidr_to_range(ip4, $cidr_v4), 'min') as min_range, struct_element(ipv4_cidr_to_range(ip4, $cidr_v4), 'max') as max_range from fn_test_ip_nullable order by id" + qt_sql_cidr_ipv6_all """ select id, ipv6_cidr_to_range(ip6, 16) from fn_test_ip_nullable order by id; """ + qt_sql_cidr_ipv4_all """ select id, ipv4_cidr_to_range(ip4, 16) from fn_test_ip_nullable order by id; """ + + + // test IPV4_STRING_TO_NUM/IPV6_STRING_TO_NUM (we have null value in ip4 and ip6 column in fn_test_ip_nullable table) + test { + sql 'select id, ipv6_string_to_num(ip6) from fn_test_ip_nullable order by id' + exception "Null Input" + } + + test { + sql "select id, ipv6_string_to_num(ip6_str) from fn_test_ip_nullable order by id" + exception "Invalid IPv6 value" + } + + test { + sql 'select id, ipv4_string_to_num(ip4) from fn_test_ip_nullable order by id' + exception "Null Input" + } + + test { + sql "select id, ipv4_string_to_num(ip4_str) from fn_test_ip_nullable order by id" + exception "Invalid IPv4 value" + } + + // test ipv_num_to_string + qt_sql_num2string_ipv6 "select id, ipv6_num_to_string(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_nullable order by id" + qt_sql_num2string_ipv6_str "select id, ipv6_num_to_string(ip6_str) from fn_test_ip_nullable order by id" + qt_sql_num2string_ipv4 "select id, ipv4_num_to_string(ipv4_string_to_num_or_default(ip4)) from fn_test_ip_nullable order by id" + qt_sql_num2string_ipv4_str "select id, ipv4_num_to_string(ip4_str) from fn_test_ip_nullable order by id" + + // test INET_NTOA/INET6_NTOA + qt_sql_inet6_ntoa "select id, inet6_ntoa(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_nullable order by id" + qt_sql_inet6_ntoa_str "select id, inet6_ntoa(ip6_str) from fn_test_ip_nullable order by id" + qt_sql_inet_ntoa "select id, inet_ntoa(ipv4_string_to_num_or_default(ip4)) from fn_test_ip_nullable order by id" + qt_sql_inet_ntoa_str "select id, inet_ntoa(ip4_str) from fn_test_ip_nullable order by id" + + // test IPV4_STRING_TO_NUM_OR_DEFAULT/IPV6_STRING_TO_NUM_OR_DEFAULT + qt_sql_string2num_or_default_ipv6 "select id, hex(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_default_ipv6_str "select id, hex(ipv6_string_to_num_or_default(ip6_str)) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_default_ipv4 "select id, ipv4_string_to_num_or_default(ip4) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_default_ipv4_str "select id, ipv4_string_to_num_or_default(ip4_str) from fn_test_ip_nullable order by id" + + // test IPV4_STRING_TO_NUM_OR_NULL/IPV6_STRING_TO_NUM_OR_NULL + qt_sql_string2num_or_null_ipv6 "select id, hex(ipv6_string_to_num_or_null(ip6)) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_null_ipv6_str "select id, hex(ipv6_string_to_num_or_null(ip6_str)) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_null_ipv4 "select id, ipv4_string_to_num_or_null(ip4) from fn_test_ip_nullable order by id" + qt_sql_string2num_or_null_ipv4_str "select id, ipv4_string_to_num_or_null(ip4_str) from fn_test_ip_nullable order by id" + + // test IS_IPV4_COMPAT/IS_IPV4_MAPPED + qt_sql_is_ipv4_compat "select id, is_ipv4_compat(ip6) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_compat_str6 "select id, is_ipv4_mapped(INET6_ATON(ip6_str)) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_compat_str4 "select id, is_ipv4_mapped(INET6_ATON(ip4_str)) from fn_test_ip_nullable order by id" + + qt_sql_is_ipv4_mapped "select id, is_ipv4_mapped(ip6) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_mapped_str6 "select id, is_ipv4_mapped(INET6_ATON(ip6_str)) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_mapped_str4 "select id, is_ipv4_mapped(INET6_ATON(ip4_str)) from fn_test_ip_nullable order by id" + + // test IS_IP_ADDRESS_IN_RANGE + def cidr_prefix_v6 = '2001:db8::/32' + def cidr_prefix_v4 = '::ffff:192.168.0.4/128' + qt_sql_is_ip_address_in_range_ipv6 "select id, is_ip_address_in_range(ip6, '$cidr_prefix_v6') from fn_test_ip_nullable order by id" + test { + sql "select id, is_ip_address_in_range(ip6_str, '$cidr_prefix_v6') from fn_test_ip_nullable order by id" + exception "Neither IPv4 nor IPv6" + } + qt_sql_is_ip_address_in_range_ipv4 "select id, is_ip_address_in_range(ip4, '$cidr_prefix_v4') from fn_test_ip_nullable order by id" + test { + sql "select id, is_ip_address_in_range(ip4_str, '$cidr_prefix_v4') from fn_test_ip_nullable order by id" + exception "Neither IPv4 nor IPv6" + } + qt_sql_is_ip_address_in_range_null "select id, is_ip_address_in_range(ip6, null) from fn_test_ip_nullable order by id" + qt_sql_is_ip_address_in_range_null_str "select id, is_ip_address_in_range(ip6_str, null) from fn_test_ip_nullable order by id" + qt_sql_is_ip_address_in_range_null "select id, is_ip_address_in_range(ip4, null) from fn_test_ip_nullable order by id" + qt_sql_is_ip_address_in_range_null_str "select id, is_ip_address_in_range(ip4_str, null) from fn_test_ip_nullable order by id" + + // test IS_IPV4_STRING/IS_IPV6_STRING + qt_sql_is_ipv4_string "select id, is_ipv4_string(ip4) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_string1 "select id, is_ipv4_string(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_is_ipv6_string "select id, is_ipv6_string(ip6) from fn_test_ip_nullable order by id" + qt_sql_is_ipv6_string1 "select id, is_ipv6_string(ip6_str) from fn_test_ip_nullable order by id" + qt_sql_is_ipv6_string "select id, is_ipv6_string(ip4) from fn_test_ip_nullable order by id" + qt_sql_is_ipv6_string1 "select id, is_ipv6_string(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_string "select id, is_ipv4_string(ip6) from fn_test_ip_nullable order by id" + qt_sql_is_ipv4_string1 "select id, is_ipv4_string(ip6_str) from fn_test_ip_nullable order by id" + + // test TO_IPV4/TO_IPV6 (we have null value in ip4 and ip6 column in fn_test_ip_nullable table) + test { + sql "select id, to_ipv4(ip4) from fn_test_ip_nullable order by id" + exception "not NULL" + } + + test { + sql "select id, to_ipv6(ip6) from fn_test_ip_nullable order by id" + exception "not NULL" + } + + // test TO_IPV4_OR_DEFAULT/TO_IPV6_OR_DEFAULT + qt_sql_to_ipv6_or_default "select id, to_ipv6_or_default(ip6) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_default_str "select id, to_ipv6_or_default(ip6_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_default "select id, to_ipv4_or_default(ip4) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_default_str "select id, to_ipv4_or_default(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_default "select id, to_ipv6_or_default(ip4) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_default_st "select id, to_ipv6_or_default(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_default "select id, to_ipv4_or_default(ip6) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_default_st "select id, to_ipv4_or_default(ip6_str) from fn_test_ip_nullable order by id" + + // test TO_IPV4_OR_NULL/TO_IPV6_OR_NULL + qt_sql_to_ipv6_or_null "select id, to_ipv6_or_null(ip6) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_null_str "select id, to_ipv6_or_null(ip6_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_null "select id, to_ipv4_or_null(ip4) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_null_str "select id, to_ipv4_or_null(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_null "select id, to_ipv6_or_null(ip4) from fn_test_ip_nullable order by id" + qt_sql_to_ipv6_or_null_str "select id, to_ipv6_or_null(ip4_str) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_null "select id, to_ipv4_or_null(ip6) from fn_test_ip_nullable order by id" + qt_sql_to_ipv4_or_null_str "select id, to_ipv4_or_null(ip6_str) from fn_test_ip_nullable order by id" + + + // for table fn_test_ip_not_nullable + qt_sql_not_null """ select count() from fn_test_ip_not_nullable; """ + // test_ip_cidr_to_range_function + qt_sql_not_null_cidr_ipv6 "select id, struct_element(ipv6_cidr_to_range(ip6,$cidr_v6), 'min') as min_range, struct_element(ipv6_cidr_to_range(ip6, $cidr_v6), 'max') as max_range from fn_test_ip_not_nullable order by id" + qt_sql_not_null_cidr_ipv4 "select id, struct_element(ipv4_cidr_to_range(ip4, $cidr_v4), 'min') as min_range, struct_element(ipv4_cidr_to_range(ip4, $cidr_v4), 'max') as max_range from fn_test_ip_not_nullable order by id" + qt_sql_not_null_cidr_ipv6_all """ select id, ipv6_cidr_to_range(ip6, 16) from fn_test_ip_not_nullable order by id; """ + qt_sql_not_null_cidr_ipv4_all """ select id, ipv4_cidr_to_range(ip4, 16) from fn_test_ip_not_nullable order by id; """ + + + // test IPV4_STRING_TO_NUM/IPV6_STRING_TO_NUM + qt_sql_not_null_ipv6_string_to_num 'select id, hex(ipv6_string_to_num(ip6)) from fn_test_ip_not_nullable order by id' + + // string has 'null' this invalid data + test { + sql "select id, ipv6_string_to_num(ip6_str) from fn_test_ip_not_nullable order by id" + exception "Invalid IPv6 value" + } + + qt_sql_not_null_ipv4_string_to_num 'select id, ipv4_string_to_num(ip4) from fn_test_ip_not_nullable order by id' + + // string has 'null' this invalid data + test { + sql "select id, ipv4_string_to_num(ip4_str) from fn_test_ip_not_nullable order by id" + exception "Invalid IPv4 value" + } + + + // test ipv_num_to_string + qt_sql_not_null_num2string_ipv6 "select id, ipv6_num_to_string(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_num2string_ipv6_str "select id, ipv6_num_to_string(ip6_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_num2string_ipv4 "select id, ipv4_num_to_string(ipv4_string_to_num_or_default(ip4)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_num2string_ipv4_str "select id, ipv4_num_to_string(ip4_str) from fn_test_ip_not_nullable order by id" + + // test INET_NTOA/INET6_NTOA + qt_sql_not_null_inet6_ntoa "select id, inet6_ntoa(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_inet6_ntoa_str "select id, inet6_ntoa(ip6_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_inet_ntoa "select id, inet_ntoa(ipv4_string_to_num_or_default(ip4)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_inet_ntoa_str "select id, inet_ntoa(ip4_str) from fn_test_ip_not_nullable order by id" + + // test IPV4_STRING_TO_NUM_OR_DEFAULT/IPV6_STRING_TO_NUM_OR_DEFAULT + qt_sql_not_null_string2num_or_default_ipv6 "select id, hex(ipv6_string_to_num_or_default(ip6)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_default_ipv6_str "select id, hex(ipv6_string_to_num_or_default(ip6_str)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_default_ipv4 "select id, ipv4_string_to_num_or_default(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_default_ipv4_str "select id, ipv4_string_to_num_or_default(ip4_str) from fn_test_ip_not_nullable order by id" + + // test IPV4_STRING_TO_NUM_OR_NULL/IPV6_STRING_TO_NUM_OR_NULL + qt_sql_not_null_string2num_or_null_ipv6 "select id, hex(ipv6_string_to_num_or_null(ip6)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_null_ipv6_str "select id, hex(ipv6_string_to_num_or_null(ip6_str)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_null_ipv4 "select id, ipv4_string_to_num_or_null(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_string2num_or_null_ipv4_str "select id, ipv4_string_to_num_or_null(ip4_str) from fn_test_ip_not_nullable order by id" + + // test IS_IPV4_COMPAT/IS_IPV4_MAPPED + qt_sql_not_null_is_ipv4_compat "select id, is_ipv4_compat(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_compat_str6 "select id, is_ipv4_mapped(INET6_ATON(ip6_str)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_compat_str4 "select id, is_ipv4_mapped(INET6_ATON(ip4_str)) from fn_test_ip_not_nullable order by id" + + qt_sql_not_null_is_ipv4_mapped "select id, is_ipv4_mapped(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_mapped_str6 "select id, is_ipv4_mapped(INET6_ATON(ip6_str)) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_mapped_str4 "select id, is_ipv4_mapped(INET6_ATON(ip4_str)) from fn_test_ip_not_nullable order by id" + + // test IS_IP_ADDRESS_IN_RANGE + qt_sql_not_null_is_ip_address_in_range_ipv6 "select id, is_ip_address_in_range(ip6, '$cidr_prefix_v6') from fn_test_ip_not_nullable order by id" + + test { + sql "select id, is_ip_address_in_range(ip6_str, '$cidr_prefix_v6') from fn_test_ip_not_nullable order by id" + exception "Neither IPv4 nor IPv6" + } + + qt_sql_not_null_is_ip_address_in_range_ipv4 "select id, is_ip_address_in_range(ip4, '$cidr_prefix_v4') from fn_test_ip_not_nullable order by id" + + test { + sql "select id, is_ip_address_in_range(ip4_str, '$cidr_prefix_v4') from fn_test_ip_not_nullable order by id" + exception "Neither IPv4 nor IPv6" + } + + qt_sql_not_null_is_ip_address_in_range_null "select id, is_ip_address_in_range(ip6, null) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ip_address_in_range_null_str "select id, is_ip_address_in_range(ip6_str, null) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ip_address_in_range_null "select id, is_ip_address_in_range(ip4, null) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ip_address_in_range_null_str "select id, is_ip_address_in_range(ip4_str, null) from fn_test_ip_not_nullable order by id" + + // test IS_IPV4_STRING/IS_IPV6_STRING + qt_sql_not_null_is_ipv4_string "select id, is_ipv4_string(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_string1 "select id, is_ipv4_string(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv6_string "select id, is_ipv6_string(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv6_string1 "select id, is_ipv6_string(ip6_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv6_string "select id, is_ipv6_string(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv6_string1 "select id, is_ipv6_string(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_string "select id, is_ipv4_string(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_is_ipv4_string1 "select id, is_ipv4_string(ip6_str) from fn_test_ip_not_nullable order by id" + + // test TO_IPV4/TO_IPV6 + qt_sql_not_null_to_ipv4 "select id, to_ipv4(ip4) from fn_test_ip_not_nullable order by id" + + test { + sql "select id, to_ipv4(ip4_str) from fn_test_ip_not_nullable order by id" + exception "Invalid IPv4 value" + } + qt_sql_not_null_to_ipv6 "select id, to_ipv6(ip6) from fn_test_ip_not_nullable order by id" + + test { + sql "select id, to_ipv6(ip6_str) from fn_test_ip_not_nullable order by id" + exception "Invalid IPv6 value" + } + + // test TO_IPV4_OR_DEFAULT/TO_IPV6_OR_DEFAULT + qt_sql_not_null_to_ipv6_or_default "select id, to_ipv6_or_default(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_default_str "select id, to_ipv6_or_default(ip6_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_default "select id, to_ipv4_or_default(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_default_str "select id, to_ipv4_or_default(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_default "select id, to_ipv6_or_default(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_default_st "select id, to_ipv6_or_default(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_default "select id, to_ipv4_or_default(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_default_st "select id, to_ipv4_or_default(ip6_str) from fn_test_ip_not_nullable order by id" + + // test TO_IPV4_OR_NULL/TO_IPV6_OR_NULL + qt_sql_not_null_to_ipv6_or_null "select id, to_ipv6_or_null(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_null_str "select id, to_ipv6_or_null(ip6_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_null "select id, to_ipv4_or_null(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_null_str "select id, to_ipv4_or_null(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_null "select id, to_ipv6_or_null(ip4) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv6_or_null_str "select id, to_ipv6_or_null(ip4_str) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_null "select id, to_ipv4_or_null(ip6) from fn_test_ip_not_nullable order by id" + qt_sql_not_null_to_ipv4_or_null_str "select id, to_ipv4_or_null(ip6_str) from fn_test_ip_not_nullable order by id" + +} \ No newline at end of file diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query10.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query10.groovy index bfe96f54652eca..f9c6d80b84f704 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query10.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query10.groovy @@ -24,6 +24,7 @@ suite("query10") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query11.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query11.groovy index 96053d3f5186a8..a99c0fb2844909 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query11.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query11.groovy @@ -24,6 +24,7 @@ suite("query11") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query12.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query12.groovy index 50c6332ab10b0a..db173ea3d81a45 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query12.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query12.groovy @@ -24,6 +24,7 @@ suite("query12") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query13.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query13.groovy index 452e4fe6c5c3a8..e6850d187a5c6d 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query13.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query13.groovy @@ -24,6 +24,7 @@ suite("query13") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query14.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query14.groovy index e1adcf8f7355f7..d508d7ba1a586c 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query14.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query14.groovy @@ -24,6 +24,7 @@ suite("query14") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query15.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query15.groovy index 3f53835b3a8f28..e244c34686be9e 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query15.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query15.groovy @@ -24,6 +24,7 @@ suite("query15") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query16.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query16.groovy index b5823a551ee298..90e5de42bdb5aa 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query16.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query16.groovy @@ -24,6 +24,7 @@ suite("query16") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query17.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query17.groovy index f6e30e0a51df4e..779f982558c90f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query17.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query17.groovy @@ -24,6 +24,7 @@ suite("query17") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query18.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query18.groovy index 8a567608bcd551..ae0ea359c1d09c 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query18.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query18.groovy @@ -24,6 +24,7 @@ suite("query18") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query19.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query19.groovy index 112a8d889278ba..70aa3e4f4131da 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query19.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query19.groovy @@ -24,6 +24,7 @@ suite("query19") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query2.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query2.groovy index 6557c5b2fe1d67..50ffb28aeb41b7 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query2.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query2.groovy @@ -24,6 +24,7 @@ suite("query2") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query20.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query20.groovy index 46ad1fc099ee88..419e9fc593ada4 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query20.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query20.groovy @@ -24,6 +24,7 @@ suite("query20") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query21.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query21.groovy index 65da4964236784..9c50c722a73ac3 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query21.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query21.groovy @@ -24,6 +24,7 @@ suite("query21") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query22.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query22.groovy index c98156b7ea98ce..5ef9a0bbbbcce2 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query22.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query22.groovy @@ -24,6 +24,7 @@ suite("query22") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query23.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query23.groovy index 4f146d3049c409..117dea7b35b5c9 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query23.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query23.groovy @@ -22,6 +22,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query25.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query25.groovy index df41dfc657a098..19d4ef8443f952 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query25.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query25.groovy @@ -24,6 +24,7 @@ suite("query25") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query26.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query26.groovy index e17e3068a2e1f1..c71bb92aac72eb 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query26.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query26.groovy @@ -24,6 +24,7 @@ suite("query26") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query27.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query27.groovy index 326056a0be7b61..ec5f4ac2f1ad93 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query27.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query27.groovy @@ -24,6 +24,7 @@ suite("query27") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query28.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query28.groovy index 15bc959e5e2774..8c43bbaf31144a 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query28.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query28.groovy @@ -24,6 +24,7 @@ suite("query28") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query29.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query29.groovy index b2184e353d8fdc..f8c7f94cf07170 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query29.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query29.groovy @@ -24,6 +24,7 @@ suite("query29") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query3.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query3.groovy index 88cef8e1149c31..db9d172dfb4cfa 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query3.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query3.groovy @@ -24,6 +24,7 @@ suite("query3") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query30.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query30.groovy index 7d49e734418c94..0d89beaab9f0c2 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query30.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query30.groovy @@ -24,6 +24,7 @@ suite("query30") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query31.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query31.groovy index 124e5780f7cc89..06d3717ca0600f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query31.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query31.groovy @@ -24,6 +24,7 @@ suite("query31") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query32.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query32.groovy index 2ed929bc09a85b..2d45b854ee61bf 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query32.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query32.groovy @@ -22,6 +22,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query34.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query34.groovy index a02f193f73bc78..79972f33bb7ee0 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query34.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query34.groovy @@ -24,6 +24,7 @@ suite("query34") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query36.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query36.groovy index 679cc544729616..580f025d597eee 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query36.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query36.groovy @@ -24,6 +24,7 @@ suite("query36") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query37.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query37.groovy index ab42a262c99724..c28de3ae6e6a13 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query37.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query37.groovy @@ -24,6 +24,7 @@ suite("query37") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query38.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query38.groovy index d76741c55f7c75..06e3857fb30f3f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query38.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query38.groovy @@ -22,6 +22,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query39.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query39.groovy index b9bdc2a49210e0..1a6d2fa5e2ee4a 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query39.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query39.groovy @@ -24,6 +24,7 @@ suite("query39") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query4.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query4.groovy index d8f0f196e4f4d5..2ed84768d45544 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query4.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query4.groovy @@ -24,6 +24,7 @@ suite("query4") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query40.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query40.groovy index d1bd18556fea5a..9daa738d8bf42b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query40.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query40.groovy @@ -24,6 +24,7 @@ suite("query40") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query41.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query41.groovy index 1a772e4381c18b..8cd49d695fc938 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query41.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query41.groovy @@ -24,6 +24,7 @@ suite("query41") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query42.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query42.groovy index b8108863abac71..9a368e763ecc1b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query42.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query42.groovy @@ -24,6 +24,7 @@ suite("query42") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query43.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query43.groovy index 2a7fa4182fb445..ec20d53433519e 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query43.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query43.groovy @@ -24,6 +24,7 @@ suite("query43") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query44.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query44.groovy index dc5ff670d46f92..a6804718503b46 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query44.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query44.groovy @@ -24,6 +24,7 @@ suite("query44") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query45.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query45.groovy index bfd50a87c7c381..3a22a33b595309 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query45.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query45.groovy @@ -24,6 +24,7 @@ suite("query45") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query46.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query46.groovy index add34e3b2b5e1c..3f255f8c6be36b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query46.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query46.groovy @@ -24,6 +24,7 @@ suite("query46") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query47.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query47.groovy index cee7e5b8063ecd..200125adf9e845 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query47.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query47.groovy @@ -24,6 +24,7 @@ suite("query47") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query48.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query48.groovy index a037b912f27ab7..f44e3f3ac46e8f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query48.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query48.groovy @@ -24,6 +24,7 @@ suite("query48") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query49.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query49.groovy index a55f97573559ec..148c9f6fbfa7a2 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query49.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query49.groovy @@ -24,6 +24,7 @@ suite("query49") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query5.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query5.groovy index cc6724b9fe4fbb..3232b380eade74 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query5.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query5.groovy @@ -24,6 +24,7 @@ suite("query5") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query50.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query50.groovy index 3fd2148402984a..77771941874530 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query50.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query50.groovy @@ -24,6 +24,7 @@ suite("query50") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query51.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query51.groovy index 28407a6f99d61e..a8e9d1b590d3cd 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query51.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query51.groovy @@ -24,6 +24,7 @@ suite("query51") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query52.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query52.groovy index d5c5cc65d773eb..4735f2cc493044 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query52.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query52.groovy @@ -24,6 +24,7 @@ suite("query52") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query53.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query53.groovy index 57817314076135..e44ccfed048967 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query53.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query53.groovy @@ -24,6 +24,7 @@ suite("query53") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query54.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query54.groovy index 089fec36a58995..8918b599a3d4ca 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query54.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query54.groovy @@ -24,6 +24,7 @@ suite("query54") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query55.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query55.groovy index aa504a78dc96f9..976dfaa71e1380 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query55.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query55.groovy @@ -24,6 +24,7 @@ suite("query55") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query56.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query56.groovy index d0986a54082146..a311cf8771405b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query56.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query56.groovy @@ -24,6 +24,7 @@ suite("query56") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query57.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query57.groovy index cbe22b62ab37b2..d344c350cc5593 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query57.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query57.groovy @@ -24,6 +24,7 @@ suite("query57") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query58.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query58.groovy index d08b3811f0b826..27238e51637d2e 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query58.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query58.groovy @@ -24,6 +24,7 @@ suite("query58") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query59.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query59.groovy index 0d00b2673d7077..c7df47907a3eb7 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query59.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query59.groovy @@ -24,6 +24,7 @@ suite("query59") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query6.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query6.groovy index 2af04b0f098ed4..ff89e5b12b621f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query6.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query6.groovy @@ -24,6 +24,7 @@ suite("query6") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query60.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query60.groovy index bdc3d839a61575..ddcd041e001be0 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query60.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query60.groovy @@ -24,6 +24,7 @@ suite("query60") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query61.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query61.groovy index 12d07343dc778a..916b9b686c82d2 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query61.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query61.groovy @@ -24,6 +24,7 @@ suite("query61") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query62.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query62.groovy index 5d9185aafc9ede..6d143dabec5686 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query62.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query62.groovy @@ -24,6 +24,7 @@ suite("query62") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query63.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query63.groovy index e0ed0dfc9f4284..d056b177e2bb9b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query63.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query63.groovy @@ -24,6 +24,7 @@ suite("query63") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query65.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query65.groovy index 350f39e995d4e5..a174c367df2993 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query65.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query65.groovy @@ -24,6 +24,7 @@ suite("query65") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query66.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query66.groovy index 5fbfc112265637..c55b1e7298d844 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query66.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query66.groovy @@ -24,6 +24,7 @@ suite("query66") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query68.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query68.groovy index 165e59030045dd..fc2debddbf87b6 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query68.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query68.groovy @@ -24,6 +24,7 @@ suite("query68") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query69.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query69.groovy index e8decd915ffc84..8037d49d7579a6 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query69.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query69.groovy @@ -24,6 +24,7 @@ suite("query69") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query7.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query7.groovy index dc80a626155ebe..205a9f191250df 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query7.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query7.groovy @@ -24,6 +24,7 @@ suite("query7") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query70.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query70.groovy index cf29c62a109ffa..d3751ca94af8cb 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query70.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query70.groovy @@ -24,6 +24,7 @@ suite("query70") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query71.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query71.groovy index fa6cfc8fa6f304..2e5e1ccb85b6ee 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query71.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query71.groovy @@ -24,6 +24,7 @@ suite("query71") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query73.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query73.groovy index fe2ac1c2c6444d..c48eb2e748875e 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query73.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query73.groovy @@ -24,6 +24,7 @@ suite("query73") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query74.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query74.groovy index 551c2b40427da2..944e97c1ef1f57 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query74.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query74.groovy @@ -24,6 +24,7 @@ suite("query74") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query75.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query75.groovy index 4671d4b2cb4525..b91fbb54fa8194 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query75.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query75.groovy @@ -24,6 +24,7 @@ suite("query75") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query76.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query76.groovy index 74f10342af9e1c..c83d452b0545e6 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query76.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query76.groovy @@ -24,6 +24,7 @@ suite("query76") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query77.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query77.groovy index 234e29fc890097..6100f7d42a8a85 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query77.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query77.groovy @@ -24,6 +24,7 @@ suite("query77") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query79.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query79.groovy index b199f9dd8e6126..d4a0140eb6f866 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query79.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query79.groovy @@ -24,6 +24,7 @@ suite("query79") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query8.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query8.groovy index 3847a5791f5267..ae733a9d404418 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query8.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query8.groovy @@ -24,6 +24,7 @@ suite("query8") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query80.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query80.groovy index 8a6cf98923dabc..57fd016217d51b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query80.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query80.groovy @@ -24,6 +24,7 @@ suite("query80") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query81.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query81.groovy index e82f2160399186..8fffbf8cb13a59 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query81.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query81.groovy @@ -24,6 +24,7 @@ suite("query81") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query82.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query82.groovy index 556ff81e6914bf..89bbce3fcaf47f 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query82.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query82.groovy @@ -24,6 +24,7 @@ suite("query82") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query84.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query84.groovy index 77f2be66c31c99..9e08f9874de012 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query84.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query84.groovy @@ -24,6 +24,7 @@ suite("query84") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query85.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query85.groovy index 634cadcdce33e9..ae21cca9e688c7 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query85.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query85.groovy @@ -24,6 +24,7 @@ suite("query85") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query86.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query86.groovy index e9040241c60f09..1756297cd401e4 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query86.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query86.groovy @@ -24,6 +24,7 @@ suite("query86") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query87.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query87.groovy index 63c347e1fc9fda..664c930f03f32b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query87.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query87.groovy @@ -24,6 +24,7 @@ suite("query87") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query88.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query88.groovy index 3d8016e08a9f13..5517225e390144 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query88.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query88.groovy @@ -24,6 +24,7 @@ suite("query88") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query89.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query89.groovy index 10331673c2d79d..50a936d91ec2cf 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query89.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query89.groovy @@ -24,6 +24,7 @@ suite("query89") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query9.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query9.groovy index e178b8830252fc..63359a154eda53 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query9.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query9.groovy @@ -24,6 +24,7 @@ suite("query9") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query90.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query90.groovy index 677bc502d810f9..3e8dd027c8a0fd 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query90.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query90.groovy @@ -24,6 +24,7 @@ suite("query90") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query91.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query91.groovy index dc5dd5fbcde699..cf8c30e1f87466 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query91.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query91.groovy @@ -24,6 +24,7 @@ suite("query91") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query92.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query92.groovy index 92d14918a4cd16..211dda8ed160df 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query92.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query92.groovy @@ -24,6 +24,7 @@ suite("query92") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query93.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query93.groovy index 06dbb93ba58268..513447b1bc0f5d 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query93.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query93.groovy @@ -24,6 +24,7 @@ suite("query93") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query94.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query94.groovy index 0dd0b234f7d50f..a0ca4799a1520b 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query94.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query94.groovy @@ -24,6 +24,7 @@ suite("query94") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query95.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query95.groovy index 41a160948ea6c7..3e32c96597d5dc 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query95.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query95.groovy @@ -24,6 +24,7 @@ suite("query95") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query96.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query96.groovy index db0de98dd96180..df1fe47d768905 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query96.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query96.groovy @@ -24,6 +24,7 @@ suite("query96") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query97.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query97.groovy index 100ad5d9758a73..49ac842916cc83 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query97.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query97.groovy @@ -22,6 +22,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query98.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query98.groovy index c53d64b5ddbfde..348379bef2af16 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query98.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query98.groovy @@ -24,6 +24,7 @@ suite("query98") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_hint_tpcds_p0/shape/query99.groovy b/regression-test/suites/nereids_hint_tpcds_p0/shape/query99.groovy index b6d762ac7465ec..b1f2ac7ab311f9 100644 --- a/regression-test/suites/nereids_hint_tpcds_p0/shape/query99.groovy +++ b/regression-test/suites/nereids_hint_tpcds_p0/shape/query99.groovy @@ -24,6 +24,7 @@ suite("query99") { } sql "use ${db}" sql 'set enable_nereids_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/nereids_p0/ddl/show_variables/show_variables_command.groovy b/regression-test/suites/nereids_p0/ddl/show_variables/show_variables_command.groovy new file mode 100644 index 00000000000000..77caf22cacd40c --- /dev/null +++ b/regression-test/suites/nereids_p0/ddl/show_variables/show_variables_command.groovy @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("show_variables_command") { + sql "UNSET VARIABLE ALL" + checkNereidsExecute("""show variables like 'li_ense'""") + checkNereidsExecute("""show variables where variable_name like 'li_ense'""") + checkNereidsExecute("""show variables where variable_name = 'license'""") + qt_cmd("""show variables like 'li_ense'""") + qt_cmd("""show variables where variable_name like 'li_ense'""") + qt_cmd("""show variables where variable_name = 'license'""") +} diff --git a/regression-test/suites/nereids_p0/ddl/show_view/show_view_command.groovy b/regression-test/suites/nereids_p0/ddl/show_view/show_view_command.groovy new file mode 100644 index 00000000000000..6054625e8fe103 --- /dev/null +++ b/regression-test/suites/nereids_p0/ddl/show_view/show_view_command.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("show_view_command") { + multi_sql """ + drop database if exists test_show_view_db1; + create database test_show_view_db1; + use test_show_view_db1; + + drop table if exists t1; + drop table if exists t2; + + create table t1 + (c1 bigint, c2 bigint) + ENGINE=OLAP + DUPLICATE KEY(c1, c2) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + + create table t2 + (c1 bigint, c2 bigint) + ENGINE=OLAP + DUPLICATE KEY(c1, c2) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + + drop view if exists tv1; + drop view if exists tv2; + + create view tv1 as select t1.c1, t2.c2 from t1 join t2 on t1.c2 = t2.c2; + create view tv2 as select t1.c1 from t1 where t1.c2 = 1; + + drop database if exists test_show_view_db2; + create database test_show_view_db2; + use test_show_view_db2; + + drop table if exists t1; + drop table if exists t2; + + create table t1 + (c1 bigint, c2 bigint) + ENGINE=OLAP + DUPLICATE KEY(c1, c2) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + + create table t2 + (c1 bigint, c2 bigint) + ENGINE=OLAP + DUPLICATE KEY(c1, c2) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + + drop view if exists ttv1; + drop view if exists ttv2; + + create view ttv1 as select t1.c1, t2.c2 from t1 join t2 on t1.c2 = t2.c2; + create view ttv2 as select t1.c1 from t1 where t1.c2 = 1; + """ + checkNereidsExecute("""show view from t1;""") + checkNereidsExecute("""show view from test_show_view_db1.t1;""") + checkNereidsExecute("""show view from test_show_view_db2.t1;""") + checkNereidsExecute("""show view from test_show_view_db1.t1 from test_show_view_db2;""") + qt_cmd("""show view from t1;""") + qt_cmd("""show view from test_show_view_db1.t1;""") + qt_cmd("""show view from test_show_view_db2.t1;""") + qt_cmd("""show view from test_show_view_db1.t1 from test_show_view_db2;""") +} \ No newline at end of file diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_nullable.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_nullable.groovy new file mode 100644 index 00000000000000..4eada47c0f676d --- /dev/null +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_nullable.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("fold_constant_nullable") { + def db = "fold_constant_string_arithmatic" + sql "create database if not exists ${db}" + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_fold_constant_by_be=false" + + // Date and Time Functions + testFoldConst("SELECT Years_Sub(NULL, NULL)") + testFoldConst("SELECT YearWeek(NULL)") + testFoldConst("SELECT Year(NULL)") + testFoldConst("SELECT Weeks_Sub(NULL, NULL)") + testFoldConst("SELECT Weeks_Diff(NULL, NULL)") + testFoldConst("SELECT Weeks_Add(NULL, NULL)") + testFoldConst("SELECT Weekday(NULL)") + testFoldConst("SELECT Week(NULL)") + testFoldConst("SELECT To_Days(NULL)") + testFoldConst("SELECT Timestamp(NULL)") + testFoldConst("SELECT Seconds_Sub(NULL, NULL)") + testFoldConst("SELECT Seconds_Add(NULL, NULL)") + testFoldConst("SELECT Quarter(NULL)") + testFoldConst("SELECT Minute(NULL)") + testFoldConst("SELECT Minutes_Sub(NULL, NULL)") + testFoldConst("SELECT Minutes_Diff(NULL, NULL)") + testFoldConst("SELECT MilliSeconds_Sub(NULL, NULL)") + testFoldConst("SELECT MilliSeconds_Add(NULL, NULL)") + testFoldConst("SELECT Microsecond(NULL)") + testFoldConst("SELECT MicroSeconds_Sub(NULL, NULL)") + testFoldConst("SELECT MicroSeconds_Add(NULL, NULL)") + testFoldConst("SELECT MicroSecond_Timestamp(NULL)") + testFoldConst("SELECT Hours_Sub(NULL, NULL)") + testFoldConst("SELECT Hours_Add(NULL, NULL)") + testFoldConst("SELECT DayOfYear(NULL)") + testFoldConst("SELECT DayName(NULL)") + + // Numeric Functions + testFoldConst("SELECT Width_Bucket(NULL, NULL, NULL, NULL)") + testFoldConst("SELECT Radians(NULL)") + testFoldConst("SELECT Negative(NULL)") + testFoldConst("SELECT Murmur_Hash3_64(NULL)") + testFoldConst("SELECT Fpow(NULL, NULL)") + testFoldConst("SELECT Field(NULL, NULL)") + testFoldConst("SELECT Exp(NULL)") + testFoldConst("SELECT Dround(NULL)") + testFoldConst("SELECT Dfloor(NULL)") + testFoldConst("SELECT Dexp(NULL)") + testFoldConst("SELECT Cos(NULL)") + testFoldConst("SELECT Abs(NULL)") + testFoldConst("SELECT Multiply(NULL, NULL)") + testFoldConst("SELECT 101>null") + testFoldConst("SELECT 101=null") + testFoldConst("SELECT 1+null") + testFoldConst("SELECT 1-null") + testFoldConst("SELECT Bitmap_Remove(NULL, NULL)") + testFoldConst("SELECT Bitmap_Has_Any(NULL, NULL)") + testFoldConst("SELECT Bitmap_And_Not(NULL, NULL)") + testFoldConst("SELECT Bitmap_And(NULL, NULL)") + testFoldConst("SELECT Bit_Length(NULL)") + testFoldConst("SELECT Width_Bucket(NULL, NULL, NULL, NULL)") + + // String Functions + testFoldConst("SELECT Upper(NULL)") + testFoldConst("SELECT Unhex(NULL)") + testFoldConst("SELECT Tokenize(NULL, NULL)") + testFoldConst("SELECT Space(NULL)") + testFoldConst("SELECT Sm3sum(NULL)") + testFoldConst("SELECT Sm3(NULL)") + testFoldConst("SELECT Sha2(NULL, 256)") + testFoldConst("SELECT Md5Sum(NULL)") + testFoldConst("SELECT Md5(NULL)") + testFoldConst("SELECT Lower(NULL)") + testFoldConst("SELECT jsonb_exists_path(NULL, 'key')") + testFoldConst("SELECT json_replace('{\"null\": 1, \"b\": [2, 3]}', '\$', null)") + testFoldConst("SELECT Initcap(NULL)") + testFoldConst("SELECT Hex(NULL)") + testFoldConst("SELECT From_Second(NULL)") + testFoldConst("SELECT From_Millisecond(NULL)") + testFoldConst("SELECT EsQuery(NULL, NULL)") + testFoldConst("SELECT domain_without_www(NULL)") + testFoldConst("SELECT Concat(NULL, 'test')") + testFoldConst("SELECT Character_Length(NULL)") + + // Other Functions + testFoldConst("SELECT Protocol(NULL)") + testFoldConst("SELECT money_format(NULL)") + testFoldConst("SELECT Not(NULL)") + testFoldConst("SELECT 3 in (1, null)") + + + // Geographic and Mathematical Functions + testFoldConst("SELECT St_Y(NULL)") + testFoldConst("SELECT St_X(NULL)") + testFoldConst("SELECT St_Distance_Sphere(NULL, NULL, 1,2)") + testFoldConst("SELECT St_Area_Square_Meters(NULL)") + testFoldConst("SELECT St_Area_Square_Km(NULL)") + testFoldConst("SELECT St_Angle_Sphere(0, NULL, 45, 0)") + testFoldConst("SELECT St_Angle(ST_Point(1, 0),ST_Point(0, 0),NULL)") + testFoldConst("SELECT Sqrt(NULL)") + testFoldConst("SELECT Pmod(NULL, NULL)") + testFoldConst("SELECT Log2(NULL)") + testFoldConst("SELECT Log10(NULL)") + testFoldConst("SELECT Ln(NULL)") + testFoldConst("SELECT Fmod(NULL, NULL)") + testFoldConst("SELECT Dsqrt(NULL)") + testFoldConst("SELECT Dlog10(NULL)") + testFoldConst("SELECT Mod(NULL, NULL)") + testFoldConst("SELECT Divide(NULL, NULL)") + testFoldConst("SELECT Bitmap_Min(NULL)") + testFoldConst("SELECT Bitmap_Max(NULL)") + testFoldConst("SELECT Asin(NULL)") + testFoldConst("SELECT Acos(NULL)") + + // String Functions + testFoldConst("SELECT Sub_Replace(NULL, 'old', 'new')") + testFoldConst("SELECT Rpad(NULL, 5, 'pad')") + testFoldConst("SELECT Repeat(NULL, 2)") + testFoldConst("SELECT regexp_replace_one(NULL, 'pattern', 'replacement')") + testFoldConst("SELECT regexp_replace(NULL, 'pattern', 'replacement')") + testFoldConst("SELECT regexp_extract_all(NULL, 'pattern')") + testFoldConst("SELECT regexp_extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', null)") + testFoldConst("SELECT parse_url('https://doris.apache.org/', NULL)") + testFoldConst("SELECT Lpad(NULL, 5, 'pad')") + testFoldConst("SELECT json_unquote(NULL)") + testFoldConst("SELECT json_length(NULL)") + testFoldConst("SELECT Json_Extract('AbCdE', '([[:lower:]]+)C([[:lower:]]+)', null)") + testFoldConst("SELECT Json_Contains(NULL, 'key')") + testFoldConst("SELECT Get_Json_String(NULL, 'key')") + testFoldConst("SELECT from_base64(NULL)") + testFoldConst("SELECT bitmap_from_string(NULL)") + testFoldConst("SELECT bitmap_from_string(NULL)") + testFoldConst("SELECT bitmap_from_array(NULL)") + testFoldConst("SELECT append_trailing_char_if_absent(NULL, 'char')") + testFoldConst("SELECT Nullable(NULL)") + testFoldConst("SELECT NullIf(NULL, 'value')") + testFoldConst("SELECT Split_Part('hello world', NULL, 1)") + + // Date Functions + testFoldConst("SELECT to_datev2(NULL)") + testFoldConst("SELECT To_Date(NULL)") + testFoldConst("SELECT Str_To_Date('2014-12-21 12:34:56', NULL)") + testFoldConst("SELECT Second_Ceil(NULL)") + testFoldConst("SELECT Month_Floor(NULL)") + testFoldConst("SELECT Month_Ceil(NULL)") + testFoldConst("SELECT Minute_Floor(NULL)") + testFoldConst("SELECT Hour_Floor(NULL)") + testFoldConst("SELECT Hour_Ceil(NULL)") + testFoldConst("SELECT From_Days(NULL)") + testFoldConst("SELECT Day_Floor(NULL)") + testFoldConst("SELECT DateV2(NULL)") + testFoldConst("SELECT Date_Format(NULL, 'format')") + testFoldConst("SELECT Date(NULL)") + testFoldConst("SELECT Convert_Tz(NULL, 'from_tz', 'to_tz')") + testFoldConst("SELECT st_geomfromtext(NULL)") + testFoldConst("SELECT St_Geometryfromtext(NULL)") + testFoldConst("SELECT St_Polyfromtext(NULL)") + testFoldConst("SELECT St_Point(NULL, 1)") + testFoldConst("SELECT St_Linefromtext(NULL)") + testFoldConst("SELECT St_GeomFromWKB(NULL)") + testFoldConst("SELECT St_Polygon(NULL)") +} diff --git a/regression-test/suites/nereids_p0/join/test_outer_join.groovy b/regression-test/suites/nereids_p0/join/test_outer_join.groovy index 0f901c070a9bc8..9264f3fc57959d 100644 --- a/regression-test/suites/nereids_p0/join/test_outer_join.groovy +++ b/regression-test/suites/nereids_p0/join/test_outer_join.groovy @@ -19,6 +19,7 @@ suite("test_outer_join", "nereids_p0") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" + sql "set enable_nereids_distribute_planner=false;" def tbl1 = "test_outer_join1" def tbl2 = "test_outer_join2" diff --git a/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy b/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy new file mode 100644 index 00000000000000..8320af92f48ff5 --- /dev/null +++ b/regression-test/suites/nereids_p0/sql_functions/table_function/posexplode.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("posexplode") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql """ DROP TABLE IF EXISTS table_test """ + sql """ + CREATE TABLE IF NOT EXISTS `table_test`( + `id` INT NULL, + `name` TEXT NULL, + `score` array NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // insert values + sql """ insert into table_test values (0, "zhangsan", ["Chinese","Math","English"]); """ + sql """ insert into table_test values (1, "lisi", ["null"]); """ + sql """ insert into table_test values (2, "wangwu", ["88a","90b","96c"]); """ + sql """ insert into table_test values (3, "lisi2", [null]); """ + sql """ insert into table_test values (4, "amory", NULL); """ + + qt_sql """ select * from table_test order by id; """ + order_qt_explode_sql """ select id,name,score, k,v from table_test lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql """ select id,name,score, k,v from table_test lateral view posexplode_outer(score) tmp as k,v order by id; """ + + // multi lateral view + order_qt_explode_sql_multi """ select id,name,score, k,v,k1,v1 from table_test lateral view posexplode_outer(score) tmp as k,v lateral view posexplode(score) tmp2 as k1,v1 order by id;""" + + // test with alias + order_qt_explode_sql_alias """ select id,name,score, tmp.k, tmp.v from table_test lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql_alias """ select id,name,score, tmp.k, tmp.v from table_test lateral view posexplode_outer(score) tmp as k,v order by id; """ + + order_qt_explode_sql_alias_multi """ select id,name,score, tmp.k, tmp.v, tmp2.k, tmp2.v from table_test lateral view posexplode_outer(score) tmp as k,v lateral view posexplode(score) tmp2 as k,v order by id;""" + + sql """ DROP TABLE IF EXISTS table_test_not """ + sql """ + CREATE TABLE IF NOT EXISTS `table_test_not`( + `id` INT NULL, + `name` TEXT NULL, + `score` array not NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // insert values + sql """ insert into table_test_not values (0, "zhangsan", ["Chinese","Math","English"]); """ + sql """ insert into table_test_not values (1, "lisi", ["null"]); """ + sql """ insert into table_test_not values (2, "wangwu", ["88a","90b","96c"]); """ + sql """ insert into table_test_not values (3, "lisi2", [null]); """ + sql """ insert into table_test_not values (4, "liuba", []); """ + + qt_sql """ select * from table_test_not order by id; """ + order_qt_explode_sql_not """ select id,name,score, k,v from table_test_not lateral view posexplode(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql_not """ select id,name,score, k,v from table_test_not lateral view posexplode_outer(score) tmp as k,v order by id; """ + order_qt_explode_sql_alias_multi2 """ select * from table_test_not lateral view posexplode(score) tmp as e1 lateral view posexplode(score) tmp2 as e2 order by id;""" + sql """ set batch_size = 1; """ + order_qt_explode_sql_alias_multi3 """ select * from table_test_not lateral view posexplode(score) tmp as e1 lateral view posexplode(score) tmp2 as e2 order by id;""" + +} diff --git a/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy b/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy index fc38e3be3372f0..4499a0ede49f70 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy @@ -25,6 +25,7 @@ suite("eliminate_outer_join") { sql 'set be_number_for_test=3' sql "set enable_parallel_result_sink=false;" sql "set disable_join_reorder=true;" + sql "set enable_nereids_distribute_planner=false;" sql """ DROP TABLE IF EXISTS t diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/push_down_alias_through_join.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/push_down_alias_through_join.groovy index c54fd49f47fef8..c4402f0f07d6ce 100644 --- a/regression-test/suites/nereids_rules_p0/filter_push_down/push_down_alias_through_join.groovy +++ b/regression-test/suites/nereids_rules_p0/filter_push_down/push_down_alias_through_join.groovy @@ -24,6 +24,7 @@ suite("push_down_alias_through_join") { sql "set disable_join_reorder=true" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql "set enable_parallel_result_sink=false;" + sql "set enable_nereids_distribute_planner=false;" // Push alias through inner join where condition not use alias qt_pushdown_inner_join""" diff --git a/regression-test/suites/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.groovy b/regression-test/suites/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.groovy index d29bac550fa6e3..b05b47c3461f90 100644 --- a/regression-test/suites/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.groovy +++ b/regression-test/suites/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.groovy @@ -21,6 +21,7 @@ suite("infer_set_operator_distinct") { sql "SET enable_fallback_to_original_planner=false" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql "set enable_parallel_result_sink=false;" + sql "set enable_nereids_distribute_planner=false;" sql """ DROP TABLE IF EXISTS t1; diff --git a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy index 03ddf46fdf1d3b..ba23c29ab9abc4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy @@ -471,7 +471,7 @@ suite("negative_partition_mv_rewrite") { on lineitem_1.l_orderkey > orders_1.o_orderkey group by l_shipdate, o_orderdate, l_partkey """ - mv_rewrite_fail(query_sql, mv_name) + mv_rewrite_success(query_sql, mv_name) // mtmv exists join but not exists agg, query exists agg mtmv_sql = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy new file mode 100644 index 00000000000000..68337b00d09da8 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/anti/other_join_conjuncts_anti.groovy @@ -0,0 +1,459 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("other_join_conjuncts_anti") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (1, 2, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (2, 4, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (3, 2, 4, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (4, 3, 3, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'), + (5, 2, 3, 6, 7.6, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (1, 1, 'o', 10.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy','2023-12-07'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (2, 1, 'o', 11.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-11', 'a', 'b', 1, 'yy','2023-12-11'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy','2023-12-10'), + (3, 1, 'o', 12.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (3, 1, 'o', 12.5, '2023-12-12', 'a', 'b', 1, 'yy','2023-12-12'), + (3, 1, 'o', 33.5, '2023-12-13', 'a', 'b', 1, 'yy','2023-12-13'), + (4, 2, 'o', 43.2, '2023-12-10', 'c','d',2, 'mm' ,'2023-12-10'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm' ,'2023-12-11'), + (4, 2, 'o', 43.2, '2023-12-13', 'c','d',2, 'mm' ,'2023-12-13'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'), + (5, 2, 'o', 56.2, '2023-12-14', 'c','d',2, 'mi' ,'2023-12-14'), + (5, 2, 'o', 56.2, '2023-12-16', 'c','d',2, 'mi' ,'2023-12-16'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + + // =, !=, >, <, <=, >= + // left anti join other conjuncts in join condition + def mv1_0 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query1_0 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success_without_check_chosen(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + def mv1_4 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left anti join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_4 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left anti join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_4_before "${query1_4}" + // other conjuncts is before equal conjuncts, should success + async_mv_rewrite_success_without_check_chosen(db, mv1_4, query1_4, "mv1_4") + order_qt_query1_4_after "${query1_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" + + + + def mv1_5 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left anti join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_5 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left anti join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_5_before "${query1_5}" + // other conjuncts has the same column name + async_mv_rewrite_success_without_check_chosen(db, mv1_5, query1_5, "mv1_5") + order_qt_query1_5_after "${query1_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" + + + def mv1_1 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query1_1 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query1_1_before "${query1_1}" + async_mv_rewrite_fail(db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + def mv1_2 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query1_2 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey + left anti join partsupp on ps_partkey = l_partkey; + """ + order_qt_query1_2_before "${query1_2}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv1_2, query1_2, "mv1_2") + order_qt_query1_2_after "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + def mv1_3 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey + left anti join partsupp on ps_partkey = l_partkey; + """ + def query1_3 = + """ + select l_orderkey + from + lineitem + left anti join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left anti join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query1_3_before "${query1_3}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv1_3, query1_3, "mv1_3") + order_qt_query1_3_after "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + + // right anti join other conjuncts in join condition + def mv2_0 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query2_0 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + order_qt_query2_0_before "${query2_0}" + async_mv_rewrite_success_without_check_chosen(db, mv2_0, query2_0, "mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" + + + def mv2_1 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query2_1 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate; + """ + order_qt_query2_1_before "${query2_1}" + async_mv_rewrite_fail(db, mv2_1, query2_1, "mv2_1") + order_qt_query2_1_after "${query2_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_1""" + + def mv2_2 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query2_2 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey; + """ + order_qt_query2_2_before "${query2_2}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv2_2, query2_2, "mv2_2") + order_qt_query2_2_after "${query2_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_2""" + + + def mv2_3 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey; + """ + def query2_3 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate; + """ + order_qt_query2_3_before "${query2_3}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv2_3, query2_3, "mv2_3") + order_qt_query2_3_after "${query2_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_3""" + + + def mv2_4 = + """ + select L_orderkeY + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdatE <= o_orderdatE; + """ + def query2_4 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + order_qt_query2_4_before "${query2_4}" + // Case sensitivity of column names in query and mv, should success + async_mv_rewrite_success_without_check_chosen(db, mv2_4, query2_4, "mv2_4") + order_qt_query2_4_after "${query2_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_4""" + + + def mv2_5 = + """ + select L_orderkeY + from + orders + right anti join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdatE, 'day') <= o_orderdatE; + """ + def query2_5 = + """ + select l_orderkey + from + orders + right anti join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate; + """ + order_qt_query2_5_before "${query2_5}" + // Complex expressions + async_mv_rewrite_success_without_check_chosen(db, mv2_5, query2_5, "mv2_5") + order_qt_query2_5_after "${query2_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_5""" + + + // left self join + def mv3_0 = + """ + select l1.l_orderkey + from + lineitem l1 + left anti join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount + """ + def query3_0 = + """ + select l1.l_orderkey + from + lineitem l1 + left anti join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query3_0_before "${query3_0}" + async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" + + // right self join + def mv4_0 = + """ + select l2.l_orderkey + from + lineitem l1 + right anti join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount + """ + def query4_0 = + """ + select l2.l_orderkey + from + lineitem l1 + right anti join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query4_0_before "${query4_0}" + async_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0") + order_qt_query4_0_after "${query4_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" + +} diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy new file mode 100644 index 00000000000000..b79e5808932044 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/inner/other_join_conjuncts_inner.groovy @@ -0,0 +1,873 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("other_join_conjuncts_inner") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (1, 2, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (2, 4, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (3, 2, 4, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (4, 3, 3, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'), + (5, 2, 3, 6, 7.6, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (1, 1, 'o', 10.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy','2023-12-07'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (2, 1, 'o', 11.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-11', 'a', 'b', 1, 'yy','2023-12-11'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy','2023-12-10'), + (3, 1, 'o', 12.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (3, 1, 'o', 12.5, '2023-12-12', 'a', 'b', 1, 'yy','2023-12-12'), + (3, 1, 'o', 33.5, '2023-12-13', 'a', 'b', 1, 'yy','2023-12-13'), + (4, 2, 'o', 43.2, '2023-12-10', 'c','d',2, 'mm' ,'2023-12-10'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm' ,'2023-12-11'), + (4, 2, 'o', 43.2, '2023-12-13', 'c','d',2, 'mm' ,'2023-12-13'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'), + (5, 2, 'o', 56.2, '2023-12-14', 'c','d',2, 'mi' ,'2023-12-14'), + (5, 2, 'o', 56.2, '2023-12-16', 'c','d',2, 'mi' ,'2023-12-16'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + + // =, !=, >, <, <=, >= + // other conjuncts in join condition + def mv1_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + def mv1_6 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + inner join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_6 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + inner join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_6_before "${query1_6}" + // other conjuncts is before equal conjuncts, should success + async_mv_rewrite_success(db, mv1_6, query1_6, "mv1_6") + order_qt_query1_6_after "${query1_6}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_6""" + + + def mv1_7 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + inner join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_7 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + inner join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_7_before "${query1_7}" + // other conjuncts has the same column name + async_mv_rewrite_success(db, mv1_7, query1_7, "mv1_7") + order_qt_query1_7_after "${query1_7}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_7""" + + def mv1_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_1_before "${query1_1}" + // query other conjucts is different from mv + async_mv_rewrite_fail(db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + def mv1_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_2_before "${query1_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv1_2, query1_2, "mv1_2") + order_qt_query1_2_after "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + // self join + def mv1_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + def query1_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query1_3_before "${query1_3}" + async_mv_rewrite_success(db, mv1_3, query1_3, "mv1_3") + order_qt_query1_3_after "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + + + def mv1_4 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_4 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey + inner join partsupp on ps_partkey; + """ + order_qt_query1_4_before "${query1_4}" + // mv has the other conjuncts but query not + async_mv_rewrite_fail(db, mv1_4, query1_4, "mv1_4") + order_qt_query1_4_after "${query1_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" + + + def mv1_5 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey; + """ + def query1_5 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + inner join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + inner join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_5_before "${query1_5}" + // query has other the conjuncts but mv not + async_mv_rewrite_fail(db, mv1_5, query1_5, "mv1_5") + order_qt_query1_5_after "${query1_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" + + + // other conjuncts above join + def mv2_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query2_0_before "${query2_0}" + async_mv_rewrite_success(db, mv2_0, query2_0, "mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" + + + def mv2_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query2_1_before "${query2_1}" + // query other conjucts is different from mv + async_mv_rewrite_fail(db, mv2_1, query2_1, "mv2_1") + order_qt_query2_1_after "${query2_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_1""" + + + def mv2_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query2_2_before "${query2_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv2_2, query2_2, "mv2_2") + order_qt_query2_2_after "${query2_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_2""" + + + // self join + def mv2_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + def query2_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query2_3_before "${query2_3}" + async_mv_rewrite_success(db, mv2_3, query2_3, "mv2_3") + order_qt_query2_3_after "${query2_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_3""" + + + def mv2_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey; + """ + order_qt_query2_4_before "${query2_4}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv2_4, query2_4, "mv2_4") + order_qt_query2_4_after "${query2_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_4""" + + + def mv2_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey; + """ + def query2_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query2_5_before "${query2_5}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv2_5, query2_5, "mv2_5") + order_qt_query2_5_after "${query2_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_5""" + + + def mv2_6 = + """ + select + o_ordeRdatE, + o_shippriority, + o_commenT, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdatE <= o_orderdatE; + """ + def query2_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query2_6_before "${query2_6}" + // Case sensitivity of column names in query and mv, should success + async_mv_rewrite_success(db, mv2_6, query2_6, "mv2_6") + order_qt_query2_6_after "${query2_6}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_6""" + + + // other conjuncts both above join and in join other conjuncts + def mv3_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_0_before "${query3_0}" + async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" + + + def mv3_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_6_before "${query3_6}" + // Combinations of different join types + async_mv_rewrite_success(db, mv3_6, query3_6, "mv3_6") + order_qt_query3_6_after "${query3_6}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_6""" + + + + def mv3_7 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_7 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_7_before "${query3_7}" + // Combinations of different join types + async_mv_rewrite_success(db, mv3_7, query3_7, "mv3_7") + order_qt_query3_7_after "${query3_7}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_7""" + + + def mv3_8 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_8 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_8_before "${query3_8}" + // Complex expressions + async_mv_rewrite_success(db, mv3_8, query3_8, "mv3_8") + order_qt_query3_8_after "${query3_8}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_8""" + + def mv3_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_1_before "${query3_1}" + // query other conjucts is different from mv + async_mv_rewrite_fail(db, mv3_1, query3_1, "mv3_1") + order_qt_query3_1_after "${query3_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1""" + + + def mv3_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_2_before "${query3_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv3_2, query3_2, "mv3_2") + order_qt_query3_2_after "${query3_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_2""" + + + // self join + def mv3_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate + where l1.l_extendedprice != l2.l_discount; + """ + def query3_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + inner join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate + where l1.l_extendedprice != l2.l_discount; + """ + order_qt_query3_3_before "${query3_3}" + async_mv_rewrite_success(db, mv3_3, query3_3, "mv3_3") + order_qt_query3_3_after "${query3_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_3""" + + + def mv3_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey; + """ + order_qt_query3_4_before "${query3_4}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv3_4, query3_4, "mv3_4") + order_qt_query3_4_after "${query3_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4""" + + + def mv3_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + inner join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + inner join lineitem on l_orderkey = o_orderkey + inner join partsupp on ps_partkey = l_partkey; + """ + order_qt_query3_5_before "${query3_5}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv3_5, query3_5, "mv3_5") + order_qt_query3_5_after "${query3_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_5""" +} diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy new file mode 100644 index 00000000000000..8a434364c89168 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/outer/other_join_conjuncts_outer.groovy @@ -0,0 +1,1503 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("other_join_conjuncts_outer") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (1, 2, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (2, 4, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (3, 2, 4, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (4, 3, 3, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'), + (5, 2, 3, 6, 7.6, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (1, 1, 'o', 10.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy','2023-12-07'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (2, 1, 'o', 11.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-11', 'a', 'b', 1, 'yy','2023-12-11'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy','2023-12-10'), + (3, 1, 'o', 12.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (3, 1, 'o', 12.5, '2023-12-12', 'a', 'b', 1, 'yy','2023-12-12'), + (3, 1, 'o', 33.5, '2023-12-13', 'a', 'b', 1, 'yy','2023-12-13'), + (4, 2, 'o', 43.2, '2023-12-10', 'c','d',2, 'mm' ,'2023-12-10'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm' ,'2023-12-11'), + (4, 2, 'o', 43.2, '2023-12-13', 'c','d',2, 'mm' ,'2023-12-13'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'), + (5, 2, 'o', 56.2, '2023-12-14', 'c','d',2, 'mi' ,'2023-12-14'), + (5, 2, 'o', 56.2, '2023-12-16', 'c','d',2, 'mi' ,'2023-12-16'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + + // =, !=, >, <, <=, >= + // left outer join + // other conjuncts in join condition + def mv1_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + def mv1_4 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left outer join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_4 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left outer join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_4_before "${query1_4}" + // other conjuncts is before equal conjuncts, should success + async_mv_rewrite_success(db, mv1_4, query1_4, "mv1_4") + order_qt_query1_4_after "${query1_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" + + + def mv1_5 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left outer join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query1_5 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left outer join partsupp on l_orderkey + o_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query1_5_before "${query1_5}" + // other conjuncts has the same column name + async_mv_rewrite_success(db, mv1_5, query1_5, "mv1_5") + order_qt_query1_5_after "${query1_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" + + + def mv1_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_1_before "${query1_1}" + async_mv_rewrite_fail(db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + def mv1_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query1_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + left outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query1_2_before "${query1_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv1_2, query1_2, "mv1_2") + order_qt_query1_2_after "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + + // self join + def mv1_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + left outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + def query1_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + left outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query1_3_before "${query1_3}" + async_mv_rewrite_success(db, mv1_3, query1_3, "mv1_3") + order_qt_query1_3_after "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + + // other conjuncts above join + def mv2_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query2_0_before "${query2_0}" + async_mv_rewrite_success(db, mv2_0, query2_0, "mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" + + + def mv2_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query2_1_before "${query2_1}" + async_mv_rewrite_fail(db, mv2_1, query2_1, "mv2_1") + order_qt_query2_1_after "${query2_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_1""" + + + def mv2_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query2_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query2_2_before "${query2_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv2_2, query2_2, "mv2_2") + order_qt_query2_2_after "${query2_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_2""" + + + // other conjuncts both above join and in join other conjuncts + def mv3_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_0_before "${query3_0}" + async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" + + + def mv3_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_1_before "${query3_1}" + async_mv_rewrite_fail(db, mv3_1, query3_1, "mv3_1") + order_qt_query3_1_after "${query3_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1""" + + + def mv3_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_2_before "${query3_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv3_2, query3_2, "mv3_2") + order_qt_query3_2_after "${query3_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_2""" + + + def mv3_3 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_3 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey; + """ + order_qt_query3_3_before "${query3_3}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv3_3, query3_3, "mv3_3") + order_qt_query3_3_after "${query3_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_3""" + + + def mv3_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey + left outer join partsupp on ps_partkey = l_partkey; + """ + def query3_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_4_before "${query3_4}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv3_4, query3_4, "mv3_4") + order_qt_query3_4_after "${query3_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4""" + + + def mv3_5 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_5 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_5_before "${query3_5}" + // Combinations of different join types + async_mv_rewrite_success(db, mv3_5, query3_5, "mv3_5") + order_qt_query3_5_after "${query3_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_5""" + + + def mv3_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query3_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query3_6_before "${query3_6}" + // Complex expressions + async_mv_rewrite_success(db, mv3_6, query3_6, "mv3_6") + order_qt_query3_6_after "${query3_6}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_6""" + + + // right outer join + // other conjuncts in join condition + def mv4_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query4_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query4_0_before "${query4_0}" + async_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0") + order_qt_query4_0_after "${query4_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" + + + def mv4_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query4_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query4_1_before "${query4_1}" + async_mv_rewrite_fail(db, mv4_1, query4_1, "mv4_1") + order_qt_query4_1_after "${query4_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_1""" + + + def mv4_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query4_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query4_2_before "${query4_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv4_2, query4_2, "mv4_2") + order_qt_query4_2_after "${query4_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_2""" + + def mv4_3 = + """ + select l_orderkEY, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderKey != ps_availQty; + """ + def query4_3 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query4_3_before "${query4_3}" + // Case sensitivity of column names in query and mv, should success + async_mv_rewrite_success(db, mv4_3, query4_3, "mv4_3") + order_qt_query4_3_after "${query4_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_3""" + + + def mv4_4 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query4_4 = + """ + select l_orderkey, o_orderdate + from + lineitem + right outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query4_4_before "${query4_4}" + // Combinations of different join types + async_mv_rewrite_success(db, mv4_4, query4_4, "mv4_4") + order_qt_query4_4_after "${query4_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_4""" + + // other conjuncts above join + def mv5_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query5_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query5_0_before "${query5_0}" + async_mv_rewrite_success(db, mv5_0, query5_0, "mv5_0") + order_qt_query5_0_after "${query5_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_0""" + + + def mv5_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query5_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query5_1_before "${query5_1}" + async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1") + order_qt_query5_1_after "${query5_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1""" + + + def mv5_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query5_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query_5_2_before "${query5_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2") + order_qt_query5_2_after "${query5_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2""" + + // self join + def mv5_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + right outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + def query5_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + right outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query5_3_before "${query5_3}" + async_mv_rewrite_success(db, mv5_3, query5_3, "mv5_3") + order_qt_query5_3_after "${query5_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_3""" + + + // other conjuncts both above join and in join other conjuncts + def mv6_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query6_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query6_0_before "${query6_0}" + async_mv_rewrite_success(db, mv6_0, query6_0, "mv6_0") + order_qt_query6_0_after "${query6_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0""" + + + def mv6_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query6_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query6_1_before "${query6_1}" + async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1") + order_qt_query6_1_after "${query6_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1""" + + + def mv6_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query6_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query6_2_before "${query6_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv6_2, query6_2, "mv6_2") + order_qt_query6_2_after "${query6_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_2""" + + + def mv6_3 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query6_3 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey; + """ + order_qt_query6_3_before "${query6_3}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv6_3, query6_3, "mv6_3") + order_qt_query6_3_after "${query6_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_3""" + + + def mv6_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey + right outer join partsupp on ps_partkey = l_partkey; + """ + def query6_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + right outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + right outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query6_4_before "${query6_4}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv6_4, query6_4, "mv6_4") + order_qt_query6_4_after "${query6_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_4""" + + + // full outer join + // other conjuncts in join condition + def mv7_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query7_0 = + """ + select l_orderkey, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query7_0_before "${query7_0}" + async_mv_rewrite_success(db, mv7_0, query7_0, "mv7_0") + order_qt_query7_0_after "${query7_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0""" + + + def mv7_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query7_1 = + """ + select l_orderkey, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query7_7_before "${query7_1}" + async_mv_rewrite_fail(db, mv7_1, query7_1, "mv7_1") + order_qt_query7_7_after "${query7_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_1""" + + + def mv7_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + def query7_2 = + """ + select l_orderkey, l_shipdate, o_orderdate + from + lineitem + full outer join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + full outer join partsupp on ps_partkey = l_partkey and l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query7_2_before "${query7_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv7_2, query7_2, "mv7_2") + order_qt_query7_2_after "${query7_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_2""" + + // other conjuncts above join + def mv8_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query8_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + order_qt_query8_0_before "${query8_0}" + async_mv_rewrite_success(db, mv8_0, query8_0, "mv8_0") + order_qt_query8_0_after "${query8_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv8_0""" + + + def mv8_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query8_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query8_1_before "${query8_1}" + async_mv_rewrite_fail(db, mv8_1, query8_1, "mv8_1") + order_qt_query8_1_after "${query8_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv8_1""" + + + def mv8_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate <= o_orderdate; + """ + def query8_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty and l_shipdate < o_orderdate; + """ + order_qt_query8_2_before "${query8_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv8_2, query8_2, "mv8_2") + order_qt_query8_2_after "${query8_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv8_2""" + + + // other conjuncts both above join and in join other conjuncts + def mv9_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_0 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_0_before "${query9_0}" + async_mv_rewrite_success(db, mv9_0, query9_0, "mv9_0") + order_qt_query9_0_after "${query9_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_0""" + + + def mv9_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_1 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_1_before "${query9_1}" + async_mv_rewrite_fail(db, mv9_1, query9_1, "mv9_1") + order_qt_query9_1_after "${query9_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_1""" + + + def mv9_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_2 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_2_before "${query9_2}" + // though select has the compensate filter column, should fail + async_mv_rewrite_fail(db, mv9_2, query9_2, "mv9_2") + order_qt_query9_2_after "${query9_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_2""" + + + // self join + def mv9_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + full outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + def query9_3 = + """ + select l1.l_orderkey, l2.l_shipdate + from + lineitem l1 + full outer join + lineitem l2 on l1.l_orderkey = l2.l_orderkey + where l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query9_3_before "${query9_3}" + async_mv_rewrite_success(db, mv9_3, query9_3, "mv9_3") + order_qt_query9_3_after "${query9_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_3""" + + def mv9_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_4 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey; + """ + order_qt_query9_4_before "${query9_4}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv9_4, query9_4, "mv9_4") + order_qt_query9_4_after "${query9_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_4""" + + + def mv9_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey + full outer join partsupp on ps_partkey = l_partkey; + """ + def query9_5 = + """ + select + o_orderdate, + l_shipdate, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_5_before "${query9_5}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv9_5, query9_5, "mv9_5") + order_qt_query9_5_after "${query9_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_5""" + + + def mv9_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_6 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + left outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + full outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_6_before "${query9_6}" + // Combinations of different join types + async_mv_rewrite_success(db, mv9_6, query9_6, "mv9_6") + order_qt_query9_6_after "${query9_6}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_6""" + + + def mv9_7 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + def query9_7 = + """ + select + o_orderdate, + o_shippriority, + o_comment, + l_orderkey, + ps_partkey + from + orders + full outer join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left outer join partsupp on ps_partkey = l_partkey + where l_orderkey + o_orderkey != ps_availqty; + """ + order_qt_query9_7_before "${query9_7}" + // Combinations of different join types + async_mv_rewrite_success(db, mv9_7, query9_7, "mv9_7") + order_qt_query9_7_after "${query9_7}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_7""" +} + + diff --git a/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy new file mode 100644 index 00000000000000..30c21c77269e1c --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/other_join_conjuncts/semi/other_join_conjuncts_semi.groovy @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("other_join_conjuncts_semi") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + lo_orderdate DATE NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (1, 2, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-08'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (2, 4, 3, 4, 5.5, 6.5, 7.6, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-09'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (3, 2, 4, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-10'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (4, 3, 3, 4, 5.5, 6.6, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy' ,'2023-12-11'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'), + (5, 2, 3, 6, 7.6, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', '2023-12-12'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (1, 1, 'o', 10.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy','2023-12-07'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (2, 1, 'o', 11.5, '2023-12-08', 'a', 'b', 1, 'yy','2023-12-08'), + (2, 1, 'o', 11.5, '2023-12-11', 'a', 'b', 1, 'yy','2023-12-11'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy','2023-12-10'), + (3, 1, 'o', 12.5, '2023-12-09', 'a', 'b', 1, 'yy','2023-12-09'), + (3, 1, 'o', 12.5, '2023-12-12', 'a', 'b', 1, 'yy','2023-12-12'), + (3, 1, 'o', 33.5, '2023-12-13', 'a', 'b', 1, 'yy','2023-12-13'), + (4, 2, 'o', 43.2, '2023-12-10', 'c','d',2, 'mm' ,'2023-12-10'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm' ,'2023-12-11'), + (4, 2, 'o', 43.2, '2023-12-13', 'c','d',2, 'mm' ,'2023-12-13'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'), + (5, 2, 'o', 56.2, '2023-12-14', 'c','d',2, 'mi' ,'2023-12-14'), + (5, 2, 'o', 56.2, '2023-12-16', 'c','d',2, 'mi' ,'2023-12-16'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi' ,'2023-12-12'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + + // left self join + def mv1_0 = + """ + select l1.l_orderkey + from + lineitem l1 + left semi join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount + """ + def query1_0 = + """ + select l1.l_orderkey + from + lineitem l1 + left semi join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + // right self join + def mv2_0 = + """ + select l2.l_orderkey + from + lineitem l1 + right semi join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount + """ + def query2_0 = + """ + select l2.l_orderkey + from + lineitem l1 + right semi join + lineitem l2 on l1.l_orderkey = l2.l_orderkey and l1.l_shipdate <= l2.l_receiptdate and l1.l_extendedprice != l2.l_discount; + """ + order_qt_query2_0_before "${query2_0}" + async_mv_rewrite_success(db, mv2_0, query2_0, "mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" + + // left semi join other conjuncts in join condition + def mv3_0 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query3_0 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query3_0_before "${query3_0}" + async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" + + + def mv3_4 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left semi join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query3_4 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey + left semi join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query3_4_before "${query3_4}" + // other conjuncts is before equal conjuncts, should success + async_mv_rewrite_success(db, mv3_4, query3_4, "mv3_4") + order_qt_query3_4_after "${query3_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4""" + + + def mv3_5 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left semi join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + def query3_5 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_shipdate <= o_orderdate and l_orderkey = o_orderkey and lineitem.lo_orderdate <= orders.lo_orderdate + left semi join partsupp on l_orderkey != ps_availqty and ps_partkey = l_partkey; + """ + order_qt_query3_5_before "${query3_5}" + // other conjuncts has the same column name + async_mv_rewrite_success(db, mv3_5, query3_5, "mv3_5") + order_qt_query3_5_after "${query3_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_5""" + + def mv3_1 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query3_1 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query3_1_before "${query3_1}" + async_mv_rewrite_fail(db, mv3_1, query3_1, "mv3_1") + order_qt_query3_1_after "${query3_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1""" + + def mv3_2 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate <= o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + def query3_2 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey + left semi join partsupp on ps_partkey = l_partkey; + """ + order_qt_query3_2_before "${query3_2}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv3_2, query3_2, "mv3_2") + order_qt_query3_2_after "${query3_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_2""" + + def mv3_3 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey + left semi join partsupp on ps_partkey = l_partkey; + """ + def query3_3 = + """ + select l_orderkey + from + lineitem + left semi join + orders on l_orderkey = o_orderkey and l_shipdate < o_orderdate + left semi join partsupp on ps_partkey = l_partkey and l_orderkey != ps_availqty; + """ + order_qt_query3_3_before "${query3_3}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv3_3, query3_3, "mv3_3") + order_qt_query3_3_after "${query3_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_3""" + + + // right semi join other conjuncts in join condition + def mv4_0 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query4_0 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + order_qt_query4_0_before "${query4_0}" + async_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0") + order_qt_query4_0_after "${query4_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" + + + def mv4_1 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query4_1 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate; + """ + order_qt_query4_1_before "${query4_1}" + async_mv_rewrite_fail(db, mv4_1, query4_1, "mv4_1") + order_qt_query4_1_after "${query4_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_1""" + + + def mv4_2 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + def query4_2 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey; + """ + order_qt_query4_2_before "${query4_2}" + // mv has other conjuncts but query not + async_mv_rewrite_fail(db, mv4_2, query4_2, "mv4_2") + order_qt_query4_2_after "${query4_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_2""" + + + def mv4_3 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey; + """ + def query4_3 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate < o_orderdate; + """ + order_qt_query4_3_before "${query4_3}" + // query has other conjuncts but mv not + async_mv_rewrite_fail(db, mv4_3, query4_3, "mv4_3") + order_qt_query4_3_after "${query4_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_3""" + + + def mv4_4 = + """ + select l_orderkEY + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipDate <= o_orderDate; + """ + def query4_4 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and l_shipdate <= o_orderdate; + """ + order_qt_query4_4_before "${query4_4}" + // Case sensitivity of column names in query and mv, should success + async_mv_rewrite_success(db, mv4_4, query4_4, "mv4_4") + order_qt_query4_4_after "${query4_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_4""" + + + + def mv4_5 = + """ + select l_orderkEY + from + orders + right semi join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipDate, 'day') <= o_orderDate; + """ + def query4_5 = + """ + select l_orderkey + from + orders + right semi join lineitem on l_orderkey = o_orderkey and date_trunc(l_shipdate, 'day') <= o_orderdate; + """ + order_qt_query4_5_before "${query4_5}" + // Complex expressions + async_mv_rewrite_success(db, mv4_5, query4_5, "mv4_5") + order_qt_query4_5_after "${query4_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_5""" +} diff --git a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy index 2e89a2124117e9..6d5524bffaa915 100644 --- a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy @@ -181,7 +181,6 @@ suite("mv_tpch_test") { n_name, s_name, p_partkey - LIMIT 100; """ def query2 = """ SELECT @@ -282,7 +281,7 @@ suite("mv_tpch_test") { ORDER BY revenue DESC, o_orderdate - LIMIT 10; + LIMIT 10; """ // contains limit, doesn't support now order_qt_query3_before "${query3}" @@ -291,6 +290,60 @@ suite("mv_tpch_test") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv3""" + def mv3_1 = """ + SELECT + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + o_orderdate, + o_shippriority + FROM + customer, + orders, + lineitem + WHERE + c_mktsegment = 'BUILDING' + AND c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderdate < DATE '1995-03-15' + AND l_shipdate > DATE '1995-03-15' + GROUP BY + l_orderkey, + o_orderdate, + o_shippriority + ORDER BY + revenue DESC, + o_orderdate + """ + def query3_1 = """ + SELECT + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + o_orderdate, + o_shippriority + FROM + customer, + orders, + lineitem + WHERE + c_mktsegment = 'BUILDING' + AND c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderdate < DATE '1995-03-15' + AND l_shipdate > DATE '1995-03-15' + GROUP BY + l_orderkey, + o_orderdate, + o_shippriority + ORDER BY + revenue DESC, + o_orderdate + LIMIT 10; + """ + order_qt_query3_1_before "${query3_1}" + async_mv_rewrite_success(db, mv3_1, query3_1, "mv3_1") + order_qt_query3_1_after "${query3_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1""" + def mv4 = """ SELECT o_orderpriority, @@ -309,7 +362,7 @@ suite("mv_tpch_test") { GROUP BY o_orderpriority ORDER BY - o_orderpriority + o_orderpriority """ def query4 = """ SELECT @@ -329,7 +382,7 @@ suite("mv_tpch_test") { GROUP BY o_orderpriority ORDER BY - o_orderpriority + o_orderpriority """ // contains subquery, doesn't support now order_qt_query4_before "${query4}" @@ -362,7 +415,7 @@ suite("mv_tpch_test") { GROUP BY n_name ORDER BY - revenue DESC + revenue DESC """ def query5 = """ SELECT @@ -388,7 +441,7 @@ suite("mv_tpch_test") { GROUP BY n_name ORDER BY - revenue DESC + revenue DESC """ order_qt_query5_before "${query5}" async_mv_rewrite_success(db, mv5, query5, "mv5") @@ -414,7 +467,7 @@ suite("mv_tpch_test") { l_shipdate >= DATE '1994-01-01' AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' YEAR AND l_discount BETWEEN 0.06 - 0.01 AND .06 + 0.01 - AND l_quantity < 24 + AND l_quantity < 24 """ order_qt_query6_before "${query6}" async_mv_rewrite_success(db, mv6, query6, "mv6") @@ -460,7 +513,7 @@ suite("mv_tpch_test") { ORDER BY supp_nation, cust_nation, - l_year + l_year """ def query7 = """ SELECT @@ -500,11 +553,10 @@ suite("mv_tpch_test") { ORDER BY supp_nation, cust_nation, - l_year + l_year """ - // contains subquery, doesn't support now order_qt_query7_before "${query7}" - async_mv_rewrite_fail(db, mv7, query7, "mv7") + async_mv_rewrite_success(db, mv7, query7, "mv7") order_qt_query7_after "${query7}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv7""" @@ -546,7 +598,7 @@ suite("mv_tpch_test") { GROUP BY o_year ORDER BY - o_year + o_year """ def query8 = """ SELECT @@ -585,7 +637,7 @@ suite("mv_tpch_test") { GROUP BY o_year ORDER BY - o_year + o_year """ order_qt_query8_before "${query8}" async_mv_rewrite_success(db, mv8, query8, "mv8") @@ -624,7 +676,7 @@ suite("mv_tpch_test") { o_year ORDER BY nation, - o_year DESC + o_year DESC """ def query9 = """ SELECT @@ -657,7 +709,7 @@ suite("mv_tpch_test") { o_year ORDER BY nation, - o_year DESC + o_year DESC """ order_qt_query9_before "${query9}" async_mv_rewrite_success(db, mv9, query9, "mv9") @@ -697,7 +749,7 @@ suite("mv_tpch_test") { c_comment ORDER BY revenue DESC - LIMIT 20 + LIMIT 20 """ def query10 = """ SELECT @@ -731,7 +783,7 @@ suite("mv_tpch_test") { c_comment ORDER BY revenue DESC - LIMIT 20 + LIMIT 20 """ // contains limit, doesn't support now order_qt_query10_before "${query10}" @@ -740,6 +792,78 @@ suite("mv_tpch_test") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv10""" + def mv10_1 = """ + SELECT + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment + FROM + customer, + orders, + lineitem, + nation + WHERE + c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderdate >= DATE '1993-10-01' + AND o_orderdate < DATE '1993-10-01' + INTERVAL '3' MONTH + AND l_returnflag = 'R' + AND c_nationkey = n_nationkey + GROUP BY + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment + ORDER BY + revenue DESC; + """ + def query10_1 = """ + SELECT + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment + FROM + customer, + orders, + lineitem, + nation + WHERE + c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderdate >= DATE '1993-10-01' + AND o_orderdate < DATE '1993-10-01' + INTERVAL '3' MONTH + AND l_returnflag = 'R' + AND c_nationkey = n_nationkey + GROUP BY + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment + ORDER BY + revenue DESC + LIMIT 20; + """ + order_qt_query10_1_before "${query10_1}" + async_mv_rewrite_success(db, mv10_1, query10_1, "mv10_1") + order_qt_query10_1_after "${query10_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv10_1""" + def mv11 = """ SELECT ps_partkey, @@ -767,7 +891,7 @@ suite("mv_tpch_test") { AND n_name = 'GERMANY' ) ORDER BY - value DESC + value DESC """ def query11 = """ SELECT @@ -796,7 +920,7 @@ suite("mv_tpch_test") { AND n_name = 'GERMANY' ) ORDER BY - value DESC + value DESC """ // contains subquery, doesn't support now order_qt_query11_before "${query11}" @@ -863,7 +987,7 @@ suite("mv_tpch_test") { GROUP BY l_shipmode ORDER BY - l_shipmode + l_shipmode """ order_qt_query12_before "${query12}" async_mv_rewrite_success(db, mv12, query12, "mv12") @@ -913,7 +1037,7 @@ suite("mv_tpch_test") { c_count ORDER BY custdist DESC, - c_count DESC + c_count DESC """ // when aggregate rewrite, should only contains one aggregate order_qt_query13_before "${query13}" @@ -948,7 +1072,7 @@ suite("mv_tpch_test") { WHERE l_partkey = p_partkey AND l_shipdate >= DATE '1995-09-01' - AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH + AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH """ order_qt_query14_before "${query14}" async_mv_rewrite_success(db, mv14, query14, "mv14") @@ -994,7 +1118,7 @@ suite("mv_tpch_test") { revenue1 ) ORDER BY - s_suppkey; + s_suppkey; """ // revenue1 in materialized view is view, can not create materialized view support now // order_qt_query15_before "${query15}" @@ -1063,7 +1187,7 @@ suite("mv_tpch_test") { supplier_cnt DESC, p_brand, p_type, - p_size + p_size """ // contains subquery, doesn't support now order_qt_query16_before "${query16}" @@ -1104,7 +1228,7 @@ suite("mv_tpch_test") { lineitem WHERE l_partkey = p_partkey - ) + ) """ // contains subquery, doesn't support now order_qt_query17_before "${query17}" @@ -1146,7 +1270,7 @@ suite("mv_tpch_test") { ORDER BY o_totalprice DESC, o_orderdate - LIMIT 100 + LIMIT 100 """ def query18 = """ SELECT @@ -1181,7 +1305,7 @@ suite("mv_tpch_test") { ORDER BY o_totalprice DESC, o_orderdate - LIMIT 100 + LIMIT 100 """ // contains limit, doesn't support now order_qt_query18_before "${query18}" @@ -1190,6 +1314,81 @@ suite("mv_tpch_test") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv18""" + def mv18_1 = """ + SELECT + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) + FROM + customer, + orders, + lineitem + WHERE + o_orderkey IN ( + SELECT l_orderkey + FROM + lineitem + GROUP BY + l_orderkey + HAVING + sum(l_quantity) > 300 + ) + AND c_custkey = o_custkey + AND o_orderkey = l_orderkey + GROUP BY + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice + ORDER BY + o_totalprice DESC, + o_orderdate + """ + def query18_1 = """ + SELECT + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) + FROM + customer, + orders, + lineitem + WHERE + o_orderkey IN ( + SELECT l_orderkey + FROM + lineitem + GROUP BY + l_orderkey + HAVING + sum(l_quantity) > 300 + ) + AND c_custkey = o_custkey + AND o_orderkey = l_orderkey + GROUP BY + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice + ORDER BY + o_totalprice DESC, + o_orderdate + LIMIT 100 + """ + order_qt_query18_1_before "${query18_1}" + async_mv_rewrite_success(db, mv18_1, query18, "mv18_1") + order_qt_query18_1_after "${query18_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv18_1""" + + def mv19 = """ SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue FROM @@ -1260,11 +1459,10 @@ suite("mv_tpch_test") { AND p_size BETWEEN 1 AND 15 AND l_shipmode IN ('AIR', 'AIR REG') AND l_shipinstruct = 'DELIVER IN PERSON' - ) + ) """ - // join condition is not conjunctions, doesn't support now order_qt_query19_before "${query19}" - async_mv_rewrite_fail(db, mv19, query19, "mv19") + async_mv_rewrite_success(db, mv19, query19, "mv19") order_qt_query19_after "${query19}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv19""" @@ -1335,11 +1533,10 @@ suite("mv_tpch_test") { ) AND s_nationkey = n_nationkey AND n_name = 'CANADA' - ORDER BY s_name + ORDER BY s_name """ - // contains subquery, doesn't support now order_qt_query20_before "${query20}" - async_mv_rewrite_fail(db, mv20, query20, "mv20") + async_mv_rewrite_success(db, mv20, query20, "mv20") order_qt_query20_after "${query20}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv20""" @@ -1382,7 +1579,7 @@ suite("mv_tpch_test") { ORDER BY numwait DESC, s_name - LIMIT 100 + LIMIT 100 """ def query21 = """ SELECT @@ -1422,7 +1619,7 @@ suite("mv_tpch_test") { ORDER BY numwait DESC, s_name - LIMIT 100 + LIMIT 100 """ // contains limit, doesn't support now order_qt_query21_before "${query21}" @@ -1431,6 +1628,91 @@ suite("mv_tpch_test") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv21""" + def mv21_1 = """ + SELECT + s_name, + count(*) AS numwait + FROM + supplier, + lineitem l1, + orders, + nation + WHERE + s_suppkey = l1.l_suppkey + AND o_orderkey = l1.l_orderkey + AND o_orderstatus = 'F' + AND l1.l_receiptdate > l1.l_commitdate + AND exists( + SELECT * + FROM + lineitem l2 + WHERE + l2.l_orderkey = l1.l_orderkey + AND l2.l_suppkey <> l1.l_suppkey + ) + AND NOT exists( + SELECT * + FROM + lineitem l3 + WHERE + l3.l_orderkey = l1.l_orderkey + AND l3.l_suppkey <> l1.l_suppkey + AND l3.l_receiptdate > l3.l_commitdate + ) + AND s_nationkey = n_nationkey + AND n_name = 'SAUDI ARABIA' + GROUP BY + s_name + ORDER BY + numwait DESC, + s_name; + """ + def query21_1 = """ + SELECT + s_name, + count(*) AS numwait + FROM + supplier, + lineitem l1, + orders, + nation + WHERE + s_suppkey = l1.l_suppkey + AND o_orderkey = l1.l_orderkey + AND o_orderstatus = 'F' + AND l1.l_receiptdate > l1.l_commitdate + AND exists( + SELECT * + FROM + lineitem l2 + WHERE + l2.l_orderkey = l1.l_orderkey + AND l2.l_suppkey <> l1.l_suppkey + ) + AND NOT exists( + SELECT * + FROM + lineitem l3 + WHERE + l3.l_orderkey = l1.l_orderkey + AND l3.l_suppkey <> l1.l_suppkey + AND l3.l_receiptdate > l3.l_commitdate + ) + AND s_nationkey = n_nationkey + AND n_name = 'SAUDI ARABIA' + GROUP BY + s_name + ORDER BY + numwait DESC, + s_name + LIMIT 100 + """ + order_qt_query21_1_before "${query21_1}" + // complex join has not support yet, support in future + async_mv_rewrite_success(db, mv21_1, query21_1, "mv21_1") + order_qt_query21_1_after "${query21_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv21_1""" + def mv22 = """ SELECT cntrycode, @@ -1501,11 +1783,10 @@ suite("mv_tpch_test") { GROUP BY cntrycode ORDER BY - cntrycode + cntrycode """ - // contains subquery, doesn't support now order_qt_query22_before "${query22}" - async_mv_rewrite_fail(db, mv22, query22, "mv22") + async_mv_rewrite_success(db, mv22, query22, "mv22") order_qt_query22_after "${query22}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv22""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy index 0d0415456f726f..866f82af156c69 100644 --- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy @@ -415,8 +415,7 @@ suite("variant_mv") { where g2.actor['id'] > 34259289 and cast(g1.actor['id'] as int) + cast(g2.repo['id'] as int) > 80000000; """ order_qt_query3_0_before "${query3_0}" - // condition in join other conjuects is not supported now, suppport later -// async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + async_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") order_qt_query3_0_after "${query3_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" @@ -554,8 +553,7 @@ suite("variant_mv") { where g2.actor['id'] > 34259300 and cast(g1.actor['id'] as int) + cast(g2.repo['id'] as int) > 80000000; """ order_qt_query3_4_before "${query3_4}" - // condition in join other conjuects is not supported now, suppport later -// async_mv_rewrite_success(db, mv3_4, query3_4, "mv3_4") + async_mv_rewrite_success(db, mv3_4, query3_4, "mv3_4") order_qt_query3_4_after "${query3_4}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4""" diff --git a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_distinct_through_join.groovy b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_distinct_through_join.groovy index 933fe3400cfb49..3f959c91fdc04e 100644 --- a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_distinct_through_join.groovy +++ b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_distinct_through_join.groovy @@ -22,6 +22,7 @@ suite("push_down_top_n_distinct_through_join") { sql "SET enable_fallback_to_original_planner=false" sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "SET disable_join_reorder=true" + sql "set enable_nereids_distribute_planner=false;" sql """ DROP TABLE IF EXISTS table_join; @@ -66,4 +67,4 @@ suite("push_down_top_n_distinct_through_join") { qt_push_down_topn_through_join_data """ select distinct * from (select t1.id from table_join t1 cross join table_join t2) t order by id limit 10; """ -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy b/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy index cef1230a89e29c..950b6171c7ca84 100644 --- a/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy +++ b/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy @@ -45,6 +45,7 @@ suite("local_shuffle") { insert into test_local_shuffle1 values (1, 1), (2, 2); insert into test_local_shuffle2 values (2, 2), (3, 3); + set enable_nereids_distribute_planner=true; set enable_pipeline_x_engine=true; set disable_join_reorder=true; set enable_local_shuffle=true; diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy index a4b3eb661768dd..cbd58007e6cf15 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. suite("test_dynamic_partition") { + config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'force_olap_table_replication_allocation'; """ + String old_conf_value = config_row[0][1] + sql """ ADMIN SET FRONTEND CONFIG ("force_olap_table_replication_allocation" = ""); """ // todo: test dynamic partition sql "drop table if exists dy_par" sql """ @@ -158,4 +161,7 @@ suite("test_dynamic_partition") { } } sql "drop table if exists dy_par_bad" + + // restore force_olap_table_replication_allocation to old_value + sql """ ADMIN SET FRONTEND CONFIG ("force_olap_table_replication_allocation" = "${old_conf_value}"); """ } diff --git a/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy new file mode 100644 index 00000000000000..48281198cfd4f7 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.text.SimpleDateFormat; +import java.util.Date; + +suite("test_query_sys_scan_rowsets", "query,p0") { + def dbName1 = "test_query_sys_scan_rowsets" + + if (!isCloudMode()) { + log.info("not cloud mode") + return + } + + + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + + // test rowsets + qt_desc_rowsets """ desc information_schema.rowsets """ + def rowsets_table_name = """ test_query_sys_scan_rowsets.test_query_rowset """ + sql """ drop table if exists ${rowsets_table_name} """ + + sql """ + create table ${rowsets_table_name}( + a int , + b boolean , + c string ) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_single_replica_compaction"="true" + ); + """ + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + def now = sdf.format(new Date()).toString(); + + def rowsets_table_name_tablets = sql_return_maparray """ show tablets from ${rowsets_table_name}; """ + def tablet_id = rowsets_table_name_tablets[0].TabletId + + sql """ select * from ${rowsets_table_name}; """ + order_qt_rowsets1 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${tablet_id} group by START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """ + + sql """ insert into ${rowsets_table_name} values (1,0,"abc"); """ + sql """ select * from ${rowsets_table_name}; """ + order_qt_rowsets2 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${tablet_id} group by START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """ + + sql """ insert into ${rowsets_table_name} values (2,1,"hello world"); """ + sql """ insert into ${rowsets_table_name} values (3,0,"dssadasdsafafdf"); """ + sql """ select * from ${rowsets_table_name}; """ + order_qt_rowsets3 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${tablet_id} group by START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """ + + sql """ insert into ${rowsets_table_name} values (4,0,"abcd"); """ + sql """ select * from ${rowsets_table_name}; """ + order_qt_rowsets4 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${tablet_id} and NEWEST_WRITE_TIMESTAMP>='${now}' group by START_VERSION,END_VERSION order by START_VERSION,END_VERSION; """ +} \ No newline at end of file diff --git a/regression-test/suites/query_p0/system/test_storage_page_size.groovy b/regression-test/suites/query_p0/system/test_storage_page_size.groovy new file mode 100644 index 00000000000000..9d7b129226d38b --- /dev/null +++ b/regression-test/suites/query_p0/system/test_storage_page_size.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_storage_page_size") { + sql """ DROP TABLE IF EXISTS table_1; """ + sql """ + create table table_1 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1"); + """ + test { + sql "show create table table_1;" + check { result, exception, startTime, endTime -> + assertFalse(result[0][1].contains("\"storage_page_size\" = \"65536\"")) + } + } + + // min - 1 + sql """ DROP TABLE IF EXISTS table_2; """ + test { + sql """ + create table table_2 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "4095"); + """ + check { result, exception, startTime, endTime -> + if (exception) { + assert exception.message.contains("Storage page size must be between 4KB and 10MB.") + } + } + } + + // min + sql """ DROP TABLE IF EXISTS table_3; """ + sql """ + create table table_3 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "4096"); + """ + test { + sql "show create table table_3;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"storage_page_size\" = \"4096\"")) + } + } + + + // min + 1 + sql """ DROP TABLE IF EXISTS table_4; """ + sql """ + create table table_4 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "4097"); + """ + test { + sql "show create table table_4;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"storage_page_size\" = \"8192\"")) + } + } + + // 65537 + sql """ DROP TABLE IF EXISTS table_5; """ + sql """ + create table table_5 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "65537"); + """ + test { + sql "show create table table_5;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"storage_page_size\" = \"69632\"")) + } + } + test { + sql """ alter table table_5 set ("storage_page_size" = "65535"); """ + check { result, exception, startTime, endTime -> + if (exception) { + assert exception.message.contains("You can not modify storage_page_size") + } + } + } + + // max - 1 + sql """ DROP TABLE IF EXISTS table_6; """ + sql """ + create table table_6 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "10485759"); + """ + test { + sql "show create table table_6;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"storage_page_size\" = \"10485760\"")) + } + } + + // max + sql """ DROP TABLE IF EXISTS table_7; """ + sql """ + create table table_7 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "10485760"); + """ + test { + sql "show create table table_7;" + check { result, exception, startTime, endTime -> + assertTrue(result[0][1].contains("\"storage_page_size\" = \"10485760\"")) + } + } + + // max + 1 + sql """ DROP TABLE IF EXISTS table_8; """ + test { + sql """ + create table table_8 ( + k1 int not null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1) + distributed BY random buckets 1 + properties("replication_num" = "1", "storage_page_size" = "10485761"); + """ + check { result, exception, startTime, endTime -> + if (exception) { + assert exception.message.contains("Storage page size must be between 4KB and 10MB.") + } + } + } +} diff --git a/regression-test/suites/query_p0/union/test_union_instance.groovy b/regression-test/suites/query_p0/union/test_union_instance.groovy index 17a0d3ef1dd614..5fc5d0293852e3 100644 --- a/regression-test/suites/query_p0/union/test_union_instance.groovy +++ b/regression-test/suites/query_p0/union/test_union_instance.groovy @@ -63,6 +63,7 @@ suite("test_union_instance") { set parallel_pipeline_task_num=1; set disable_nereids_rules='PRUNE_EMPTY_PARTITION'; + set enable_nereids_distribute_planner=false; """ explain { sql """ diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy index d5dcb8393799f3..34a62e6da0c990 100644 --- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy +++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update.groovy @@ -257,7 +257,7 @@ suite("test_primary_key_partial_update", "p0") { qt_select_timestamp "select count(*) from ${tableName} where `ctime` > \"1970-01-01\"" - sql "set time_zone = 'America/New_York'" + sql "set time_zone = 'Asia/Tokyo'" Thread.sleep(5000) diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy index ea3dde5bf5b3f2..9643f9501ed48a 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update.groovy @@ -207,7 +207,7 @@ suite("test_primary_key_partial_update", "p0") { qt_select_timestamp "select count(*) from ${tableName} where `ctime` > \"1970-01-01\"" - sql "set time_zone = 'America/New_York'" + sql "set time_zone = 'Asia/Tokyo'" Thread.sleep(5000) diff --git a/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy b/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy new file mode 100644 index 00000000000000..cc9289f49e0c9a --- /dev/null +++ b/regression-test/suites/vault_p0/alter/test_alter_use_path_style.groovy @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_alter_use_path_style", "nonConcurrent") { + def suiteName = name; + if (!isCloudMode()) { + logger.info("skip ${suiteName} case, because not cloud mode") + return + } + + if (!enableStoragevault()) { + logger.info("skip ${suiteName} case, because storage vault not enabled") + return + } + + sql """ + CREATE STORAGE VAULT IF NOT EXISTS ${suiteName} + PROPERTIES ( + "type"="S3", + "s3.endpoint"="${getS3Endpoint()}", + "s3.region" = "${getS3Region()}", + "s3.access_key" = "${getS3AK()}", + "s3.secret_key" = "${getS3SK()}", + "s3.root.path" = "${suiteName}", + "s3.bucket" = "${getS3BucketName()}", + "s3.external_endpoint" = "", + "provider" = "${getS3Provider()}", + "use_path_style" = "false" + ); + """ + + sql """ + ALTER STORAGE VAULT ${suiteName} + PROPERTIES ( + "type"="S3", + "use_path_style" = "true" + ); + """ + + def vaultInfos = sql """ SHOW STORAGE VAULT; """ + boolean exist = false + + for (int i = 0; i < vaultInfos.size(); i++) { + def name = vaultInfos[i][0] + logger.info("name is ${name}, info ${vaultInfos[i]}") + if (name.equals(suiteName)) { + assertTrue(vaultInfos[i][2].contains("""use_path_style: true""")) + exist = true + } + } + assertTrue(exist) + + + sql """ + ALTER STORAGE VAULT ${suiteName} + PROPERTIES ( + "type"="S3", + "use_path_style" = "false" + ); + """ + + vaultInfos = sql """ SHOW STORAGE VAULT; """ + exist = false + + for (int i = 0; i < vaultInfos.size(); i++) { + def name = vaultInfos[i][0] + logger.info("name is ${name}, info ${vaultInfos[i]}") + if (name.equals(suiteName)) { + assertTrue(vaultInfos[i][2].contains("""use_path_style: false""")) + exist = true + } + } + assertTrue(exist) + + expectExceptionLike({ + sql """ + ALTER STORAGE VAULT ${suiteName} + PROPERTIES ( + "type"="S3", + "use_path_style" = "" + ); + """ + }, "use_path_style cannot be empty") + + expectExceptionLike({ + sql """ + ALTER STORAGE VAULT ${suiteName} + PROPERTIES ( + "type"="S3", + "use_path_style" = "abc" + ); + """ + }, "Invalid use_path_style value") +} \ No newline at end of file diff --git a/regression-test/suites/vault_p0/default/test_default_vault.groovy b/regression-test/suites/vault_p0/default/test_default_vault.groovy index 0ee871458b083d..2fa445bdd57a1d 100644 --- a/regression-test/suites/vault_p0/default/test_default_vault.groovy +++ b/regression-test/suites/vault_p0/default/test_default_vault.groovy @@ -92,9 +92,11 @@ suite("test_default_vault", "nonConcurrent") { """ sql """ insert into ${tableName} values(1, 1); """ - result """ select * from ${tableName}; """ - assertEqual(result.size(), 1) - assertEqual(result[0][0], 1) + sql """ sync;""" + def result = sql """ select * from ${tableName}; """ + logger.info("result:${result}"); + assertTrue(result.size() == 1) + assertTrue(result[0][0].toInteger() == 1) def create_table_stmt = sql """ show create table ${tableName} """ assertTrue(create_table_stmt[0][1].contains("built_in_storage_vault"))