From ec517a53a8f46ae9749b4d3b52dcf698f261b5d5 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 28 Apr 2023 14:14:51 +0800 Subject: [PATCH] [Chore](build) upgrade clang-format version to 16 && move thrift to fe-common (#19155) upgrade clang-format version to 16 move thrift to fe-common fix core dump on pipeline engine when operator canceled and not prepared --- .github/actions/action-sh-checker | 2 +- .github/actions/ccache-action | 2 +- .github/actions/clang-format-lint-action | 2 +- .github/actions/clang-tidy-review | 2 +- .github/actions/get-workflow-origin | 2 +- .github/actions/paths-filter | 2 +- .github/workflows/be-ut-clang.yml | 2 +- .github/workflows/build-thirdparty.yml | 2 +- .github/workflows/clang-format.yml | 2 +- .github/workflows/code-checks.yml | 2 +- .gitignore | 4 +- be/src/common/factory_creator.h | 46 ++++++++++++------- .../olap/rowset/segment_v2/segment_writer.cpp | 2 +- be/src/pipeline/exec/operator.h | 22 ++++++--- be/src/vec/common/hash_table/hash.h | 10 ++-- be/src/vec/functions/like.cpp | 2 +- bin/start_fe.sh | 2 +- build-for-release.sh | 3 +- build-support/check-format.sh | 2 +- build-support/clang-format.sh | 2 +- build.sh | 9 ++-- build_plugin.sh | 2 +- contrib/udf/build_udf.sh | 1 - dist/download_scripts/download_base.sh | 2 +- .../thirdparties/run-thirdparties-docker.sh | 3 -- .../install/source-install/compilation-win.md | 2 +- .../install/source-install/compilation-win.md | 2 +- env.sh | 10 ++-- fe/fe-common/pom.xml | 20 -------- .../doris/utframe/TestWithFeService.java | 7 +-- .../apache_hdfs_broker/bin/start_broker.sh | 2 +- gensrc/script/gen_build_version.sh | 2 +- run-be-ut.sh | 2 +- run-fe-ut.sh | 8 ++++ run-regression-test.sh | 3 +- samples/insert/shell/insert_utils.sh | 3 +- .../create-clickbench-table.sh | 2 +- .../clickbench-tools/load-clickbench-data.sh | 3 +- .../run-clickbench-queries.sh | 2 +- tools/restore_tablet_tool.sh | 1 - tools/ssb-tools/bin/create-ssb-tables.sh | 3 +- tools/ssb-tools/bin/gen-ssb-data.sh | 3 +- tools/ssb-tools/bin/load-ssb-data.sh | 3 +- tools/ssb-tools/bin/run-ssb-flat-queries.sh | 3 +- tools/ssb-tools/bin/run-ssb-queries.sh | 3 +- tools/tpcds-tools/bin/create-tpcds-tables.sh | 2 +- tools/tpcds-tools/bin/gen-tpcds-data.sh | 2 +- tools/tpcds-tools/bin/gen-tpcds-queries.sh | 2 +- tools/tpcds-tools/bin/load-tpcds-data.sh | 2 +- tools/tpcds-tools/bin/run-tpcds-queries.sh | 2 +- tools/tpch-tools/bin/create-tpch-tables.sh | 3 +- tools/tpch-tools/bin/gen-tpch-data.sh | 3 +- tools/tpch-tools/bin/load-tpch-data.sh | 3 +- tools/tpch-tools/bin/run-tpch-queries.sh | 3 +- 54 files changed, 115 insertions(+), 123 deletions(-) diff --git a/.github/actions/action-sh-checker b/.github/actions/action-sh-checker index edd0e45ecff35b..76ab0b22e1f194 160000 --- a/.github/actions/action-sh-checker +++ b/.github/actions/action-sh-checker @@ -1 +1 @@ -Subproject commit edd0e45ecff35b05f162052b50df50976c1b74fc +Subproject commit 76ab0b22e1f194e4a582edc7969df6485c4e9246 diff --git a/.github/actions/ccache-action b/.github/actions/ccache-action index 657bab9aeb32eb..ca3acd2731eef1 160000 --- a/.github/actions/ccache-action +++ b/.github/actions/ccache-action @@ -1 +1 @@ -Subproject commit 657bab9aeb32eb6d913c2e6882efad4bb7536d94 +Subproject commit ca3acd2731eef11f1572ccb126356c2f9298d35e diff --git a/.github/actions/clang-format-lint-action b/.github/actions/clang-format-lint-action index c3b2c943e92402..6adbe14579e5b8 160000 --- a/.github/actions/clang-format-lint-action +++ b/.github/actions/clang-format-lint-action @@ -1 +1 @@ -Subproject commit c3b2c943e924028b93a707a5b1b017976ab8d50c +Subproject commit 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7 diff --git a/.github/actions/clang-tidy-review b/.github/actions/clang-tidy-review index 79d70e7c89de1b..2c55ef8cfc9acb 160000 --- a/.github/actions/clang-tidy-review +++ b/.github/actions/clang-tidy-review @@ -1 +1 @@ -Subproject commit 79d70e7c89de1bda6ad68610cde18d2783749852 +Subproject commit 2c55ef8cfc9acb3715d433e58aea086dcec9b206 diff --git a/.github/actions/get-workflow-origin b/.github/actions/get-workflow-origin index 70849a55e9a8df..e2dae063368361 160000 --- a/.github/actions/get-workflow-origin +++ b/.github/actions/get-workflow-origin @@ -1 +1 @@ -Subproject commit 70849a55e9a8df457b37092ecd6c30eefc71286f +Subproject commit e2dae063368361e4cd1f510e8785cd73bca9352e diff --git a/.github/actions/paths-filter b/.github/actions/paths-filter index 4512585405083f..4067d885736b84 160000 --- a/.github/actions/paths-filter +++ b/.github/actions/paths-filter @@ -1 +1 @@ -Subproject commit 4512585405083f25c027a35db413c2b3b9006d50 +Subproject commit 4067d885736b84de7c414f582ac45897079b0a78 diff --git a/.github/workflows/be-ut-clang.yml b/.github/workflows/be-ut-clang.yml index 3b6aab7b1a9242..4688d5e0240183 100644 --- a/.github/workflows/be-ut-clang.yml +++ b/.github/workflows/be-ut-clang.yml @@ -62,7 +62,7 @@ jobs: export DEFAULT_DIR='/opt/doris' mkdir "${DEFAULT_DIR}" - wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.14.2/ldb_toolchain_gen.sh \ + wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.17/ldb_toolchain_gen.sh \ -q -O /tmp/ldb_toolchain_gen.sh bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 329d15125e63d0..d81c2f0932e6d9 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -95,7 +95,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt install --yes "${packages[@]}" mkdir -p "${DEFAULT_DIR}" - wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.14.2/ldb_toolchain_gen.sh \ + wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.17/ldb_toolchain_gen.sh \ -q -O /tmp/ldb_toolchain_gen.sh bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index a2166961fc17a0..d676f171ca8b86 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -47,7 +47,7 @@ jobs: id: be_clang_format with: source: "be/src be/test" - clangFormatVersion: 15 + clangFormatVersion: 16 inplace: False - name: Ignore it! diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 3ce04e53fd3968..ae837649bef1ee 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -78,7 +78,7 @@ jobs: export DEFAULT_DIR='/opt/doris' mkdir "${DEFAULT_DIR}" - wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.14.2/ldb_toolchain_gen.sh \ + wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.17/ldb_toolchain_gen.sh \ -q -O /tmp/ldb_toolchain_gen.sh bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" diff --git a/.gitignore b/.gitignore index 52134571a5384b..5c9626e2ec37e0 100644 --- a/.gitignore +++ b/.gitignore @@ -74,8 +74,8 @@ fe/fe-core/gen/ fe/fe-core/src/main/resources/static/ fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.tokens fe/fe-core/src/main/antlr4/org/apache/doris/nereids/gen/ -fe/fe-core/src/main/java/org/apache/doris/thrift -fe/fe-core/src/main/java/org/apache/parquet +fe/fe-common/src/main/java/org/apache/doris/thrift +fe/fe-common/src/main/java/org/apache/parquet # BE be/build*/ diff --git a/be/src/common/factory_creator.h b/be/src/common/factory_creator.h index f3ece3268574a4..0dea4d16b80807 100644 --- a/be/src/common/factory_creator.h +++ b/be/src/common/factory_creator.h @@ -35,21 +35,33 @@ // during inherits // TODO try to allow make_unique // -#define ENABLE_FACTORY_CREATOR(TypeName) \ -private: \ - void* operator new(std::size_t size) { return ::operator new(size); } \ - void* operator new[](std::size_t size) { return ::operator new[](size); } \ - \ -public: \ - void* operator new(std::size_t count, void* ptr) { return ::operator new(count, ptr); } \ - void operator delete(void* ptr) noexcept { ::operator delete(ptr); } \ - void operator delete[](void* ptr) noexcept { ::operator delete[](ptr); } \ - void operator delete(void* ptr, void* place) noexcept { ::operator delete(ptr, place); } \ - template \ - static std::shared_ptr create_shared(Args&&... args) { \ - return std::make_shared(std::forward(args)...); \ - } \ - template \ - static std::unique_ptr create_unique(Args&&... args) { \ - return std::unique_ptr(new TypeName(std::forward(args)...)); \ +#define ENABLE_FACTORY_CREATOR(TypeName) \ +private: \ + void* operator new(std::size_t size) { \ + return ::operator new(size); \ + } \ + void* operator new[](std::size_t size) { \ + return ::operator new[](size); \ + } \ + \ +public: \ + void* operator new(std::size_t count, void* ptr) { \ + return ::operator new(count, ptr); \ + } \ + void operator delete(void* ptr) noexcept { \ + ::operator delete(ptr); \ + } \ + void operator delete[](void* ptr) noexcept { \ + ::operator delete[](ptr); \ + } \ + void operator delete(void* ptr, void* place) noexcept { \ + ::operator delete(ptr, place); \ + } \ + template \ + static std::shared_ptr create_shared(Args&&... args) { \ + return std::make_shared(std::forward(args)...); \ + } \ + template \ + static std::unique_ptr create_unique(Args&&... args) { \ + return std::unique_ptr(new TypeName(std::forward(args)...)); \ } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 9f772e63e3aec0..6b3a2a1039cc1e 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -134,7 +134,7 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key, _column_writers.reserve(_tablet_schema->columns().size()); _column_ids.insert(_column_ids.end(), col_ids.begin(), col_ids.end()); _olap_data_convertor = std::make_unique(); - auto create_column_writer = [&](uint32_t cid, const auto& column) -> auto{ + auto create_column_writer = [&](uint32_t cid, const auto& column) -> auto { ColumnWriterOptions opts; opts.meta = _footer.add_columns(); diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h index e48be7c49a6b3c..5e80eb988b359f 100644 --- a/be/src/pipeline/exec/operator.h +++ b/be/src/pipeline/exec/operator.h @@ -42,13 +42,15 @@ class RuntimeState; class TDataSink; } // namespace doris -#define OPERATOR_CODE_GENERATOR(NAME, SUBCLASS) \ - NAME##Builder::NAME##Builder(int32_t id, ExecNode* exec_node) \ - : OperatorBuilder(id, #NAME, exec_node) {} \ - \ - OperatorPtr NAME##Builder::build_operator() { return std::make_shared(this, _node); } \ - \ - NAME::NAME(OperatorBuilderBase* operator_builder, ExecNode* node) \ +#define OPERATOR_CODE_GENERATOR(NAME, SUBCLASS) \ + NAME##Builder::NAME##Builder(int32_t id, ExecNode* exec_node) \ + : OperatorBuilder(id, #NAME, exec_node) {} \ + \ + OperatorPtr NAME##Builder::build_operator() { \ + return std::make_shared(this, _node); \ + } \ + \ + NAME::NAME(OperatorBuilderBase* operator_builder, ExecNode* node) \ : SUBCLASS(operator_builder, node) {}; namespace doris::pipeline { @@ -309,6 +311,9 @@ class DataSinkOperator : public OperatorBase { protected: void _fresh_exec_timer(NodeType* node) { + if (_runtime_profile == nullptr) { + return; + } node->profile()->total_time_counter()->update( _runtime_profile->total_time_counter()->value()); } @@ -387,6 +392,9 @@ class StreamingOperator : public OperatorBase { protected: void _fresh_exec_timer(NodeType* node) { + if (_runtime_profile == nullptr) { + return; + } node->runtime_profile()->total_time_counter()->update( _runtime_profile->total_time_counter()->value()); } diff --git a/be/src/vec/common/hash_table/hash.h b/be/src/vec/common/hash_table/hash.h index d2a50ad36dd897..be1841a47a0a40 100644 --- a/be/src/vec/common/hash_table/hash.h +++ b/be/src/vec/common/hash_table/hash.h @@ -122,10 +122,12 @@ inline size_t hash_crc32(doris::vectorized::Int128 u) { doris::vectorized::UInt128((u >> 64) & int64_t(-1), u & int64_t(-1))); } -#define DEFINE_HASH(T) \ - template <> \ - struct HashCRC32 { \ - size_t operator()(T key) const { return hash_crc32(key); } \ +#define DEFINE_HASH(T) \ + template <> \ + struct HashCRC32 { \ + size_t operator()(T key) const { \ + return hash_crc32(key); \ + } \ }; DEFINE_HASH(doris::vectorized::UInt8) diff --git a/be/src/vec/functions/like.cpp b/be/src/vec/functions/like.cpp index d75bea36c50722..0f5b7c66999888 100644 --- a/be/src/vec/functions/like.cpp +++ b/be/src/vec/functions/like.cpp @@ -451,7 +451,7 @@ Status FunctionLikeBase::execute_impl(FunctionContext* context, Block& block, context->get_function_state(FunctionContext::THREAD_LOCAL)); // for constant_substring_fn, use long run length search for performance if (constant_substring_fn == - *(state->function.targetfunction.target())) { RETURN_IF_ERROR(execute_substring(values->get_chars(), values->get_offsets(), vec_res, &state->search_state)); diff --git a/bin/start_fe.sh b/bin/start_fe.sh index f8af11cf90560f..9d21651fc0fa4c 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -108,7 +108,7 @@ if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then fi if [[ -z "${JAVA_HOME}" ]]; then - JAVA="$(which java)" + JAVA="$(command -v java)" else JAVA="${JAVA_HOME}/bin/java" fi diff --git a/build-for-release.sh b/build-for-release.sh index 869f9a6c9e12dc..a976035443931d 100755 --- a/build-for-release.sh +++ b/build-for-release.sh @@ -58,6 +58,7 @@ eval set -- "${OPTS}" _USE_AVX2=1 TAR=0 +HELP=0 VERSION= if [[ "$#" == 1 ]]; then _USE_AVX2=1 @@ -94,13 +95,11 @@ fi if [[ "${HELP}" -eq 1 ]]; then usage - exit fi if [[ -z ${VERSION} ]]; then echo "Must specify version" usage - exit 1 fi echo "Get params: diff --git a/build-support/check-format.sh b/build-support/check-format.sh index 4941673c07e70b..805628dcb6e29d 100755 --- a/build-support/check-format.sh +++ b/build-support/check-format.sh @@ -31,6 +31,6 @@ DORIS_HOME=$( ) export DORIS_HOME -CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}" +CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(command -v clang-format)}" python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "false" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" diff --git a/build-support/clang-format.sh b/build-support/clang-format.sh index 9dfeb251675f47..788efb4afc3255 100755 --- a/build-support/clang-format.sh +++ b/build-support/clang-format.sh @@ -31,6 +31,6 @@ DORIS_HOME=$( ) export DORIS_HOME -CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}" +CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(command -v clang-format)}" python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" diff --git a/build.sh b/build.sh index a7b9080abae02a..1a3befcabcd4da 100755 --- a/build.sh +++ b/build.sh @@ -232,7 +232,6 @@ fi if [[ "${HELP}" -eq 1 ]]; then usage - exit fi # build thirdparty libraries if necessary if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a" ]]; then @@ -391,9 +390,9 @@ echo "Build generated code" cd "${DORIS_HOME}/gensrc" # DO NOT using parallel make(-j) for gensrc make -rm -rf "${DORIS_HOME}/fe/fe-core/src/main/java/org/apache/doris/thrift ${DORIS_HOME}/fe/fe-core/src/main/java/org/apache/parquet" -cp -r "build/gen_java/org/apache/doris/thrift" "${DORIS_HOME}/fe/fe-core/src/main/java/org/apache/doris" -cp -r "build/gen_java/org/apache/parquet" "${DORIS_HOME}/fe/fe-core/src/main/java/org/apache/" +rm -rf "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/doris/thrift ${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/parquet" +cp -r "build/gen_java/org/apache/doris/thrift" "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/doris" +cp -r "build/gen_java/org/apache/parquet" "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/" # Assesmble FE modules FE_MODULES='' @@ -432,7 +431,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then if [[ "${CLEAN}" -eq 1 ]]; then clean_be fi - MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")" + MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")" echo "-- Make program: ${MAKE_PROGRAM}" echo "-- Use ccache: ${CMAKE_USE_CCACHE}" echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}" diff --git a/build_plugin.sh b/build_plugin.sh index 6b42862cd72321..c0c95bad396979 100755 --- a/build_plugin.sh +++ b/build_plugin.sh @@ -52,6 +52,7 @@ eval set -- "${OPTS}" ALL_PLUGIN=1 CLEAN=0 +HELP=0 if [[ "$#" == 1 ]]; then # defuat ALL_PLUGIN=1 @@ -89,7 +90,6 @@ fi if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Get params: diff --git a/contrib/udf/build_udf.sh b/contrib/udf/build_udf.sh index bd793ee29eed3e..af260522b52fd6 100644 --- a/contrib/udf/build_udf.sh +++ b/contrib/udf/build_udf.sh @@ -102,7 +102,6 @@ fi if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Get params: diff --git a/dist/download_scripts/download_base.sh b/dist/download_scripts/download_base.sh index 72a4017639eaf0..c5df7aa4a9ba1a 100755 --- a/dist/download_scripts/download_base.sh +++ b/dist/download_scripts/download_base.sh @@ -39,7 +39,7 @@ DOWNLOAD_DIR="$5" FILE_SUFFIX=".tar.xz" # Check if curl cmd exists -if [[ -n $(which curl >/dev/null 2>&1) ]]; then +if [[ -n $(command -v curl >/dev/null 2>&1) ]]; then echo "curl command not found on the system" exit 1 fi diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index ea6f903449fea6..1851b03051f67a 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -99,13 +99,11 @@ fi if [[ "${HELP}" -eq 1 ]]; then usage - exit 0 fi if [[ "${COMPONENTS}"x == ""x ]]; then echo "Invalid arguments" usage - exit 1 fi if [[ "${CONTAINER_UID}"x == "doris--"x ]]; then @@ -150,7 +148,6 @@ for element in "${COMPONENTS_ARR[@]}"; do else echo "Invalid component: ${element}" usage - exit 1 fi done diff --git a/docs/en/docs/install/source-install/compilation-win.md b/docs/en/docs/install/source-install/compilation-win.md index 86c50fe9e1cb46..9d217576b60f30 100644 --- a/docs/en/docs/install/source-install/compilation-win.md +++ b/docs/en/docs/install/source-install/compilation-win.md @@ -73,7 +73,7 @@ This topic is about how to compile Doris from source with Windows. - [Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz) - [Apache Maven 3.6.3](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz) - [Node v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz) - - [LDB_TOOLCHAIN](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.14.2/ldb_toolchain_gen.sh) + - [LDB_TOOLCHAIN](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.17/ldb_toolchain_gen.sh) 5. Configure environment variables diff --git a/docs/zh-CN/docs/install/source-install/compilation-win.md b/docs/zh-CN/docs/install/source-install/compilation-win.md index a2bfbddbcc1999..e03661784f35ca 100644 --- a/docs/zh-CN/docs/install/source-install/compilation-win.md +++ b/docs/zh-CN/docs/install/source-install/compilation-win.md @@ -73,7 +73,7 @@ under the License. - [Java8](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u131-linux-x64.tar.gz) - [Apache Maven 3.6.3](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/apache-maven-3.6.3-bin.tar.gz) - [Node v12.13.0](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v12.13.0-linux-x64.tar.gz) - - [LDB_TOOLCHAIN](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.14.2/ldb_toolchain_gen.sh) + - [LDB_TOOLCHAIN](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.17/ldb_toolchain_gen.sh) 5. 配置环境变量 diff --git a/env.sh b/env.sh index 4d5f063f15d4fc..75a8203e40d16f 100755 --- a/env.sh +++ b/env.sh @@ -117,7 +117,7 @@ fi if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then # set GCC HOME if [[ -z "${DORIS_GCC_HOME}" ]]; then - DORIS_GCC_HOME="$(dirname "$(which gcc)")"/.. + DORIS_GCC_HOME="$(dirname "$(command -v gcc)")"/.. export DORIS_GCC_HOME fi @@ -135,7 +135,7 @@ if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then # set CLANG HOME if [[ -z "${DORIS_CLANG_HOME}" ]]; then - DORIS_CLANG_HOME="$(dirname "$(which clang)")"/.. + DORIS_CLANG_HOME="$(dirname "$(command -v clang)")"/.. export DORIS_CLANG_HOME fi @@ -163,7 +163,7 @@ if [[ -z "${DORIS_BIN_UTILS}" ]]; then fi if [[ -z "${DORIS_GCC_HOME}" ]]; then - DORIS_GCC_HOME="$(dirname "$(which gcc)")/.." + DORIS_GCC_HOME="$(dirname "$(command -v gcc)")/.." export DORIS_GCC_HOME fi @@ -191,8 +191,8 @@ if test -z "${BUILD_THIRDPARTY_WIP:-}"; then # check java home if [[ -z "${JAVA_HOME}" ]]; then - JAVA="$(which java)" - JAVAP="$(which javap)" + JAVA="$(command -v java)" + JAVAP="$(command -v javap)" else JAVA="${JAVA_HOME}/bin/java" JAVAP="${JAVA_HOME}/bin/javap" diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index 6385d2e0d3369b..dde56af811faf5 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -99,26 +99,6 @@ under the License. doris-fe-common - - - org.apache.thrift.tools - maven-thrift-plugin - 0.1.11 - - ${doris.thirdparty}/installed/bin/thrift - ${doris.home}/gensrc/thrift - java:fullcamel - - - - thrift-sources - generate-sources - - compile - - - - org.apache.maven.plugins maven-source-plugin diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index c3f5be46cf4f3b..d7713b4347b278 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -417,7 +417,6 @@ protected void cleanDorisFeDir() { try { cleanDir(dorisHome + "/" + runningDir); cleanDir(Config.plugin_dir); - cleanDir(Config.custom_config_dir); } catch (IOException e) { e.printStackTrace(); } @@ -472,7 +471,8 @@ public Planner getSQLPlanner(String queryStr) throws Exception { if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) { return stmtExecutor.planner(); } else { - return null; + throw new Exception( + connectContext.getState().toString() + ", " + connectContext.getState().getErrorMessage()); } } @@ -599,7 +599,8 @@ protected void dropSqlBlockRule(String sql) throws Exception { protected void assertSQLPlanOrErrorMsgContains(String sql, String expect) throws Exception { // Note: adding `EXPLAIN` is necessary for non-query SQL, e.g., DDL, DML, etc. // TODO: Use a graceful way to get explain plan string, rather than modifying the SQL string. - Assertions.assertTrue(getSQLPlanOrErrorMsg("EXPLAIN " + sql).contains(expect)); + Assertions.assertTrue(getSQLPlanOrErrorMsg("EXPLAIN " + sql).contains(expect), + getSQLPlanOrErrorMsg("EXPLAIN " + sql)); } protected void assertSQLPlanOrErrorMsgContains(String sql, String... expects) throws Exception { diff --git a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh index 3e6108490f4b23..b6511332a6163c 100755 --- a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh +++ b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh @@ -67,7 +67,7 @@ export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" export BROKER_LOG_DIR="${BROKER_HOME}/log" # java if [[ -z "${JAVA_HOME}" ]]; then - JAVA="$(which java)" + JAVA="$(command -v java)" else JAVA="${JAVA_HOME}/bin/java" fi diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index c2fc94fcf73b93..53e7dcd5b9d30f 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -76,7 +76,7 @@ build_time="${date}" build_info="${hostname}" if [[ -z "${JAVA_HOME}" ]]; then - java_cmd="$(which java)" + java_cmd="$(command -v java)" else java_cmd="${JAVA_HOME}/bin/java" fi diff --git a/run-be-ut.sh b/run-be-ut.sh index ec3a98e686a779..a80aa2e4701845 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -178,7 +178,7 @@ if [[ -z "${USE_DWARF}" ]]; then USE_DWARF='OFF' fi -MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")" +MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")" echo "-- Make program: ${MAKE_PROGRAM}" echo "-- Use ccache: ${CMAKE_USE_CCACHE}" echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}" diff --git a/run-fe-ut.sh b/run-fe-ut.sh index ca1dafacace856..bf13f6d1c37748 100755 --- a/run-fe-ut.sh +++ b/run-fe-ut.sh @@ -96,6 +96,14 @@ cd "${DORIS_HOME}/docs" cp build/help-resource.zip "${DORIS_HOME}"/fe/fe-core/src/test/resources/real-help-resource.zip cd "${DORIS_HOME}" +echo "Build generated code" +cd "${DORIS_HOME}/gensrc" +make +rm -rf "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/doris/thrift ${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/parquet" +cp -r "build/gen_java/org/apache/doris/thrift" "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/doris" +cp -r "build/gen_java/org/apache/parquet" "${DORIS_HOME}/fe/fe-common/src/main/java/org/apache/" +cd "${DORIS_HOME}" + cd "${DORIS_HOME}/fe" mkdir -p build/compile diff --git a/run-regression-test.sh b/run-regression-test.sh index a8ea5f976cd35c..5b6d0f396bb630 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -109,9 +109,8 @@ else done fi -if [[ "${WRONG_CMD}" -eq 1 ]]; then +if [[ ${WRONG_CMD} -eq 1 ]]; then usage - exit 1 fi # set maven diff --git a/samples/insert/shell/insert_utils.sh b/samples/insert/shell/insert_utils.sh index 0ff3a5b4421651..f90b080d9c8fd5 100644 --- a/samples/insert/shell/insert_utils.sh +++ b/samples/insert/shell/insert_utils.sh @@ -34,7 +34,7 @@ function check_insert_load_doris_func() { fi wait_seconds=3600 - while [[ "${wait_seconds}" -gt 0 ]]; do + while [[ ${wait_seconds} -gt 0 ]]; do echo "${wait_seconds}" echo "${doris} -e show load where label = '${label}' order by createtime desc limit 1" result=$(${doris} -e "show load where label = '${label}' order by createtime desc limit 1" -N) @@ -47,7 +47,6 @@ function check_insert_load_doris_func() { else return 1 fi - break else echo "insert status: ${load_status}" sleep 5s diff --git a/tools/clickbench-tools/create-clickbench-table.sh b/tools/clickbench-tools/create-clickbench-table.sh index d95e134e6fea46..76ce56e8d065bc 100755 --- a/tools/clickbench-tools/create-clickbench-table.sh +++ b/tools/clickbench-tools/create-clickbench-table.sh @@ -68,7 +68,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi diff --git a/tools/clickbench-tools/load-clickbench-data.sh b/tools/clickbench-tools/load-clickbench-data.sh index 1b23457f7a8fee..aff122edff4689 100755 --- a/tools/clickbench-tools/load-clickbench-data.sh +++ b/tools/clickbench-tools/load-clickbench-data.sh @@ -66,9 +66,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() { diff --git a/tools/clickbench-tools/run-clickbench-queries.sh b/tools/clickbench-tools/run-clickbench-queries.sh index 45011ab20da9c7..e2e636a39e9158 100755 --- a/tools/clickbench-tools/run-clickbench-queries.sh +++ b/tools/clickbench-tools/run-clickbench-queries.sh @@ -70,7 +70,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi diff --git a/tools/restore_tablet_tool.sh b/tools/restore_tablet_tool.sh index f98d90d97a2ce7..bb6c9bb8a674dc 100644 --- a/tools/restore_tablet_tool.sh +++ b/tools/restore_tablet_tool.sh @@ -86,7 +86,6 @@ while true; do ;; -h | --help) usage - shift ;; --) shift diff --git a/tools/ssb-tools/bin/create-ssb-tables.sh b/tools/ssb-tools/bin/create-ssb-tables.sh index f8860f35a73545..eee4d59cc7a4a1 100755 --- a/tools/ssb-tools/bin/create-ssb-tables.sh +++ b/tools/ssb-tools/bin/create-ssb-tables.sh @@ -71,9 +71,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() { diff --git a/tools/ssb-tools/bin/gen-ssb-data.sh b/tools/ssb-tools/bin/gen-ssb-data.sh index 76c85f2b327d36..2285d6491f4ebc 100755 --- a/tools/ssb-tools/bin/gen-ssb-data.sh +++ b/tools/ssb-tools/bin/gen-ssb-data.sh @@ -88,9 +88,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Scale Factor: ${SCALE_FACTOR}" diff --git a/tools/ssb-tools/bin/load-ssb-data.sh b/tools/ssb-tools/bin/load-ssb-data.sh index db958b3921a2bb..efd3bf3109f86e 100755 --- a/tools/ssb-tools/bin/load-ssb-data.sh +++ b/tools/ssb-tools/bin/load-ssb-data.sh @@ -81,9 +81,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Parallelism: ${PARALLEL}" diff --git a/tools/ssb-tools/bin/run-ssb-flat-queries.sh b/tools/ssb-tools/bin/run-ssb-flat-queries.sh index a46887381cb254..b94693be530780 100755 --- a/tools/ssb-tools/bin/run-ssb-flat-queries.sh +++ b/tools/ssb-tools/bin/run-ssb-flat-queries.sh @@ -70,9 +70,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() { diff --git a/tools/ssb-tools/bin/run-ssb-queries.sh b/tools/ssb-tools/bin/run-ssb-queries.sh index 24f811e6ac1dff..ba3b0e04eb0525 100755 --- a/tools/ssb-tools/bin/run-ssb-queries.sh +++ b/tools/ssb-tools/bin/run-ssb-queries.sh @@ -70,9 +70,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() { diff --git a/tools/tpcds-tools/bin/create-tpcds-tables.sh b/tools/tpcds-tools/bin/create-tpcds-tables.sh index ede65b039bf42e..a7808023d79955 100755 --- a/tools/tpcds-tools/bin/create-tpcds-tables.sh +++ b/tools/tpcds-tools/bin/create-tpcds-tables.sh @@ -68,7 +68,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage fi diff --git a/tools/tpcds-tools/bin/gen-tpcds-data.sh b/tools/tpcds-tools/bin/gen-tpcds-data.sh index d18bb702f04384..ebd5a3f694ddda 100755 --- a/tools/tpcds-tools/bin/gen-tpcds-data.sh +++ b/tools/tpcds-tools/bin/gen-tpcds-data.sh @@ -87,7 +87,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage fi diff --git a/tools/tpcds-tools/bin/gen-tpcds-queries.sh b/tools/tpcds-tools/bin/gen-tpcds-queries.sh index 61b83563bbd100..eb4f292606d4c2 100755 --- a/tools/tpcds-tools/bin/gen-tpcds-queries.sh +++ b/tools/tpcds-tools/bin/gen-tpcds-queries.sh @@ -79,7 +79,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage fi diff --git a/tools/tpcds-tools/bin/load-tpcds-data.sh b/tools/tpcds-tools/bin/load-tpcds-data.sh index 117bc838212e9a..a5fe926f652ec6 100755 --- a/tools/tpcds-tools/bin/load-tpcds-data.sh +++ b/tools/tpcds-tools/bin/load-tpcds-data.sh @@ -80,7 +80,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage fi diff --git a/tools/tpcds-tools/bin/run-tpcds-queries.sh b/tools/tpcds-tools/bin/run-tpcds-queries.sh index 87ea92f1e00911..8f01192602434a 100755 --- a/tools/tpcds-tools/bin/run-tpcds-queries.sh +++ b/tools/tpcds-tools/bin/run-tpcds-queries.sh @@ -69,7 +69,7 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage fi diff --git a/tools/tpch-tools/bin/create-tpch-tables.sh b/tools/tpch-tools/bin/create-tpch-tables.sh index bd0b75e570a569..385f936a4dcca0 100755 --- a/tools/tpch-tools/bin/create-tpch-tables.sh +++ b/tools/tpch-tools/bin/create-tpch-tables.sh @@ -68,9 +68,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() { diff --git a/tools/tpch-tools/bin/gen-tpch-data.sh b/tools/tpch-tools/bin/gen-tpch-data.sh index bcac551e883f1f..fa436d361cd4ee 100755 --- a/tools/tpch-tools/bin/gen-tpch-data.sh +++ b/tools/tpch-tools/bin/gen-tpch-data.sh @@ -88,9 +88,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Scale Factor: ${SCALE_FACTOR}" diff --git a/tools/tpch-tools/bin/load-tpch-data.sh b/tools/tpch-tools/bin/load-tpch-data.sh index dd59284f3edbd4..33865ca073cfe6 100755 --- a/tools/tpch-tools/bin/load-tpch-data.sh +++ b/tools/tpch-tools/bin/load-tpch-data.sh @@ -81,9 +81,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi echo "Parallelism: ${PARALLEL}" diff --git a/tools/tpch-tools/bin/run-tpch-queries.sh b/tools/tpch-tools/bin/run-tpch-queries.sh index a198f76703be1c..d437ea4ae667cd 100755 --- a/tools/tpch-tools/bin/run-tpch-queries.sh +++ b/tools/tpch-tools/bin/run-tpch-queries.sh @@ -69,9 +69,8 @@ while true; do esac done -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage - exit fi check_prerequest() {