Merge branch 'master' into patch-2

.github/workflows/auto_trigger_teamcity.yml

@@ -57,40 +57,32 @@ jobs:
           echo "latest_commit_id : ${{ env.LATEST_COMMIT }}"
           
           set -x
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "buildall" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression ${trigger_pipelines}"
+          reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
+          comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)"
+          comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)"
+          if [[ "${comment_trigger_type}" == "buildall" ]]; then 
+            trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression"
+          elif [[ "${comment_trigger_type}" == "compile" ]]; then 
+            trigger_pipelines="Doris_DorisCompile_Compile"
+          elif [[ "${comment_trigger_type}" == "p0" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P0Regression"
+          elif [[ "${comment_trigger_type}" == "p1" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P1Regression"
+          elif [[ "${comment_trigger_type}" == "feut" ]]; then 
+            trigger_pipelines="Doris_Doris_FeUt"
+          elif [[ "${comment_trigger_type}" == "beut" ]]; then 
+            trigger_pipelines="Doris_DorisBeUt_BeUt"
+          elif [[ "${comment_trigger_type}" == "external" ]]; then 
+            trigger_pipelines="Doris_External_Regression"
+          elif [[ "${comment_trigger_type}" == "clickbench" ]]; then 
+            trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew"
+          elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX"
+          elif [[ "${comment_trigger_type}" == "arm" ]]; then 
+            trigger_pipelines="Doris_ArmPipeline_P0Regression"
+          elif [[ "${comment_trigger_type}" == "tpch" ]]; then
+            trigger_pipelines="Tpch_TpchSf100"
           fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P0Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " pipelinex_p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "p1" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P1Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "feut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "beut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisBeUt_BeUt  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "compile" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisCompile_Compile  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "clickbench" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "arm" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_ArmPipeline_P0Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "external" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_External_Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "just_for_test" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_ExternalRegression  ${trigger_pipelines}"
-          fi
-          
           if [ -z "${trigger_pipelines}" ];then
             echo "Just a general comment that doesn't match any pipeline rules"
           fi
@@ -133,7 +125,12 @@ jobs:
             if [ "_""${same_build_sign}" == "_false" ];then
               sleep 10s
               echo "there is no running build or queue build, so trigger a new !"
-              execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              echo "comment_repeat_times: ${comment_repeat_times}"
+              if [[ -n "${comment_repeat_times}" ]]; then
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}"
+              else
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              fi
               echo "${execute_command}"
               eval "${execute_command}"
               echo "-----------------------------------------------------------------"

.github/workflows/clang-format.yml

@@ -31,14 +31,21 @@ jobs:
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths filter
         uses: ./.github/actions/paths-filter
@@ -49,6 +56,15 @@ jobs:
               - 'be/src/**'
               - 'be/test/**'
 
+      - name: Checkout clang-format-lint-action
+        run: |
+          rm -rf ./.github/actions/clang-format-lint-action
+          git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
+
+          pushd .github/actions/clang-format-lint-action &>/dev/null
+          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          popd &>/dev/null
+
       - name: "Format it!"
         if: ${{ steps.filter.outputs.be_changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action

.github/workflows/code-checks.yml

@@ -27,21 +27,22 @@ jobs:
       - name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
-      - name: Patch
+      - name: Checkout action-sh-checker
         run: |
-          pushd .github/actions/action-sh-checker >/dev/null
+          rm -rf ./.github/actions/action-sh-checker
+          git clone https://github.com/luizm/action-sh-checker .github/actions/action-sh-checker
+
+          pushd .github/actions/action-sh-checker &>/dev/null
+          git checkout 76ab0b22e1f194e4a582edc7969df6485c4e9246
           sed -i 's/\[ "$GITHUB_EVENT_NAME" == "pull_request" \]/\[\[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "pull_request_target" \]\]/' entrypoint.sh
-          popd >/dev/null
+          popd &>/dev/null
 
       - name: Run ShellCheck
         uses: ./.github/actions/action-sh-checker
@@ -63,7 +64,15 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths Filter
         uses: ./.github/actions/paths-filter
@@ -117,14 +126,22 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Download
         uses: actions/download-artifact@v3
         with:
           name: compile_commands
           path: ./be/build_Release
 
+      - name: Checkout clang-tidy review
+        run: |
+          rm -rf ./.github/actions/clang-tidy-review
+          git clone https://github.com/ZedThree/clang-tidy-review .github/actions/clang-tidy-review
+
+          pushd .github/actions/clang-tidy-review &>/dev/null
+          git checkout 2c55ef8cfc9acb3715d433e58aea086dcec9b206
+          popd &>/dev/null
+
       - name: Run clang-tidy review
         uses: ./.github/actions/clang-tidy-review
         id: review

.github/workflows/license-eyes.yml

@@ -30,15 +30,12 @@ jobs:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Check License
         uses: apache/[email protected]

.github/workflows/pr-approve-status.yml

@@ -33,7 +33,7 @@ jobs:
               echo "PR number is not set"
               exit 1
           fi
-          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews")
+          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews?per_page=100")
           # shellcheck disable=SC2207
           reviewers=($(echo $response | jq -r '.[] | .user.login'))
           # shellcheck disable=SC2207

.gitmodules

@@ -4,9 +4,6 @@
 [submodule ".github/actions/get-workflow-origin"]
 	path = .github/actions/get-workflow-origin
 	url = https://github.com/potiuk/get-workflow-origin.git
-[submodule ".github/actions/clang-format-lint-action"]
-	path = .github/actions/clang-format-lint-action
-	url = https://github.com/DoozyX/clang-format-lint-action.git
 [submodule ".github/actions/setup-maven"]
 	path = .github/actions/setup-maven
 	url = https://github.com/stCarolas/setup-maven.git
@@ -19,12 +16,6 @@
 [submodule ".github/actions/ccache-action"]
 	path = .github/actions/ccache-action
 	url = https://github.com/hendrikmuhs/ccache-action
-[submodule ".github/actions/action-sh-checker"]
-	path = .github/actions/action-sh-checker
-	url = https://github.com/luizm/action-sh-checker
-[submodule ".github/actions/clang-tidy-review"]
-	path = .github/actions/clang-tidy-review
-	url = https://github.com/ZedThree/clang-tidy-review.git
 [submodule "be/src/apache-orc"]
 	path = be/src/apache-orc
 	url = https://github.com/apache/doris-thirdparty.git

README.md

@@ -35,8 +35,6 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
-Doris Summit Asia 2023 is coming and warmly invite you to join! Click Now 🔗[doris-summit.org.cn](https://doris-summit.org.cn/?utm_source=website&utm_medium=readme&utm_campaign=2023&utm_id=2023)
-
 🎉 Version 2.0.2 version released now. The 2.0.2 version has achieved over 10x performance improvements on standard Benchmark, comprehensive enhancement in log analysis and lakehouse scenarios, more efficient and stable data update and write efficiency, support for more comprehensive multi-tenant and resource isolation mechanisms, and take a new step in the direction of resource elasticity and storage computing separation. It has also been added a series of usability features for enterprise users. We welcome all users who have requirements for the new features of the 2.0 version to deploy and upgrade. Check out the 🔗[Release Notes](https://github.com/apache/doris/issues/25011) here. 
 
 🎉 Version 1.2.7 released now! It is fully evolved release and all users are encouraged to upgrade to this release. Check out the 🔗[Release Notes](https://doris.apache.org/docs/dev/releasenotes/release-1.2.7) here. 

be/cmake/thirdparty.cmake

@@ -145,17 +145,6 @@ endif()
 add_thirdparty(minizip LIB64)
 add_thirdparty(simdjson LIB64)
 add_thirdparty(idn LIB64)
-add_thirdparty(opentelemetry_common LIB64)
-add_thirdparty(opentelemetry_exporter_zipkin_trace LIB64)
-add_thirdparty(opentelemetry_resources LIB64)
-add_thirdparty(opentelemetry_version LIB64)
-add_thirdparty(opentelemetry_exporter_ostream_span LIB64)
-add_thirdparty(opentelemetry_trace LIB64)
-add_thirdparty(opentelemetry_http_client_curl LIB64)
-add_thirdparty(opentelemetry_exporter_otlp_http LIB64)
-add_thirdparty(opentelemetry_exporter_otlp_http_client LIB64)
-add_thirdparty(opentelemetry_otlp_recordable LIB64)
-add_thirdparty(opentelemetry_proto LIB64)
 add_thirdparty(xml2 LIB64)
 add_thirdparty(lzma LIB64)
 add_thirdparty(gsasl)

be/src/agent/cgroup_cpu_ctl.cpp

@@ -54,6 +54,19 @@ void CgroupCpuCtl::update_cpu_hard_limit(int cpu_hard_limit) {
     }
 }
 
+void CgroupCpuCtl::update_cpu_soft_limit(int cpu_shares) {
+    if (!_init_succ) {
+        return;
+    }
+    std::lock_guard<std::shared_mutex> w_lock(_lock_mutex);
+    if (_cpu_shares != cpu_shares) {
+        Status ret = modify_cg_cpu_soft_limit_no_lock(cpu_shares);
+        if (ret.ok()) {
+            _cpu_shares = cpu_shares;
+        }
+    }
+}
+
 Status CgroupCpuCtl::write_cg_sys_file(std::string file_path, int value, std::string msg,
                                        bool is_append) {
     int fd = open(file_path.c_str(), is_append ? O_RDWR | O_APPEND : O_RDWR);
@@ -97,9 +110,11 @@ Status CgroupV1CpuCtl::init() {
         }
     }
 
-    // quota path
+    // quota file
     _cgroup_v1_cpu_tg_quota_file = _cgroup_v1_cpu_tg_path + "/cpu.cfs_quota_us";
-    // task path
+    // cpu.shares file
+    _cgroup_v1_cpu_tg_shares_file = _cgroup_v1_cpu_tg_path + "/cpu.shares";
+    // task file
     _cgroup_v1_cpu_tg_task_file = _cgroup_v1_cpu_tg_path + "/tasks";
     LOG(INFO) << "cgroup v1 cpu path init success"
               << ", query tg path=" << _cgroup_v1_cpu_tg_path
@@ -110,6 +125,11 @@ Status CgroupV1CpuCtl::init() {
     return Status::OK();
 }
 
+Status CgroupV1CpuCtl::modify_cg_cpu_soft_limit_no_lock(int cpu_shares) {
+    std::string msg = "modify cpu shares to " + std::to_string(cpu_shares);
+    return CgroupCpuCtl::write_cg_sys_file(_cgroup_v1_cpu_tg_shares_file, cpu_shares, msg, false);
+}
+
 Status CgroupV1CpuCtl::modify_cg_cpu_hard_limit_no_lock(int cpu_hard_limit) {
     int val = _cpu_cfs_period_us * _cpu_core_num * cpu_hard_limit / 100;
     std::string msg = "modify cpu quota value to " + std::to_string(val);
@@ -120,9 +140,14 @@ Status CgroupV1CpuCtl::add_thread_to_cgroup() {
     if (!_init_succ) {
         return Status::OK();
     }
+#if defined(__APPLE__)
+    //unsupported now
+    return Status::OK();
+#else
     int tid = static_cast<int>(syscall(SYS_gettid));
     std::string msg = "add thread " + std::to_string(tid) + " to group";
     std::lock_guard<std::shared_mutex> w_lock(_lock_mutex);
     return CgroupCpuCtl::write_cg_sys_file(_cgroup_v1_cpu_tg_task_file, tid, msg, true);
+#endif
 }
 } // namespace doris