From fa1de3a9a22c26b577bf85bb97450bcc59b6726d Mon Sep 17 00:00:00 2001
From: tbs60 <bktbs60@gmail.com>
Date: Mon, 30 Sep 2024 09:47:59 +0800
Subject: [PATCH 1/2] feat:opt local idle resource use, issue: #303

---
 src/backend/booster/bk_dist/common/env/env.go | 10 ++++++++
 .../controller/pkg/manager/local/executor.go  |  8 +++++++
 .../controller/pkg/manager/local/mgr.go       | 14 +++++++----
 .../bk_dist/controller/pkg/manager/manager.go | 23 ++++++++++++++++---
 .../bk_dist/controller/pkg/types/interface.go |  5 +++-
 .../booster/bk_dist/handler/cc/handler.go     |  4 ++++
 .../booster/bk_dist/handler/custom/handler.go |  4 ++++
 .../booster/bk_dist/handler/echo/handler.go   |  4 ++++
 .../booster/bk_dist/handler/find/handler.go   |  4 ++++
 .../booster/bk_dist/handler/handler.go        |  4 ++++
 .../booster/bk_dist/handler/tc/handler.go     |  4 ++++
 .../bk_dist/handler/ue4/astc/handler.go       |  4 ++++
 .../booster/bk_dist/handler/ue4/cc/handler.go | 18 ++++++++++++++-
 .../booster/bk_dist/handler/ue4/cl/handler.go | 18 ++++++++++++++-
 .../bk_dist/handler/ue4/clfilter/handler.go   |  8 +++++++
 .../booster/bk_dist/handler/ue4/handler.go    | 14 +++++++++++
 .../bk_dist/handler/ue4/lib/handler.go        | 17 ++++++++++++++
 .../bk_dist/handler/ue4/link/handler.go       | 17 ++++++++++++++
 .../bk_dist/handler/ue4/linkfilter/handler.go |  8 +++++++
 .../bk_dist/handler/ue4/shader/handler.go     | 20 ++++++++++++++++
 .../bk_dist/handler/winclangcl/handler.go     |  4 ++++
 .../bk_dist/shadertool/common/types.go        |  1 +
 .../bk_dist/shadertool/pkg/shadertool.go      |  1 +
 23 files changed, 203 insertions(+), 11 deletions(-)

diff --git a/src/backend/booster/bk_dist/common/env/env.go b/src/backend/booster/bk_dist/common/env/env.go
index 83313817b..1c633565a 100644
--- a/src/backend/booster/bk_dist/common/env/env.go
+++ b/src/backend/booster/bk_dist/common/env/env.go
@@ -65,6 +65,16 @@ const (
 	KeyExecutorUseWebSocket            = "USE_WEBSOCKET"
 	KeyExecutorNewShader               = "NEW_SHADER"
 	KeyExecutorSearchToolchain         = "SEARCH_TOOLCHAIN"
+	KeyExecutorUECLNotUseLocal         = "UE_CL_NOT_USE_LOCAL"
+	KeyExecutorUECCNotUseLocal         = "UE_CC_NOT_USE_LOCAL"
+	KeyExecutorUELibNotUseLocal        = "UE_LIB_NOT_USE_LOCAL"
+	KeyExecutorUELinkNotUseLocal       = "UE_LINK_NOT_USE_LOCAL"
+	KeyExecutorUEShaderNotUseLocal     = "UE_SHADER_NOT_USE_LOCAL"
+	KeyExecutorUECLLocalCPUWeight      = "UE_CL_LOCAL_CPU_WEIGHT"
+	KeyExecutorUECCLocalCPUWeight      = "UE_CC_LOCAL_CPU_WEIGHT"
+	KeyExecutorUELibLocalCPUWeight     = "UE_LIB_LOCAL_CPU_WEIGHT"
+	KeyExecutorUELinkLocalCPUWeight    = "UE_LINK_LOCAL_CPU_WEIGHT"
+	KeyExecutorUEShaderLocalCPUWeight  = "UE_SHADER_LOCAL_CPU_WEIGHT"
 
 	KeyUserDefinedLogLevel         = "USER_DEFINED_LOG_LEVEL"
 	KeyUserDefinedExecutorLogLevel = "USER_DEFINED_EXECUTOR_LOG_LEVEL"
diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go
index b62f634f8..1a2d90ecd 100644
--- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go
+++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go
@@ -523,3 +523,11 @@ func (e *executor) retryAndSuccessTooManyAndDegradeDirectly() bool {
 
 	return false
 }
+
+func (e *executor) canExecuteWithLocalIdleResource() bool {
+	if e.handler != nil {
+		return e.handler.CanExecuteWithLocalIdleResource(e.req.Commands)
+	}
+
+	return false
+}
diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go
index 011421d07..427618717 100644
--- a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go
+++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go
@@ -108,7 +108,8 @@ func (m *Mgr) GetPumpCache() (*analyser.FileCache, *analyser.RootCache) {
 func (m *Mgr) ExecuteTask(
 	req *types.LocalTaskExecuteRequest,
 	globalWork *types.Work,
-	withlocalresource bool) (*types.LocalTaskExecuteResult, error) {
+	canUseLocalIdleResource bool,
+	f types.CallbackCheckLocalResource) (*types.LocalTaskExecuteResult, error) {
 	blog.Infof("local: try to execute task(%s) for work(%s) from pid(%d) in env(%v) dir(%s)",
 		strings.Join(req.Commands, " "), m.work.ID(), req.Pid, req.Environments, req.Dir)
 
@@ -136,11 +137,14 @@ func (m *Mgr) ExecuteTask(
 		return e.executeLocalTask(), nil
 	}
 
+	// TODO : 本地空闲资源执行任务需要更多条件判断
 	// 该任务已确定用本地资源运行，则直接走本地执行
-	if withlocalresource {
-		blog.Infof("local: execute task for work(%s) from pid(%d) degrade to local for with local resource",
-			m.work.ID(), req.Pid)
-		return e.executeLocalTask(), nil
+	if canUseLocalIdleResource {
+		if e.canExecuteWithLocalIdleResource() && f() {
+			blog.Infof("local: execute task [%s] for work(%s) from pid(%d) degrade to local with local idle resource",
+				req.Commands[0], m.work.ID(), req.Pid)
+			return e.executeLocalTask(), nil
+		}
 	}
 
 	// 优化没有远程资源转本地的逻辑； 如果没有远程资源，则先获取本地锁，然后转本地执行
diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/manager.go b/src/backend/booster/bk_dist/controller/pkg/manager/manager.go
index c8186f586..c35b8bed4 100644
--- a/src/backend/booster/bk_dist/controller/pkg/manager/manager.go
+++ b/src/backend/booster/bk_dist/controller/pkg/manager/manager.go
@@ -407,11 +407,23 @@ func (m *mgr) ExecuteLocalTask(
 	dcSDK.StatsTimeNow(&req.Stats.EnterTime)
 	defer dcSDK.StatsTimeNow(&req.Stats.LeaveTime)
 	work.Basic().UpdateJobStats(req.Stats)
-	withlocalresource := m.checkRunWithLocalResource(work)
+
+	// TODO : 是否转本地执行的判断，下放到basic执行，结合handle的信息
+	//        另外，本地锁需要加上本地权重
+	// withlocalresource := m.checkRunWithLocalResource(work)
+	// if withlocalresource {
+	// 	defer m.decLocalResourceTask()
+	// }
+	var withlocalresource bool
+	result, err := work.Local().ExecuteTask(req, globalWork, m.canUseLocalIdleResource(), func() bool {
+		withlocalresource = m.checkRunWithLocalResource(work)
+		blog.Infof("mgr: check run with local resource for work(%s) from pid(%d) got %v",
+			workID, req.Pid, withlocalresource)
+		return withlocalresource
+	})
 	if withlocalresource {
-		defer m.decLocalResourceTask()
+		m.decLocalResourceTask()
 	}
-	result, err := work.Local().ExecuteTask(req, globalWork, withlocalresource)
 	if err != nil {
 		if result == nil {
 			result = &types.LocalTaskExecuteResult{Result: &dcSDK.LocalTaskResult{
@@ -821,6 +833,10 @@ func sdkToolChain2Types(sdkToolChain *dcSDK.OneToolChain) *types.ToolChain {
 	}
 }
 
+func (m *mgr) canUseLocalIdleResource() bool {
+	return m.conf.UseLocalCPUPercent > 0 && m.conf.UseLocalCPUPercent <= 100
+}
+
 func (m *mgr) checkRunWithLocalResource(work *types.Work) bool {
 	if m.conf.UseLocalCPUPercent <= 0 || m.conf.UseLocalCPUPercent > 100 {
 		return false
@@ -880,6 +896,7 @@ func (m *mgr) checkRunWithLocalResource(work *types.Work) bool {
 		work.Basic().Info().WorkID())
 
 	atomic.AddInt32(&m.localResourceTaskNum, 1)
+
 	return true
 }
 
diff --git a/src/backend/booster/bk_dist/controller/pkg/types/interface.go b/src/backend/booster/bk_dist/controller/pkg/types/interface.go
index 518824816..628b2d59f 100644
--- a/src/backend/booster/bk_dist/controller/pkg/types/interface.go
+++ b/src/backend/booster/bk_dist/controller/pkg/types/interface.go
@@ -112,6 +112,8 @@ type RemoteMgr interface {
 	DecRemoteJobs()
 }
 
+type CallbackCheckLocalResource func() bool
+
 // LocalMgr describe a manager for handling all actions with local execution for work
 type LocalMgr interface {
 	// init handler
@@ -131,7 +133,8 @@ type LocalMgr interface {
 	// do task execution
 	ExecuteTask(req *LocalTaskExecuteRequest,
 		globalWork *Work,
-		withlocalresource bool) (*LocalTaskExecuteResult, error)
+		canUseLocalIdleResource bool,
+		f CallbackCheckLocalResource) (*LocalTaskExecuteResult, error)
 
 	// get caches in pump mode
 	GetPumpCache() (*analyser.FileCache, *analyser.RootCache)
diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go
index a70e24be5..0e8ca25b4 100644
--- a/src/backend/booster/bk_dist/handler/cc/handler.go
+++ b/src/backend/booster/bk_dist/handler/cc/handler.go
@@ -127,6 +127,10 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) {
 	cc.sandbox = sandbox
 }
 
+func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock 如果编译本身是预处理, 那么不需要pre-lock, 因为它会在pre-execute中转本地, 不会真正地执行预处理
 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool {
 	for _, arg := range command {
diff --git a/src/backend/booster/bk_dist/handler/custom/handler.go b/src/backend/booster/bk_dist/handler/custom/handler.go
index 501833923..7f3002cef 100644
--- a/src/backend/booster/bk_dist/handler/custom/handler.go
+++ b/src/backend/booster/bk_dist/handler/custom/handler.go
@@ -139,6 +139,10 @@ func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDi
 	return c.innerHandler.PreExecute(command)
 }
 
+func (c *Custom) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock 决定是否需要在执行PreExecute之前获取一个pre-lock
 func (c *Custom) PreExecuteNeedLock(command []string) bool {
 	return c.innerHandler.PreExecuteNeedLock(command)
diff --git a/src/backend/booster/bk_dist/handler/echo/handler.go b/src/backend/booster/bk_dist/handler/echo/handler.go
index d29597ebc..a3f965ef3 100644
--- a/src/backend/booster/bk_dist/handler/echo/handler.go
+++ b/src/backend/booster/bk_dist/handler/echo/handler.go
@@ -107,6 +107,10 @@ func (c *Echo) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *Echo) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock decide whether should lock when executor do the pre-process
 func (c *Echo) PreExecuteNeedLock(command []string) bool {
 	return false
diff --git a/src/backend/booster/bk_dist/handler/find/handler.go b/src/backend/booster/bk_dist/handler/find/handler.go
index b656f8d15..2a1226058 100644
--- a/src/backend/booster/bk_dist/handler/find/handler.go
+++ b/src/backend/booster/bk_dist/handler/find/handler.go
@@ -152,6 +152,10 @@ func (c *Finder) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *Finder) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock decide whether should lock when executor do the pre-process
 func (c *Finder) PreExecuteNeedLock(command []string) bool {
 	return false
diff --git a/src/backend/booster/bk_dist/handler/handler.go b/src/backend/booster/bk_dist/handler/handler.go
index ac109f9e8..5c5928ded 100644
--- a/src/backend/booster/bk_dist/handler/handler.go
+++ b/src/backend/booster/bk_dist/handler/handler.go
@@ -40,6 +40,10 @@ type Handler interface {
 	// GetFilterRules return files which will be used by multi-executor, send only once
 	GetFilterRules() ([]dcSDK.FilterRuleItem, error)
 
+	// CanExecuteWithLocalIdleResource return whether can execute this with local idle resource,
+	// true by default
+	CanExecuteWithLocalIdleResource(command []string) bool
+
 	// PreExecuteNeedLock decide whether executor should lock before pre execution
 	PreExecuteNeedLock(command []string) bool
 
diff --git a/src/backend/booster/bk_dist/handler/tc/handler.go b/src/backend/booster/bk_dist/handler/tc/handler.go
index de685ecf6..acc5d021c 100644
--- a/src/backend/booster/bk_dist/handler/tc/handler.go
+++ b/src/backend/booster/bk_dist/handler/tc/handler.go
@@ -142,6 +142,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock no need
 func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool {
 	return false
diff --git a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go
index 4899cdf34..d6f46a06d 100644
--- a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go
@@ -120,6 +120,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (cc *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock no need
 func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool {
 	return false
diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go
index 90c578357..f214e0ce0 100644
--- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go
@@ -127,6 +127,14 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) {
 	cc.sandbox = sandbox
 }
 
+func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cc.sandbox.Env.GetEnv(env.KeyExecutorUECCNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 需要pre-lock来保证预处理不会跑满本地资源
 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool {
 	return true
@@ -154,6 +162,14 @@ func (cc *TaskCC) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (cc *TaskCC) LocalLockWeight(command []string) int32 {
+	envvalue := cc.sandbox.Env.GetEnv(env.KeyExecutorUECCLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
@@ -1129,7 +1145,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 	if r.Results[0].RetCode == 0 {
 		blog.Infof("cc: success done post execute for: %v", cc.originArgs)
 		// set output to inputFile
-		r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile))
+		// r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile))
 		// if remote succeed with pump,do not need copy head file
 		if cc.pumpremote {
 			cc.needcopypumpheadfile = false
diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go
index 92d9709f5..d51af3118 100644
--- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go
@@ -223,6 +223,14 @@ func (cl *TaskCL) getPreLoadConfigPath(config dcType.BoosterConfig) string {
 	return dcConfig.GetFile(hookConfigPathCCCommon)
 }
 
+func (cl *TaskCL) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cl.sandbox.Env.GetEnv(env.KeyExecutorUECLNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (cl *TaskCL) PreExecuteNeedLock(command []string) bool {
 	return true
@@ -280,6 +288,14 @@ func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.B
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (cl *TaskCL) LocalLockWeight(command []string) int32 {
+	envvalue := cl.sandbox.Env.GetEnv(env.KeyExecutorUECLLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
@@ -1071,7 +1087,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 			}
 		} else {
 			// simulate output with inputFile
-			r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile))
+			// r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile))
 		}
 
 		// if remote succeed with pump,do not need copy head file
diff --git a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go
index 79af3a1b7..39b7aafb3 100644
--- a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go
@@ -90,6 +90,14 @@ func (cf *TaskCLFilter) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Pr
 	return nil, nil
 }
 
+func (cf *TaskCLFilter) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cf.clhandle != nil {
+		return cf.clhandle.CanExecuteWithLocalIdleResource(command)
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (cf *TaskCLFilter) PreExecuteNeedLock(command []string) bool {
 	return true
diff --git a/src/backend/booster/bk_dist/handler/ue4/handler.go b/src/backend/booster/bk_dist/handler/ue4/handler.go
index d8d0987ae..6aaeacdea 100644
--- a/src/backend/booster/bk_dist/handler/ue4/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/handler.go
@@ -150,6 +150,20 @@ func (u *UE4) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	}, nil
 }
 
+func (u *UE4) CanExecuteWithLocalIdleResource(command []string) bool {
+	if u.innerhandler == nil {
+		u.initInnerHandle(command)
+	}
+	if u.innerhandler != nil {
+		if u.sandbox != nil {
+			u.innerhandler.InitSandbox(u.sandbox.Fork())
+		}
+		return u.innerhandler.CanExecuteWithLocalIdleResource(command)
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (u *UE4) PreExecuteNeedLock(command []string) bool {
 	return true
diff --git a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go
index 851a911b2..78f490aec 100644
--- a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go
@@ -12,6 +12,7 @@ package lib
 import (
 	"fmt"
 	"path/filepath"
+	"runtime"
 	"strconv"
 
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env"
@@ -79,6 +80,14 @@ func (l *TaskLib) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.PreloadC
 	return nil, nil
 }
 
+func (l *TaskLib) CanExecuteWithLocalIdleResource(command []string) bool {
+	if l.sandbox.Env.GetEnv(env.KeyExecutorUELibNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock
 func (l *TaskLib) PreExecuteNeedLock(command []string) bool {
 	return false
@@ -136,6 +145,14 @@ func (l *TaskLib) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (l *TaskLib) LocalLockWeight(command []string) int32 {
+	envvalue := l.sandbox.Env.GetEnv(env.KeyExecutorUELibLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
diff --git a/src/backend/booster/bk_dist/handler/ue4/link/handler.go b/src/backend/booster/bk_dist/handler/ue4/link/handler.go
index f6c855adc..063afadfd 100644
--- a/src/backend/booster/bk_dist/handler/ue4/link/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/link/handler.go
@@ -12,6 +12,7 @@ package link
 import (
 	"fmt"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -95,6 +96,14 @@ func (l *TaskLink) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Preload
 	return nil, nil
 }
 
+func (l *TaskLink) CanExecuteWithLocalIdleResource(command []string) bool {
+	if l.sandbox.Env.GetEnv(env.KeyExecutorUELinkNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock
 func (l *TaskLink) PreExecuteNeedLock(command []string) bool {
 	return false
@@ -152,6 +161,14 @@ func (l *TaskLink) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (l *TaskLink) LocalLockWeight(command []string) int32 {
+	envvalue := l.sandbox.Env.GetEnv(env.KeyExecutorUELinkLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
diff --git a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go
index 5798804a4..70db2fa3b 100644
--- a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go
@@ -80,6 +80,14 @@ func (lf *TaskLinkFilter) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.
 	return nil, nil
 }
 
+func (lf *TaskLinkFilter) CanExecuteWithLocalIdleResource(command []string) bool {
+	if lf.handle != nil {
+		return lf.handle.CanExecuteWithLocalIdleResource(command)
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (lf *TaskLinkFilter) PreExecuteNeedLock(command []string) bool {
 	return true
diff --git a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go
index ff8a34872..14a620747 100644
--- a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go
+++ b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go
@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env"
@@ -83,6 +84,17 @@ func (u *UE4Shader) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (u *UE4Shader) CanExecuteWithLocalIdleResource(command []string) bool {
+	// only for debug
+	blog.Infof("shader: BK_DIST_UE_SHADER_NOT_USE_LOCAL=[%v]", u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderNotUseLocal))
+
+	if u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock
 func (u *UE4Shader) PreExecuteNeedLock(command []string) bool {
 	return false
@@ -366,6 +378,14 @@ func (u *UE4Shader) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (u *UE4Shader) LocalLockWeight(command []string) int32 {
+	envvalue := u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	/*	// default setting of ue 4.26
 		; Make sure we don't starve loading threads
 		NumUnusedShaderCompilingThreads=3
diff --git a/src/backend/booster/bk_dist/handler/winclangcl/handler.go b/src/backend/booster/bk_dist/handler/winclangcl/handler.go
index 201c82552..04108a494 100644
--- a/src/backend/booster/bk_dist/handler/winclangcl/handler.go
+++ b/src/backend/booster/bk_dist/handler/winclangcl/handler.go
@@ -69,6 +69,10 @@ func (cc *WinClangCl) RenderArgs(config dcType.BoosterConfig, originArgs string)
 	return originArgs
 }
 
+func (cc *WinClangCl) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 func (cc *WinClangCl) PreExecuteNeedLock(command []string) bool {
 
 	return true
diff --git a/src/backend/booster/bk_dist/shadertool/common/types.go b/src/backend/booster/bk_dist/shadertool/common/types.go
index 02caa77e1..ff59c8b6f 100644
--- a/src/backend/booster/bk_dist/shadertool/common/types.go
+++ b/src/backend/booster/bk_dist/shadertool/common/types.go
@@ -86,6 +86,7 @@ type ApplyParameters struct {
 	MaxLocalPostJobs              int               `json:"max_Local_post_jobs"`
 	Env                           map[string]string `json:"env"`
 	ContinueOnError               bool              `json:"continue_on_error" usage:"if false, the program will stop on error immediately"`
+	ControllerUseLocalCPUPercent  int               `json:"controller_use_local_cpu_percent"`
 }
 
 // Actionresult define action result
diff --git a/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go b/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go
index cb20a4f44..43e98fd47 100644
--- a/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go
+++ b/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go
@@ -195,6 +195,7 @@ func (h *ShaderTool) getControllerConfig() dcSDK.ControllerConfig {
 		LongTCP:             h.settings.ControllerLongTCP,
 		DynamicPort:         h.settings.ControllerDynamicPort,
 		WorkerOfferSlot:     h.settings.ControllerWorkerOfferSlot,
+		UseLocalCPUPercent:  h.settings.ControllerUseLocalCPUPercent,
 	}
 
 	return *h.controllerconfig

From 658634421fac41de4f4a8e94fd5b81d13970abcf Mon Sep 17 00:00:00 2001
From: tbs60 <bktbs60@gmail.com>
Date: Tue, 8 Oct 2024 15:04:21 +0800
Subject: [PATCH 2/2] fix: opt pump cache, issue: #296

---
 src/backend/booster/bk_dist/handler/cc/utils.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend/booster/bk_dist/handler/cc/utils.go b/src/backend/booster/bk_dist/handler/cc/utils.go
index 2286b3941..e35fbc956 100644
--- a/src/backend/booster/bk_dist/handler/cc/utils.go
+++ b/src/backend/booster/bk_dist/handler/cc/utils.go
@@ -756,12 +756,12 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) {
 				return nil, ErrorNotSupportDr
 			}
 
-			// ++ by tomtian 2021-05-18
-			if strings.HasPrefix(arg, "-fsanitize") {
-				blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg)
-				return nil, ErrorNotSupportFsanitize
-			}
-			// --
+			// // ++ by tomtian 2021-05-18
+			// if strings.HasPrefix(arg, "-fsanitize") {
+			// 	blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg)
+			// 	return nil, ErrorNotSupportFsanitize
+			// }
+			// // --
 
 			if strings.HasPrefix(arg, "-I") {
 				// if -I just a prefix, save the remain of this line.