Merge pull request #304 from tbs60/dev_tming

Dev tming

src/backend/booster/bk_dist/common/env/env.go

@@ -65,6 +65,16 @@ const (
 	KeyExecutorUseWebSocket            = "USE_WEBSOCKET"
 	KeyExecutorNewShader               = "NEW_SHADER"
 	KeyExecutorSearchToolchain         = "SEARCH_TOOLCHAIN"
+	KeyExecutorUECLNotUseLocal         = "UE_CL_NOT_USE_LOCAL"
+	KeyExecutorUECCNotUseLocal         = "UE_CC_NOT_USE_LOCAL"
+	KeyExecutorUELibNotUseLocal        = "UE_LIB_NOT_USE_LOCAL"
+	KeyExecutorUELinkNotUseLocal       = "UE_LINK_NOT_USE_LOCAL"
+	KeyExecutorUEShaderNotUseLocal     = "UE_SHADER_NOT_USE_LOCAL"
+	KeyExecutorUECLLocalCPUWeight      = "UE_CL_LOCAL_CPU_WEIGHT"
+	KeyExecutorUECCLocalCPUWeight      = "UE_CC_LOCAL_CPU_WEIGHT"
+	KeyExecutorUELibLocalCPUWeight     = "UE_LIB_LOCAL_CPU_WEIGHT"
+	KeyExecutorUELinkLocalCPUWeight    = "UE_LINK_LOCAL_CPU_WEIGHT"
+	KeyExecutorUEShaderLocalCPUWeight  = "UE_SHADER_LOCAL_CPU_WEIGHT"
 
 	KeyUserDefinedLogLevel         = "USER_DEFINED_LOG_LEVEL"
 	KeyUserDefinedExecutorLogLevel = "USER_DEFINED_EXECUTOR_LOG_LEVEL"

src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go

@@ -523,3 +523,11 @@ func (e *executor) retryAndSuccessTooManyAndDegradeDirectly() bool {
 
 	return false
 }
+
+func (e *executor) canExecuteWithLocalIdleResource() bool {
+	if e.handler != nil {
+		return e.handler.CanExecuteWithLocalIdleResource(e.req.Commands)
+	}
+
+	return false
+}

src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go

@@ -108,7 +108,8 @@ func (m *Mgr) GetPumpCache() (*analyser.FileCache, *analyser.RootCache) {
 func (m *Mgr) ExecuteTask(
 	req *types.LocalTaskExecuteRequest,
 	globalWork *types.Work,
-	withlocalresource bool) (*types.LocalTaskExecuteResult, error) {
+	canUseLocalIdleResource bool,
+	f types.CallbackCheckLocalResource) (*types.LocalTaskExecuteResult, error) {
 	blog.Infof("local: try to execute task(%s) for work(%s) from pid(%d) in env(%v) dir(%s)",
 		strings.Join(req.Commands, " "), m.work.ID(), req.Pid, req.Environments, req.Dir)
 
@@ -136,11 +137,14 @@ func (m *Mgr) ExecuteTask(
 		return e.executeLocalTask(), nil
 	}
 
+	// TODO : 本地空闲资源执行任务需要更多条件判断
 	// 该任务已确定用本地资源运行，则直接走本地执行
-	if withlocalresource {
-		blog.Infof("local: execute task for work(%s) from pid(%d) degrade to local for with local resource",
-			m.work.ID(), req.Pid)
-		return e.executeLocalTask(), nil
+	if canUseLocalIdleResource {
+		if e.canExecuteWithLocalIdleResource() && f() {
+			blog.Infof("local: execute task [%s] for work(%s) from pid(%d) degrade to local with local idle resource",
+				req.Commands[0], m.work.ID(), req.Pid)
+			return e.executeLocalTask(), nil
+		}
 	}
 
 	// 优化没有远程资源转本地的逻辑； 如果没有远程资源，则先获取本地锁，然后转本地执行

src/backend/booster/bk_dist/controller/pkg/manager/manager.go

@@ -407,11 +407,23 @@ func (m *mgr) ExecuteLocalTask(
 	dcSDK.StatsTimeNow(&req.Stats.EnterTime)
 	defer dcSDK.StatsTimeNow(&req.Stats.LeaveTime)
 	work.Basic().UpdateJobStats(req.Stats)
-	withlocalresource := m.checkRunWithLocalResource(work)
+
+	// TODO : 是否转本地执行的判断，下放到basic执行，结合handle的信息
+	//        另外，本地锁需要加上本地权重
+	// withlocalresource := m.checkRunWithLocalResource(work)
+	// if withlocalresource {
+	// 	defer m.decLocalResourceTask()
+	// }
+	var withlocalresource bool
+	result, err := work.Local().ExecuteTask(req, globalWork, m.canUseLocalIdleResource(), func() bool {
+		withlocalresource = m.checkRunWithLocalResource(work)
+		blog.Infof("mgr: check run with local resource for work(%s) from pid(%d) got %v",
+			workID, req.Pid, withlocalresource)
+		return withlocalresource
+	})
 	if withlocalresource {
-		defer m.decLocalResourceTask()
+		m.decLocalResourceTask()
 	}
-	result, err := work.Local().ExecuteTask(req, globalWork, withlocalresource)
 	if err != nil {
 		if result == nil {
 			result = &types.LocalTaskExecuteResult{Result: &dcSDK.LocalTaskResult{
@@ -821,6 +833,10 @@ func sdkToolChain2Types(sdkToolChain *dcSDK.OneToolChain) *types.ToolChain {
 	}
 }
 
+func (m *mgr) canUseLocalIdleResource() bool {
+	return m.conf.UseLocalCPUPercent > 0 && m.conf.UseLocalCPUPercent <= 100
+}
+
 func (m *mgr) checkRunWithLocalResource(work *types.Work) bool {
 	if m.conf.UseLocalCPUPercent <= 0 || m.conf.UseLocalCPUPercent > 100 {
 		return false
@@ -880,6 +896,7 @@ func (m *mgr) checkRunWithLocalResource(work *types.Work) bool {
 		work.Basic().Info().WorkID())
 
 	atomic.AddInt32(&m.localResourceTaskNum, 1)
+
 	return true
 }
 

src/backend/booster/bk_dist/controller/pkg/types/interface.go

@@ -112,6 +112,8 @@ type RemoteMgr interface {
 	DecRemoteJobs()
 }
 
+type CallbackCheckLocalResource func() bool
+
 // LocalMgr describe a manager for handling all actions with local execution for work
 type LocalMgr interface {
 	// init handler
@@ -131,7 +133,8 @@ type LocalMgr interface {
 	// do task execution
 	ExecuteTask(req *LocalTaskExecuteRequest,
 		globalWork *Work,
-		withlocalresource bool) (*LocalTaskExecuteResult, error)
+		canUseLocalIdleResource bool,
+		f CallbackCheckLocalResource) (*LocalTaskExecuteResult, error)
 
 	// get caches in pump mode
 	GetPumpCache() (*analyser.FileCache, *analyser.RootCache)

src/backend/booster/bk_dist/handler/cc/handler.go

@@ -127,6 +127,10 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) {
 	cc.sandbox = sandbox
 }
 
+func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock 如果编译本身是预处理, 那么不需要pre-lock, 因为它会在pre-execute中转本地, 不会真正地执行预处理
 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool {
 	for _, arg := range command {

src/backend/booster/bk_dist/handler/cc/utils.go

@@ -756,12 +756,12 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) {
 				return nil, ErrorNotSupportDr
 			}
 
-			// ++ by tomtian 2021-05-18
-			if strings.HasPrefix(arg, "-fsanitize") {
-				blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg)
-				return nil, ErrorNotSupportFsanitize
-			}
-			// --
+			// // ++ by tomtian 2021-05-18
+			// if strings.HasPrefix(arg, "-fsanitize") {
+			// 	blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg)
+			// 	return nil, ErrorNotSupportFsanitize
+			// }
+			// // --
 
 			if strings.HasPrefix(arg, "-I") {
 				// if -I just a prefix, save the remain of this line.

src/backend/booster/bk_dist/handler/custom/handler.go

@@ -139,6 +139,10 @@ func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDi
 	return c.innerHandler.PreExecute(command)
 }
 
+func (c *Custom) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock 决定是否需要在执行PreExecute之前获取一个pre-lock
 func (c *Custom) PreExecuteNeedLock(command []string) bool {
 	return c.innerHandler.PreExecuteNeedLock(command)

src/backend/booster/bk_dist/handler/echo/handler.go

@@ -107,6 +107,10 @@ func (c *Echo) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *Echo) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock decide whether should lock when executor do the pre-process
 func (c *Echo) PreExecuteNeedLock(command []string) bool {
 	return false

src/backend/booster/bk_dist/handler/find/handler.go

@@ -152,6 +152,10 @@ func (c *Finder) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *Finder) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock decide whether should lock when executor do the pre-process
 func (c *Finder) PreExecuteNeedLock(command []string) bool {
 	return false

src/backend/booster/bk_dist/handler/handler.go

@@ -40,6 +40,10 @@ type Handler interface {
 	// GetFilterRules return files which will be used by multi-executor, send only once
 	GetFilterRules() ([]dcSDK.FilterRuleItem, error)
 
+	// CanExecuteWithLocalIdleResource return whether can execute this with local idle resource,
+	// true by default
+	CanExecuteWithLocalIdleResource(command []string) bool
+
 	// PreExecuteNeedLock decide whether executor should lock before pre execution
 	PreExecuteNeedLock(command []string) bool
 

src/backend/booster/bk_dist/handler/tc/handler.go

@@ -142,6 +142,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (c *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock no need
 func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool {
 	return false

src/backend/booster/bk_dist/handler/ue4/astc/handler.go

@@ -120,6 +120,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	return nil, nil
 }
 
+func (cc *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool {
+	return true
+}
+
 // PreExecuteNeedLock no need
 func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool {
 	return false

src/backend/booster/bk_dist/handler/ue4/cc/handler.go

@@ -127,6 +127,14 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) {
 	cc.sandbox = sandbox
 }
 
+func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cc.sandbox.Env.GetEnv(env.KeyExecutorUECCNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 需要pre-lock来保证预处理不会跑满本地资源
 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool {
 	return true
@@ -154,6 +162,14 @@ func (cc *TaskCC) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (cc *TaskCC) LocalLockWeight(command []string) int32 {
+	envvalue := cc.sandbox.Env.GetEnv(env.KeyExecutorUECCLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
@@ -1129,7 +1145,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 	if r.Results[0].RetCode == 0 {
 		blog.Infof("cc: success done post execute for: %v", cc.originArgs)
 		// set output to inputFile
-		r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile))
+		// r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile))
 		// if remote succeed with pump,do not need copy head file
 		if cc.pumpremote {
 			cc.needcopypumpheadfile = false

src/backend/booster/bk_dist/handler/ue4/cl/handler.go

@@ -223,6 +223,14 @@ func (cl *TaskCL) getPreLoadConfigPath(config dcType.BoosterConfig) string {
 	return dcConfig.GetFile(hookConfigPathCCCommon)
 }
 
+func (cl *TaskCL) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cl.sandbox.Env.GetEnv(env.KeyExecutorUECLNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (cl *TaskCL) PreExecuteNeedLock(command []string) bool {
 	return true
@@ -280,6 +288,14 @@ func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.B
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (cl *TaskCL) LocalLockWeight(command []string) int32 {
+	envvalue := cl.sandbox.Env.GetEnv(env.KeyExecutorUECLLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }
 
@@ -1071,7 +1087,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 			}
 		} else {
 			// simulate output with inputFile
-			r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile))
+			// r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile))
 		}
 
 		// if remote succeed with pump,do not need copy head file

src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go

@@ -90,6 +90,14 @@ func (cf *TaskCLFilter) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Pr
 	return nil, nil
 }
 
+func (cf *TaskCLFilter) CanExecuteWithLocalIdleResource(command []string) bool {
+	if cf.clhandle != nil {
+		return cf.clhandle.CanExecuteWithLocalIdleResource(command)
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (cf *TaskCLFilter) PreExecuteNeedLock(command []string) bool {
 	return true

src/backend/booster/bk_dist/handler/ue4/handler.go

@@ -150,6 +150,20 @@ func (u *UE4) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	}, nil
 }
 
+func (u *UE4) CanExecuteWithLocalIdleResource(command []string) bool {
+	if u.innerhandler == nil {
+		u.initInnerHandle(command)
+	}
+	if u.innerhandler != nil {
+		if u.sandbox != nil {
+			u.innerhandler.InitSandbox(u.sandbox.Fork())
+		}
+		return u.innerhandler.CanExecuteWithLocalIdleResource(command)
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 防止预处理跑满本机CPU
 func (u *UE4) PreExecuteNeedLock(command []string) bool {
 	return true

src/backend/booster/bk_dist/handler/ue4/lib/handler.go

@@ -12,6 +12,7 @@ package lib
 import (
 	"fmt"
 	"path/filepath"
+	"runtime"
 	"strconv"
 
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env"
@@ -79,6 +80,14 @@ func (l *TaskLib) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.PreloadC
 	return nil, nil
 }
 
+func (l *TaskLib) CanExecuteWithLocalIdleResource(command []string) bool {
+	if l.sandbox.Env.GetEnv(env.KeyExecutorUELibNotUseLocal) == "true" {
+		return false
+	}
+
+	return true
+}
+
 // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock
 func (l *TaskLib) PreExecuteNeedLock(command []string) bool {
 	return false
@@ -136,6 +145,14 @@ func (l *TaskLib) LocalExecuteNeed(command []string) bool {
 
 // LocalLockWeight decide local-execute lock weight, default 1
 func (l *TaskLib) LocalLockWeight(command []string) int32 {
+	envvalue := l.sandbox.Env.GetEnv(env.KeyExecutorUELibLocalCPUWeight)
+	if envvalue != "" {
+		w, err := strconv.Atoi(envvalue)
+		if err == nil && w > 0 && w <= runtime.NumCPU() {
+			return int32(w)
+		}
+	}
+
 	return 1
 }