重构进程reload逻辑，新增查看日志命令：log 和 logf

DEVELOP.md

@@ -0,0 +1,18 @@
+# pmon2 开发
+
+## 环境构建
+
+```shell
+# 运行init_dev.sh 初始化开发配置文件
+sh init_dev.sh
+```
+`init_dev.sh` 会负责生成项目的开发配置文件，以及相关目录、环境变量等。
+
+## 测试开发
+
+```shell
+sudo PMON2_CONF=config/config-dev.yml ./bin/pmond
+sudo PMON2_CONF=config/config-dev.yml ./bin/pmon2 exec bin/test 
+```
+
+因为 `pmon2` 启动进程使用的是fork/exec,所以需要sudo或root级别权限。

README.md

@@ -89,6 +89,8 @@ Available Commands:
   reload      reload some process
   start       start some process by id or name
   stop        stop running process
+  log         display process log by id or name
+  logf        display process log dynamic by id or name
   version     show pmon2 version
 ```
 
@@ -150,6 +152,16 @@ sudo pmon2 stop [id or name]
 sudo pmon2 reload [id or name]
 ```
 
+#### 插件进程日志
+
+```shell
+# 查看最近进程的日志
+sudo pmon2 log [id or name]
+
+# 动态查看进程日志，类似系统tail -f xxx.log
+sudo pmon2 logf [id or name]
+```
+
 仅仅重启配置文件，该命令需要所启动的进程配合使用，`reload` 命令默认仅仅发送 `SIGUSR2` 信号给启动的进程
 
 如果你希望 `reload` 时自定义信号，那么请使用 `--sig` 参数：

app/boot/conf.go

@@ -1,6 +1,7 @@
 package boot
 
 import (
+	"github.com/ntt360/errors"
 	"github.com/ntt360/pmon2/app/conf"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
@@ -9,7 +10,7 @@ import (
 func Conf(confFile string) (*conf.Tpl, error) {
 	d, err := ioutil.ReadFile(confFile)
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	var c conf.Tpl

app/conf/app.go

@@ -1,4 +1,15 @@
 package conf
 
+import "os"
+
 // current app version
-const Version = "1.9.0"
+var Version = "1.10.0"
+
+func GetDefaultConf() string{
+	conf := os.Getenv("PMON2_CONF")
+	if len(conf) == 0 {
+		conf = "/etc/pmon2/config/config.yml"
+	}
+	return conf
+
+}

app/conf/tpl.go

@@ -1,57 +1,15 @@
 package conf
 
-import (
-	"os"
-	"path"
-)
-
 type Tpl struct {
-	AppDir string
-	Data   string `yaml:"data"`
-	Logs   string `yaml:"logs"`
-	Sock   string `yaml:"sock"`
-	Bin    string `yaml:"bin"`
-	Conf   string
-}
-
-func (c *Tpl) GetSockFile() string {
-	if len(c.Sock) <= 0 {
-		panic("run sock file is empty")
-	}
-
-	sockDir := path.Dir(c.Sock)
-	_, err := os.Stat(sockDir)
-	if os.IsNotExist(err) {
-		err = os.MkdirAll(sockDir, 0755)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// if sock file exist, try to clear
-	_, err = os.Stat(c.Sock)
-	if !os.IsNotExist(err) {
-		err = os.Remove(c.Sock)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	return c.Sock
+	Data string `yaml:"data"`
+	Logs string `yaml:"logs"`
+	Conf string
 }
 
 func (c *Tpl) GetDataDir() string {
-	if path.IsAbs(c.Data) {
-		return c.Data
-	}
-
-	return c.AppDir + "/" + c.Data
+	return c.Data
 }
 
 func (c *Tpl) GetLogsDir() string {
-	if path.IsAbs(c.Logs) {
-		return c.Logs
-	}
-
-	return c.AppDir + "/" + c.Logs
+	return c.Logs
 }

app/executor/executor.go

@@ -1,6 +1,7 @@
 package executor
 
 import (
+	"github.com/ntt360/errors"
 	"github.com/ntt360/pmon2/app/model"
 	"github.com/ntt360/pmon2/app/utils/crypto"
 	"os"
@@ -42,7 +43,7 @@ func Exec(processFile, customLogFile, name, extArgs string, user *user.User, aut
 
 	process, err := os.StartProcess(processFile, processParams, attr)
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	pModel := model.Process{

app/executor/process_log.go

@@ -1,6 +1,7 @@
 package executor
 
 import (
+	"github.com/ntt360/errors"
 	"github.com/ntt360/pmon2/app"
 	"os"
 	"strings"
@@ -16,7 +17,7 @@ func getLogPath(customLogFile string, hash string) (string, error) {
 		if os.IsNotExist(err) {
 			err := os.MkdirAll(prjDir, 0755)
 			if err != nil {
-				return "", err
+				return "", errors.Wrapf(err, "err: %s, logs dir: '%s'", err.Error(), prjDir)
 			}
 		}
 		customLogFile = prjDir + "/" + hash + logSuffix
@@ -29,7 +30,7 @@ func getLogFile(customLogFile string) (*os.File, error) {
 	// 创建进程日志文件
 	logFile, err := os.OpenFile(customLogFile, syscall.O_CREAT|syscall.O_APPEND|syscall.O_WRONLY, 0755)
 	if err != nil {
-		return nil, err
+		return nil, errors.WithStack(err)
 	}
 
 	return logFile, nil

app/god/monitor.go

@@ -18,46 +18,58 @@ type Monitor struct {
 }
 
 func NewMonitor() {
-	go runMonitor()
+	runMonitor()
 }
 
 func runMonitor() {
 	timer := time.NewTicker(time.Millisecond * 500)
 	for {
-		select {
-		case <-timer.C:
-			runningTask()
-		}
+		<-timer.C
+		runningTask()
 	}
 }
 
 var pendingTask sync.Map
 
 func runningTask() {
 	var all []model.Process
-	err := app.Db().Find(&all).Error
+	err := app.Db().Find(&all, "status = ? or status = ?", model.StatusRunning, model.StatusFailed).Error
 	if err != nil {
 		return
 	}
 
 	for _, process := range all {
-		if process.Status == model.StatusStopped || process.Status == model.StatusReload || process.Status == model.StatusInit {
-			// state no need restart
-			continue
-		}
-
+		// just check failed process
 		key := "process_id:" + strconv.Itoa(int(process.ID))
 		_, ok := pendingTask.LoadOrStore(key, process.ID)
 		if ok {
-			// start process already running
-			continue
+			return
 		}
 
-		// just check failed process
-		go func(p model.Process) {
-			_ = restartProcess(p)
-			pendingTask.Delete(key)
-		}(process)
+		go func(p model.Process, key string) {
+			var cur model.Process
+			defer func() {
+				pendingTask.Delete(key)
+			}()
+			err = app.Db().First(&cur, p.ID).Error
+			if err != nil {
+				return
+			}
+
+			if cur.Status != model.StatusRunning && cur.Status != model.StatusFailed {
+				return
+			}
+
+			// 启动大于5秒后的进程才进行检查
+			if time.Since(cur.UpdatedAt).Seconds() <= 5 {
+				return
+			}
+
+			err = restartProcess(p)
+			if err != nil {
+				app.Log.Error(err)
+			}
+		}(process, key)
 	}
 }
 
@@ -71,25 +83,24 @@ func checkFork(process model.Process) bool {
 		if newPid != 0 && newPid != process.Pid {
 			process.Pid = newPid
 			process.Status = model.StatusRunning
-			if app.Db().Save(&process).Error != nil {
-				return false
-			}
-
-			return true
+			return app.Db().Save(&process).Error == nil
 		}
 	}
 
 	return false
 }
 
 func restartProcess(p model.Process) error {
+	//fmt.Printf("Monitor: try get process (%d) status \n", p.Pid)
 	_, err := os.Stat(fmt.Sprintf("/proc/%d/status", p.Pid))
 	if err == nil { // process already running
+		//fmt.Printf("Monitor: process (%d) already running \n", p.Pid)
 		return nil
 	}
 
 	// proc status file not exit
 	if os.IsNotExist(err) && (p.Status == model.StatusRunning || p.Status == model.StatusFailed) {
+		//fmt.Printf("Monitor: process not exist %s \n", p.Status)
 		if checkFork(p) {
 			return nil
 		}
@@ -103,6 +114,7 @@ func restartProcess(p model.Process) error {
 			return nil
 		}
 
+		//fmt.Printf("try to restart process %d \n", p.Pid)
 		_, err := process2.TryStart(p)
 		if err != nil {
 			return err

app/god/proc/bpf.go

@@ -0,0 +1,41 @@
+package proc
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/mdlayher/netlink/nlenc"
+	"golang.org/x/net/bpf"
+)
+
+func loadBPF(filters []EventType) ([]bpf.RawInstruction, error) {
+
+	fmt.Printf("Filter List: %#v\n", filters)
+	var totals uint32
+	for _, toFilter := range filters {
+		totals = totals | uint32(toFilter)
+	}
+
+	inst, err := bpf.Assemble([]bpf.Instruction{
+		bpf.LoadAbsolute{Off: (16 + 20), Size: 4},
+		bpf.JumpIf{Cond: bpf.JumpBitsNotSet, Val: swap(totals), SkipTrue: 1},
+		bpf.RetConstant{Val: 4096},
+		bpf.RetConstant{Val: 0},
+	})
+
+	return inst, err
+}
+
+//if needed, change the endian-ness before we file the bitmask off to bpf
+func swap(val uint32) uint32 {
+
+	if nlenc.NativeEndian() == binary.LittleEndian {
+		return (val << 24) |
+			((val << 8) & 0x00ff0000) |
+			((val >> 8) & 0x0000ff00) |
+			((val >> 24) & 0x000000ff)
+	}
+
+	return val
+
+}