feat: Add json-output params to watch command (#235)

* feat: Add json-output params to watch command * docs: modify some field explain * docs: modify column name
hengyoush · Jan 2, 2025 · 1c0dd72 · 1c0dd72
1 parent f07909b
commit 1c0dd72
Show file tree

Hide file tree

Showing 10 changed files with 331 additions and 4 deletions.
diff --git a/agent/analysis/common/json.go b/agent/analysis/common/json.go
@@ -0,0 +1,65 @@
+package common
+
+import (
+	"encoding/json"
+	"kyanos/bpf"
+	"kyanos/common"
+	"time"
+)
+
+type annotatedRecordAlias struct {
+	StartTime                  string               `json:"start_time"`
+	EndTime                    string               `json:"end_time"`
+	Protocol                   string               `json:"protocol"`
+	Side                       string               `json:"side"`
+	LocalAddr                  string               `json:"local_addr"`
+	LocalPort                  uint16               `json:"local_port"`
+	RemoteAddr                 string               `json:"remote_addr"`
+	RemotePort                 uint16               `json:"remote_port"`
+	Pid                        uint32               `json:"pid"`
+	IsSsl                      bool                 `json:"is_ssl"`
+	TotalDuration              float64              `json:"total_duration_ms"`
+	BlackBoxDuration           float64              `json:"black_box_duration_ms"`
+	ReadSocketDuration         float64              `json:"read_socket_duration_ms"`
+	CopyToSocketBufferDuration float64              `json:"copy_to_socket_buffer_duration_ms"`
+	ReqSize                    int                  `json:"req_size_bytes"`
+	RespSize                   int                  `json:"resp_size_bytes"`
+	ReqPlainTextSize           int                  `json:"req_plain_text_size_bytes"`
+	RespPlainTextSize          int                  `json:"resp_plain_text_size_bytes"`
+	Request                    string               `json:"request"`
+	Response                   string               `json:"response"`
+	ReqSyscallEventDetails     []SyscallEventDetail `json:"req_syscall_events"`
+	RespSyscallEventDetails    []SyscallEventDetail `json:"resp_syscall_events"`
+	ReqNicEventDetails         []NicEventDetail     `json:"req_nic_events"`
+	RespNicEventDetails        []NicEventDetail     `json:"resp_nic_events"`
+}
+
+// MarshalJSON implements custom JSON marshaling for AnnotatedRecord
+func (r *AnnotatedRecord) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&annotatedRecordAlias{
+		StartTime:                  time.Unix(0, int64(r.StartTs)).Format(time.RFC3339Nano),
+		EndTime:                    time.Unix(0, int64(r.EndTs)).Format(time.RFC3339Nano),
+		Protocol:                   bpf.ProtocolNamesMap[bpf.AgentTrafficProtocolT(r.ConnDesc.Protocol)],
+		Side:                       r.ConnDesc.Side.String(),
+		LocalAddr:                  r.ConnDesc.LocalAddr.String(),
+		LocalPort:                  uint16(r.ConnDesc.LocalPort),
+		RemoteAddr:                 r.ConnDesc.RemoteAddr.String(),
+		RemotePort:                 uint16(r.ConnDesc.RemotePort),
+		Pid:                        r.ConnDesc.Pid,
+		IsSsl:                      r.ConnDesc.IsSsl,
+		TotalDuration:              r.GetTotalDurationMills(),
+		BlackBoxDuration:           r.GetBlackBoxDurationMills(),
+		ReadSocketDuration:         r.GetReadFromSocketBufferDurationMills(),
+		CopyToSocketBufferDuration: common.NanoToMills(int32(r.CopyToSocketBufferDuration)),
+		ReqSize:                    r.ReqSize,
+		RespSize:                   r.RespSize,
+		ReqPlainTextSize:           r.ReqPlainTextSize,
+		RespPlainTextSize:          r.RespPlainTextSize,
+		Request:                    r.Req.FormatToString(),
+		Response:                   r.Resp.FormatToString(),
+		ReqSyscallEventDetails:     r.ReqSyscallEventDetails,
+		RespSyscallEventDetails:    r.RespSyscallEventDetails,
+		ReqNicEventDetails:         r.ReqNicEventDetails,
+		RespNicEventDetails:        r.RespNicEventDetails,
+	})
+}
diff --git a/agent/render/watch/option.go b/agent/render/watch/option.go
@@ -1,12 +1,15 @@
 package watch
 
-import "strings"
+import (
+	"strings"
+)
 
 type WatchOptions struct {
 	WideOutput                   bool
 	StaticRecord                 bool
 	Opts                         string
 	DebugOutput                  bool
+	JsonOutput                   string
 	MaxRecordContentDisplayBytes int
 	MaxRecords                   int
 }

diff --git a/agent/render/watch/watch_render.go b/agent/render/watch/watch_render.go
@@ -3,6 +3,7 @@ package watch
 import (
 	"cmp"
 	"context"
+	"encoding/json"
 	"fmt"
 	"kyanos/agent/analysis/common"
 	"kyanos/agent/protocol"
@@ -565,6 +566,36 @@ func RunWatchRender(ctx context.Context, ch chan *common.AnnotatedRecord, option
 				}))
 			}
 		}
+	} else if options.JsonOutput != "" {
+		var jsonFile *os.File
+		var err error
+		if options.JsonOutput != "stdout" {
+			jsonFile, err = os.OpenFile(options.JsonOutput, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+			if err != nil {
+				c.BPFEventLog.Errorln("Failed to open JSON output file:", err)
+				return
+			}
+			defer jsonFile.Close()
+		}
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case r := <-ch:
+				jsonData, err := json.Marshal(r)
+				if err != nil {
+					c.BPFEventLog.Errorln("Failed to marshal record to JSON:", err)
+					continue
+				}
+				if jsonFile != nil {
+					if _, err := jsonFile.Write(append(jsonData, '\n')); err != nil {
+						c.BPFEventLog.Errorln("Failed to write JSON to file:", err)
+					}
+				} else {
+					fmt.Println(string(jsonData))
+				}
+			}
+		}
 	} else {
 		c.SetLogToFile()
 		records := &[]*common.AnnotatedRecord{}
@@ -592,7 +623,6 @@ func RunWatchRender(ctx context.Context, ch chan *common.AnnotatedRecord, option
 			os.Exit(1)
 		}
 	}
-
 }
 
 func (m *model) SortBy() rc.SortBy {

diff --git a/cmd/watch.go b/cmd/watch.go
@@ -44,6 +44,7 @@ func init() {
 	watchCmd.PersistentFlags().Int64("resp-size", 0, "Filter based on response bytes size")
 	watchCmd.PersistentFlags().IntVar(&maxRecords, "max-records", 100, "Limit the max number of table records")
 	watchCmd.PersistentFlags().BoolVar(&options.WatchOptions.DebugOutput, "debug-output", false, "Print output to console instead display ui")
+	watchCmd.PersistentFlags().StringVar(&options.WatchOptions.JsonOutput, "json-output", "", "Output in JSON format. Use 'stdout' to print to terminal, or provide a file path to write to a file")
 	watchCmd.PersistentFlags().StringVar(&SidePar, "side", "all", "Filter based on connection side. can be: server | client")
 	watchCmd.PersistentFlags().StringVarP(&options.WatchOptions.Opts, "output", "o", "", "Can be `wide`")
 	watchCmd.PersistentFlags().IntVar(&options.WatchOptions.MaxRecordContentDisplayBytes, "max-print-bytes", 1024, "Control how may bytes of record's req/resp can be printed, \n exceeded part are truncated")

diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts
@@ -36,7 +36,14 @@ export default defineConfig({
         ]
       },
       {
-        text: "Development",
+        text: 'Reference',
+        items: [
+
+          { text: 'JSON Output Format', link: './json-output' },
+        ]
+      },
+      {
+        text: 'Development',
         items: [
           { text: "How to build", link: "./how-to-build" },
           {

diff --git a/docs/cn/json-output.md b/docs/cn/json-output.md
@@ -0,0 +1,90 @@
+# JSON 输出格式
+
+本文档描述了使用 kyanos 的 `--json-output` 参数时的 JSON 输出格式。
+
+## 使用方法
+
+使用 `--json-output` 参数输出 JSON 格式数据，可以指定以下值：
+
+```bash
+# 输出到终端
+kyanos watch --json-output=stdout
+
+# 输出到文件
+kyanos watch --json-output=/path/to/custom.json
+```
+
+## 输出格式
+
+每个请求-响应对都表示为一个 JSON 对象，包含以下字段：
+
+### 时间信息
+| 字段 | 类型 | 描述 |
+|-------|------|-------------|
+| `start_time` | string | 请求开始时间，RFC3339Nano 格式 |
+| `end_time` | string | 请求结束时间，RFC3339Nano 格式 |
+| `total_duration_ms` | number | 请求-响应总耗时，单位毫秒 |
+| `black_box_duration_ms` | number | 对于客户端：请求离开和响应到达网络接口之间的持续时间<br>对于服务器端：请求到达进程和响应离开进程之间的持续时间 |
+| `read_socket_duration_ms` | number | 从 socket 缓冲区读取数据的耗时 |
+| `copy_to_socket_buffer_duration_ms` | number | 复制数据到 socket 缓冲区的耗时 |
+
+### 连接信息
+| 字段 | 类型 | 描述 |
+|-------|------|-------------|
+| `protocol` | string | 协议名称（如 "HTTP"、"Redis"、"MySQL"） |
+| `side` | string | 连接的角色（客户端或服务端） |
+| `local_addr` | string | 本地 IP 地址 |
+| `local_port` | number | 本地端口号 |
+| `remote_addr` | string | 远程 IP 地址 |
+| `remote_port` | number | 远程端口号 |
+| `pid` | number | 进程 ID |
+| `is_ssl` | boolean | 是否是 SSL/TLS 加密连接 |
+
+### 内容信息
+| 字段 | 类型 | 描述 |
+|-------|------|-------------|
+| `req_size_bytes` | number | 请求总大小，单位字节 |
+| `resp_size_bytes` | number | 响应总大小，单位字节 |
+| `req_plain_text_size_bytes` | number | 请求明文大小，单位字节 |
+| `resp_plain_text_size_bytes` | number | 响应明文大小，单位字节 |
+| `request` | string | 格式化后的请求内容 |
+| `response` | string | 格式化后的响应内容 |
+
+### 事件详情
+| 字段 | 类型 | 描述 |
+|-------|------|-------------|
+| `req_syscall_events` | array | 请求相关的系统调用事件 |
+| `resp_syscall_events` | array | 响应相关的系统调用事件 |
+| `req_nic_events` | array | 请求相关的网卡事件 |
+| `resp_nic_events` | array | 响应相关的网卡事件 |
+
+## 示例
+
+```json
+{
+  "start_time": "2024-01-01T12:00:00.123456789Z",
+  "end_time": "2024-01-01T12:00:00.234567890Z",
+  "protocol": "HTTP",
+  "side": "client",
+  "local_addr": "127.0.0.1",
+  "local_port": 54321,
+  "remote_addr": "192.168.1.1",
+  "remote_port": 80,
+  "pid": 12345,
+  "is_ssl": false,
+  "total_duration_ms": 111.111111,
+  "black_box_duration_ms": 50.505050,
+  "read_socket_duration_ms": 30.303030,
+  "copy_to_socket_buffer_duration_ms": 20.202020,
+  "req_size_bytes": 256,
+  "resp_size_bytes": 1024,
+  "req_plain_text_size_bytes": 256,
+  "resp_plain_text_size_bytes": 1024,
+  "request": "GET /api/v1/users HTTP/1.1\r\nHost: example.com\r\n\r\n",
+  "response": "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"status\":\"success\"}",
+  "req_syscall_events": [...],
+  "resp_syscall_events": [...],
+  "req_nic_events": [...],
+  "resp_nic_events": [...]
+}
+``` 
diff --git a/docs/cn/watch.md b/docs/cn/watch.md
@@ -55,6 +55,27 @@ Net/Internal
 **请求响应的具体内容**，分为 Request 和 Response 两部分，超过 1024 字节会截断展示（通过
 `--max-print-bytes` 选项可以调整这个限制）。
 
+## JSON 输出
+
+如果你需要以编程方式处理采集到的数据，可以使用 `--json-output` 参数将结果输出为 JSON 格式：
+
+```bash
+# 输出到终端
+kyanos watch --json-output=stdout
+
+# 输出到文件
+kyanos watch --json-output=/path/to/custom.json
+```
+
+JSON 输出中包含每个请求-响应对的详细信息，包括：
+- 请求和响应的时间戳
+- 连接详情（地址和端口）
+- 协议特定信息
+- 详细的耗时指标
+- 请求和响应内容
+
+完整的 JSON 输出格式规范，请参考 [JSON 输出格式](./json-output.md) 文档。
+
 ## 如何发现你感兴趣的请求响应 {#how-to-filter}
 
 默认 kyanos 会抓取所有它目前支持协议的请求响应，在很多场景下，我们需要更加精确的过滤，比如想要发送给某个远程端口的请求，抑或是某个进程或者容器的关联的请求，又或者是某个 Redis 命令或者 HTTP 路径相关的请求。下面介绍如何使用 kyanos 的各种选项找到我们感兴趣的请求响应。

diff --git a/docs/json-output.md b/docs/json-output.md
@@ -0,0 +1,90 @@
+# JSON Output Format
+
+This document describes the JSON output format when using kyanos with the `--json-output` flag.
+
+## Usage
+
+To output data in JSON format, use the `--json-output` flag with one of these values:
+
+```bash
+# Output to terminal
+kyanos watch --json-output=stdout
+
+# Output to a file
+kyanos watch --json-output=/path/to/custom.json
+```
+
+## Output Format
+
+Each request-response pair is represented as a JSON object with the following fields:
+
+### Timing Information
+| Field | Type | Description |
+|-------|------|-------------|
+| `start_time` | string | Start time of the request in RFC3339Nano format |
+| `end_time` | string | End time of the request in RFC3339Nano format |
+| `total_duration_ms` | number | Total duration of the request-response in milliseconds |
+| `black_box_duration_ms` | number | For client side: Duration between request leaving and response arriving at the network interface.<br> For server side: Duration between request arriving at process and response leaving process. |
+| `read_socket_duration_ms` | number | Time spent reading from socket buffer |
+| `copy_to_socket_buffer_duration_ms` | number | Time spent copying data to socket buffer |
+
+### Connection Information
+| Field | Type | Description |
+|-------|------|-------------|
+| `protocol` | string | Protocol name (e.g., "HTTP", "Redis", "MySQL") |
+| `side` | string | Whether this is a client or server side connection |
+| `local_addr` | string | Local IP address |
+| `local_port` | number | Local port number |
+| `remote_addr` | string | Remote IP address |
+| `remote_port` | number | Remote port number |
+| `pid` | number | Process ID |
+| `is_ssl` | boolean | Whether the connection is SSL/TLS encrypted |
+
+### Content Information
+| Field | Type | Description |
+|-------|------|-------------|
+| `req_size_bytes` | number | Total size of request in bytes |
+| `resp_size_bytes` | number | Total size of response in bytes |
+| `req_plain_text_size_bytes` | number | Size of request plain text in bytes |
+| `resp_plain_text_size_bytes` | number | Size of response plain text in bytes |
+| `request` | string | Formatted request content |
+| `response` | string | Formatted response content |
+
+### Event Details
+| Field | Type | Description |
+|-------|------|-------------|
+| `req_syscall_events` | array | Syscall events related to the request |
+| `resp_syscall_events` | array | Syscall events related to the response |
+| `req_nic_events` | array | Network interface events related to the request |
+| `resp_nic_events` | array | Network interface events related to the response |
+
+## Example
+
+```json
+{
+  "start_time": "2024-01-01T12:00:00.123456789Z",
+  "end_time": "2024-01-01T12:00:00.234567890Z",
+  "protocol": "HTTP",
+  "side": "client",
+  "local_addr": "127.0.0.1",
+  "local_port": 54321,
+  "remote_addr": "192.168.1.1",
+  "remote_port": 80,
+  "pid": 12345,
+  "is_ssl": false,
+  "total_duration_ms": 111.111111,
+  "black_box_duration_ms": 50.505050,
+  "read_socket_duration_ms": 30.303030,
+  "copy_to_socket_buffer_duration_ms": 20.202020,
+  "req_size_bytes": 256,
+  "resp_size_bytes": 1024,
+  "req_plain_text_size_bytes": 256,
+  "resp_plain_text_size_bytes": 1024,
+  "request": "GET /api/v1/users HTTP/1.1\r\nHost: example.com\r\n\r\n",
+  "response": "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"status\":\"success\"}",
+  "req_syscall_events": [...],
+  "resp_syscall_events": [...],
+  "req_nic_events": [...],
+  "resp_nic_events": [...]
+}
+``` 
diff --git a/docs/watch.md b/docs/watch.md
@@ -66,6 +66,27 @@ The second section contains the **request and response content**, split into
 Request and Response parts. Content exceeding 1024 bytes is truncated, but you
 can adjust this limit using the `--max-print-bytes` option.
 
+## JSON Output
+
+If you need to process the captured data programmatically, you can use the `--json-output` flag to output the results in JSON format:
+
+```bash
+# Output to terminal
+kyanos watch --json-output=stdout
+
+# Output to a file
+kyanos watch --json-output=/path/to/custom.json
+```
+
+The JSON output will contain detailed information for each request-response pair including:
+- Timestamps for request and response
+- Connection details (addresses and ports)
+- Protocol-specific information
+- Detailed latency metrics
+- Request and response content
+
+For the complete JSON output format specification, please refer to the [JSON Output Format](./json-output.md) documentation.
+
 ## How to Filter Requests and Responses ? {#how-to-filter}
 
 By default, `kyanos` captures all traffic for the protocols it currently