From fe8f06b42c973b14a8a697220c66b5b5594c95d9 Mon Sep 17 00:00:00 2001 From: legendlliu Date: Fri, 31 May 2024 13:10:18 +0800 Subject: [PATCH 01/16] adapted to lib Signed-off-by: legendlliu --- eBPF_Supermarket/Stack_Analyser/Makefile | 11 +++++++---- eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap | 1 - 2 files changed, 7 insertions(+), 5 deletions(-) delete mode 160000 eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap diff --git a/eBPF_Supermarket/Stack_Analyser/Makefile b/eBPF_Supermarket/Stack_Analyser/Makefile index 37cbf813e..61751466c 100644 --- a/eBPF_Supermarket/Stack_Analyser/Makefile +++ b/eBPF_Supermarket/Stack_Analyser/Makefile @@ -19,8 +19,10 @@ OUTPUT := .output BPF_SKEL := bpf_skel CLANG ?= clang -LIBBPF_SRC := $(abspath libbpf-bootstrap/libbpf/src) -BPFTOOL_SRC := $(abspath libbpf-bootstrap/bpftool/src) +CXX := clang +LIBBPF_ROOT := $(abspath ../lib/libbpf) +LIBBPF_SRC := $(LIBBPF_ROOT)/src +BPFTOOL_SRC := $(abspath ../lib/bpftool/src) LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a) BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool) BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool @@ -31,11 +33,12 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/mips.*/mips/' \ | sed 's/riscv64/riscv/' \ | sed 's/loongarch64/loongarch/') -VMLINUX := libbpf-bootstrap/vmlinux/$(ARCH)/vmlinux.h + +VMLINUX := $(shell ls ../lib/vmlinux.h || bpftool btf dump file /sys/kernel/btf/vmlinux format c > ../lib/vmlinux.h && ls ../lib/vmlinux.h) # Use our own libbpf API headers and Linux UAPI headers distributed with # libbpf to avoid dependency on system-wide headers, which could be missing or # outdated -INCLUDES := -I./include -I./$(OUTPUT) -I./$(BPF_SKEL) -I./libbpf-bootstrap/libbpf/include/uapi -I$(dir $(VMLINUX)) +INCLUDES := -I./include -I./$(OUTPUT) -I./$(BPF_SKEL) -I$(LIBBPF_ROOT)/include/uapi -I$(dir $(VMLINUX)) CFLAGS := -Og -Wall ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap b/eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap deleted file mode 160000 index b0c8234df..000000000 --- a/eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b0c8234dfb8f31eb12c99b26bb2bec96eb76aff3 From f293815a3f415e68102a5c155347499b1bb1479e Mon Sep 17 00:00:00 2001 From: legendlliu Date: Fri, 31 May 2024 14:17:50 +0800 Subject: [PATCH 02/16] update docs Signed-off-by: legendlliu --- eBPF_Supermarket/Stack_Analyser/README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/README.md b/eBPF_Supermarket/Stack_Analyser/README.md index efcea400e..2df17eb1b 100644 --- a/eBPF_Supermarket/Stack_Analyser/README.md +++ b/eBPF_Supermarket/Stack_Analyser/README.md @@ -61,21 +61,34 @@ Stack_Analyzer是一个基于eBPF的按照指定时间间隔（默认为5s）来 ## 编译要求 -Ubuntu下需要安装一下依赖，其他发行版类似 +初始化并更新libbpf和bpftool的代码仓库： ```shell -$ git submodule update --init --recursive -$ apt install clang libelf1 libelf-dev zlib1g-dev +git submodule update --init --recursive ../lib/* ``` +需要安装一下依赖： -g++-10以上，clang-12以上 +Ubuntu下 + +```shell +sudo apt update +sudo apt install -y clang libelf1 libelf-dev zlib1g-dev bpftool +``` + +CentOS下 + +```shell +sudo dnf install clang elfutils-libelf elfutils-libelf-devel zlib-devel bpftool +``` + +clang-12以上 # 使用方法 ## 工具编译 ```shell -$ make +$ make -j$(nproc) ``` ## 命令使用方法 From ee2044bdaefb3d7285c156c3bbdaeec917458d6e Mon Sep 17 00:00:00 2001 From: legendlliu Date: Fri, 31 May 2024 14:43:27 +0800 Subject: [PATCH 03/16] remove child submodule Signed-off-by: legendlliu --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 122b510a0..a2d3cd09c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,9 +46,6 @@ [submodule "eBPF_Supermarket/CPU_Subsystem/libbpf"] path = eBPF_Supermarket/CPU_Subsystem/libbpf url = https://github.com/libbpf/libbpf.git -[submodule "eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap"] - path = eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap - url = https://github.com/libbpf/libbpf-bootstrap.git [submodule "eBPF_Supermarket/Network_Subsystem/net_manager/lib/libbpf"] path = eBPF_Supermarket/Network_Subsystem/net_manager/lib/libbpf url = https://github.com/libbpf/libbpf.git From feb3b758f47a124e4fe040f7ee42978f76c65ed2 Mon Sep 17 00:00:00 2001 From: legendlliu Date: Fri, 31 May 2024 14:48:54 +0800 Subject: [PATCH 04/16] update action Signed-off-by: legendlliu --- .github/workflows/ebpf_stack_analyser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index ef5a99d5b..d5a37e6a8 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -23,7 +23,7 @@ jobs: - name: Install native lib dependencies run: | - git submodule update --init --recursive eBPF_Supermarket/Stack_Analyser/ MagicEyes/ + git submodule update --init --recursive eBPF_Supermarket/lib/ MagicEyes/ sudo apt install clang libelf1 libelf-dev zlib1g-dev - name: Compile test examples From 02762cb11b573a44d2bc5e1cb7e092f5e604df87 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Sun, 2 Jun 2024 14:11:58 +0000 Subject: [PATCH 05/16] update Makefile Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/Makefile b/eBPF_Supermarket/Stack_Analyser/Makefile index 61751466c..0b17426e8 100644 --- a/eBPF_Supermarket/Stack_Analyser/Makefile +++ b/eBPF_Supermarket/Stack_Analyser/Makefile @@ -20,9 +20,10 @@ OUTPUT := .output BPF_SKEL := bpf_skel CLANG ?= clang CXX := clang -LIBBPF_ROOT := $(abspath ../lib/libbpf) +LIB := ../lib +LIBBPF_ROOT := $(abspath $(LIB)/libbpf) LIBBPF_SRC := $(LIBBPF_ROOT)/src -BPFTOOL_SRC := $(abspath ../lib/bpftool/src) +BPFTOOL_SRC := $(abspath $(LIB)/bpftool/src) LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a) BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool) BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool @@ -34,7 +35,7 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/riscv64/riscv/' \ | sed 's/loongarch64/loongarch/') -VMLINUX := $(shell ls ../lib/vmlinux.h || bpftool btf dump file /sys/kernel/btf/vmlinux format c > ../lib/vmlinux.h && ls ../lib/vmlinux.h) +VMLINUX := $(LIB)/vmlinux.h # Use our own libbpf API headers and Linux UAPI headers distributed with # libbpf to avoid dependency on system-wide headers, which could be missing or # outdated @@ -88,6 +89,16 @@ clean: $(call msg,CLEAN) $(Q)rm -rf $(OUTPUT) $(TARGETS) $(BPF_SKEL) +init: + $(call msg,INIT,$(LIB)) + $(Q)git submodule update --init --recursive ../lib/ + +$(LIBBPF_SRC) $(BPFTOOL_SRC): init + +$(VMLINUX): + $(call msg,BTFDUMP,$@) + $(Q)bpftool btf dump file /sys/kernel/btf/vmlinux format c > $@ + $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT) $(BPF_SKEL): $(call msg,MKDIR,$@) $(Q)mkdir -p $@ From c67c07083d3526b9cff0f2dac651c45bdcbbf527 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Sun, 2 Jun 2024 14:14:18 +0000 Subject: [PATCH 06/16] adapt mem mod to local vmlinux Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c index 292066b21..24f2a14e7 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c @@ -484,7 +484,7 @@ int memleak__mm_page_free(struct trace_event_raw_mm_page_free *ctx) SEC("tracepoint/percpu/percpu_alloc_percpu") int memleak__percpu_alloc_percpu(struct trace_event_raw_percpu_alloc_percpu *ctx) { - gen_alloc_enter(ctx->bytes_alloc); + gen_alloc_enter(ctx->size); return gen_alloc_exit2(ctx, (u64)(ctx->ptr)); } From c029b1e3d009b11bf31f010f88a59f0184ba8f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=99=97=E5=8D=9A?= Date: Thu, 6 Jun 2024 15:16:32 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E4=B8=BAcpuwatcher=E5=A2=9E=E5=8A=A0ewma?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CPU_Subsystem/cpu_watcher/cpu_watcher.c | 31 +++++++++++++-- .../cpu_watcher/include/cpu_watcher_helper.h | 38 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index 8e3f85370..b1274577f 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -29,6 +29,7 @@ #include #include #include "cpu_watcher.h" +#include "cpu_watcher_helper.h" #include "sar.skel.h" #include "cs_delay.skel.h" #include "sc_delay.skel.h" @@ -64,6 +65,8 @@ static struct env { bool SCHEDULE_DELAY; bool MQ_DELAY; int freq; + bool EWMA; + int cycle; } env = { .time = 0, .period = 1, @@ -75,7 +78,9 @@ static struct env { .PREEMPT = false, .SCHEDULE_DELAY = false, .MQ_DELAY = false, - .freq = 99 + .freq = 99, + .EWMA = false, + .cycle = 0, }; @@ -119,6 +124,8 @@ static const struct argp_option opts[] = { {"preempt_time", 'p', 0,0,"print preempt_time (the data of preempt_schedule)"}, {"schedule_delay", 'd', 0,0,"print schedule_delay (the data of cpu)"}, {"mq_delay", 'm', 0,0,"print mq_delay(the data of proc)"}, + {"ewma", 'e', 0,0,"dynamic filte the data"}, + {"cycle", 'T', "CYCLE",0,"Periods of the ewma"}, { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, {0}, }; @@ -153,6 +160,12 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'm': env.MQ_DELAY = true; break; + case 'e': + env.EWMA = true; + break; + case 'T': + env.cycle = strtol(arg, NULL, 10);; + break; case 'h': argp_state_help(state, stderr, ARGP_HELP_STD_HELP); break; @@ -458,13 +471,23 @@ static void histogram() printf("per_len = %d\n",per_len); } - +struct ewma_info ewma_syscall_delay = {}; static int syscall_delay_print(void *ctx, void *data,unsigned long data_sz) { - const struct syscall_events *e = data; - printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", + if(env.EWMA==0){ + printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", e->pid,e->comm,e->syscall_id,e->delay); + } + else{ + ewma_syscall_delay.cycle = env.cycle; + if(dynamic_filter(&ewma_syscall_delay,e->delay)){ + printf("yes!!!!\n"); + printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", + e->pid,e->comm,e->syscall_id,e->delay); + } + } + return 0; } diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h new file mode 100644 index 000000000..0d8ab4047 --- /dev/null +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h @@ -0,0 +1,38 @@ +#include +/*----------------------------------------------*/ +/* ewma算法 */ +/*----------------------------------------------*/ +//滑动窗口周期，用于计算alpha +#define CYCLE 10 +//阈值容错空间； +#define TOLERANCE 1.0 +struct ewma_info{ + double previousEWMA; + int count; + int cycle;//cycle是滑动窗口周期大小 +}; + +double calculateEWMA(double previousEWMA, double dataPoint, double alpha) { + return alpha * dataPoint + (1 - alpha) * previousEWMA;//当前时间点的ewma +} + +bool dynamic_filter(struct ewma_info *ewma_syscall_delay, double dataPoint) { + double alpha,ewma,threshold;; + if(ewma_syscall_delay->cycle==0) alpha = 2.0 /(CYCLE + 1); // 计算 alpha + else alpha = 2.0 /(ewma_syscall_delay->cycle + 1); + + if(ewma_syscall_delay->previousEWMA == 0) {//初始化ewma算法，则赋值previousEWMA = dataPoint 并打印 + ewma_syscall_delay->previousEWMA = dataPoint; + return 1; + } + if(ewma_syscall_delay->count <30){ + ewma_syscall_delay->previousEWMA = calculateEWMA(ewma_syscall_delay->previousEWMA,dataPoint,alpha);//计算 + return 1; + } + else{ + ewma_syscall_delay->previousEWMA = calculateEWMA(ewma_syscall_delay->previousEWMA,dataPoint,alpha);//计算 + threshold = ewma_syscall_delay->previousEWMA * TOLERANCE; + if(dataPoint >= threshold) return 1; + } + return 0; +} \ No newline at end of file From 8ed97a1f034d5e505653af836206db998abd0012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=99=97=E5=8D=9A?= Date: Thu, 6 Jun 2024 15:43:45 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E4=B8=BAcpywatcher=E5=A2=9E=E5=8A=A0ewma?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CPU_Subsystem/cpu_watcher/cpu_watcher.c | 281 ++++++++++-------- 1 file changed, 162 insertions(+), 119 deletions(-) diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index e7ff22c3a..8605fc18a 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -39,7 +39,7 @@ typedef long long unsigned int u64; typedef unsigned int u32; -#define MAX_BUF 512 + struct list_head { struct list_head *next; @@ -55,35 +55,40 @@ struct msg_msg { static struct env { int time; - int period; - bool percent; + int period; + bool percent; bool enable_proc; bool SAR; bool CS_DELAY; bool SYSCALL_DELAY; + bool MIN_US_SET; + int MIN_US; bool PREEMPT; bool SCHEDULE_DELAY; - bool MQ_DELAY; + bool MQ_DELAY; int freq; - bool EWMA; - int cycle; + bool EWMA; + int cycle; } env = { .time = 0, - .period = 1, - .percent = false, + .period = 1, + .percent = false, .enable_proc = false, .SAR = false, .CS_DELAY = false, .SYSCALL_DELAY = false, + .MIN_US_SET = false, + .MIN_US = 10000, .PREEMPT = false, .SCHEDULE_DELAY = false, - .MQ_DELAY = false, + .MQ_DELAY = false, .freq = 99, - .EWMA = false, - .cycle = 0, + .EWMA = false, + .cycle = 0, }; + struct cs_delay_bpf *cs_skel; struct sar_bpf *sar_skel; struct sc_delay_bpf *sc_skel; @@ -113,67 +118,82 @@ int sum_preemptTime = 0 ; int preempt_start_print = 0 ; /*设置传参*/ -const char argp_program_doc[] ="cpu wacher is in use ....\n"; +const char argp_program_doc[] = "cpu watcher is in use ....\n"; static const struct argp_option opts[] = { - { "time", 't', "TIME-SEC", 0, "Max Running Time(0 for infinite)" }, - { "period", 'i', "INTERVAL", 0, "Period interval in seconds" }, - {"percent",'P',0,0,"format data as percentages"}, - {"libbpf_sar", 's', 0,0,"print sar_info (the data of cpu)"}, - {"cs_delay", 'c', 0,0,"print cs_delay (the data of cpu)"}, - {"syscall_delay", 'S', 0,0,"print syscall_delay (the data of syscall)"}, - {"preempt_time", 'p', 0,0,"print preempt_time (the data of preempt_schedule)"}, - {"schedule_delay", 'd', 0,0,"print schedule_delay (the data of cpu)"}, - {"mq_delay", 'm', 0,0,"print mq_delay(the data of proc)"}, - {"ewma", 'e', 0,0,"dynamic filte the data"}, - {"cycle", 'T', "CYCLE",0,"Periods of the ewma"}, - { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, - {0}, + { "time", 't', "TIME-SEC", 0, "Max Running Time(0 for infinite)" }, + { "period", 'i', "INTERVAL", 0, "Period interval in seconds" }, + {"percent", 'P', 0, 0, "Format data as percentages" }, + {"libbpf_sar", 's', 0, 0, "Print sar_info (the data of cpu)" }, + {"cs_delay", 'c', 0, 0, "Print cs_delay (the data of cpu)" }, + {"syscall_delay", 'S', 0, 0, "Print syscall_delay (the data of syscall)" }, + {"preempt_time", 'p', 0, 0, "Print preempt_time (the data of preempt_schedule)" }, + {"schedule_delay", 'd', 0, 0, "Print schedule_delay (the data of cpu)" }, + {"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" }, + {"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" }, + {"ewma", 'E',0,0,"dynamic filte the data"}, + {"cycle", 'T',"CYCLE",0,"Periods of the ewma"}, + { NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" }, + { 0 }, }; + static error_t parse_arg(int key, char *arg, struct argp_state *state) { - switch (key) { - case 't': - env.time = strtol(arg, NULL, 10); - if(env.time) alarm(env.time); - break; - case 'i': - env.period = strtol(arg, NULL, 10); - break; - case 'P': - env.percent = true; - break; - case 's': - env.SAR = true; - break; - case 'c': - env.CS_DELAY = true; - break; - case 'S': - env.SYSCALL_DELAY = true; - break; - case 'p': - env.PREEMPT = true; - break; - case 'd': - env.SCHEDULE_DELAY = true; - break; - case 'm': - env.MQ_DELAY = true; - break; - case 'e': + switch (key) { + case 't': + env.time = strtol(arg, NULL, 10); + if (env.time) alarm(env.time); + break; + case 'i': + env.period = strtol(arg, NULL, 10); + break; + case 'P': + env.percent = true; + break; + case 's': + env.SAR = true; + break; + case 'c': + env.CS_DELAY = true; + break; + case 'S': + env.SYSCALL_DELAY = true; + break; + case 'p': + env.PREEMPT = true; + break; + case 'd': + env.SCHEDULE_DELAY = true; + break; + case 'e': + env.MIN_US_SET = true; + if (arg) { + env.MIN_US = strtol(arg, NULL, 10); + if (env.MIN_US <= 0) { + fprintf(stderr, "Invalid value for min_us: %d\n", env.MIN_US); + argp_usage(state); + } + } else { + env.MIN_US = 10000; + } + break; + case 'm': + env.MQ_DELAY = true; + break; + case 'E': env.EWMA = true; break; case 'T': - env.cycle = strtol(arg, NULL, 10);; + env.cycle = strtol(arg, NULL, 10); break; - case 'h': - argp_state_help(state, stderr, ARGP_HELP_STD_HELP); - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; + case 'h': + argp_state_help(state, stderr, ARGP_HELP_STD_HELP); + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; } + static const struct argp argp = { .options = opts, .parser = parse_arg, @@ -471,6 +491,7 @@ static void histogram() printf("per_len = %d\n",per_len); } + struct ewma_info ewma_syscall_delay = {}; static int syscall_delay_print(void *ctx, void *data,unsigned long data_sz) { @@ -482,7 +503,6 @@ static int syscall_delay_print(void *ctx, void *data,unsigned long data_sz) else{ ewma_syscall_delay.cycle = env.cycle; if(dynamic_filter(&ewma_syscall_delay,e->delay)){ - printf("yes!!!!\n"); printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", e->pid,e->comm,e->syscall_id,e->delay); } @@ -501,66 +521,82 @@ static int preempt_print(void *ctx, void *data, unsigned long data_sz) return 0; } -char* get_process_name_by_pid(int pid) { - static char buf[MAX_BUF]; - char command[MAX_BUF]; - snprintf(command, sizeof(command), "cat /proc/%d/status | grep Name", pid); - FILE* fp = popen(command, "r"); - if (fp == NULL) { - perror("popen"); - return NULL; - } - char* name = NULL; - while (fgets(buf, sizeof(buf), fp)) { - if (strncmp(buf, "Name:", 5) == 0) { - name = strdup(buf + 6); - break; - } - } - pclose(fp); - if (name != NULL) { - size_t len = strlen(name); - if (len > 0 && name[len - 1] == '\n') { - name[len - 1] = '\0'; +// 定义一个结构来存储已输出的条目 +struct output_entry { + int pid; + char comm[16]; + long long delay; +}; + +// 定义一个数组来存储已输出的条目 +struct output_entry seen_entries[MAX_ENTRIES]; +int seen_count = 0; + +// 检查条目是否已存在 +bool entry_exists(int pid, const char *comm, long long delay) { + for (int i = 0; i < seen_count; i++) { + if (seen_entries[i].pid == pid && + strcmp(seen_entries[i].comm, comm) == 0 && + seen_entries[i].delay == delay) { + return true; } } - return name; + return false; } -static int schedule_print(struct bpf_map *sys_fd) +// 添加条目到已输出的条目列表 +void add_entry(int pid, const char *comm, long long delay) { + if (seen_count < MAX_ENTRIES) { + seen_entries[seen_count].pid = pid; + strncpy(seen_entries[seen_count].comm, comm, sizeof(seen_entries[seen_count].comm)); + seen_entries[seen_count].delay = delay; + seen_count++; + } +} +static int schedule_print() { int key = 0; - struct sum_schedule info; - int err, fd = bpf_map__fd(sys_fd); - time_t now = time(NULL); - struct tm *localTime = localtime(&now); - int hour = localTime->tm_hour; - int min = localTime->tm_min; - int sec = localTime->tm_sec; - unsigned long long avg_delay; - err = bpf_map_lookup_elem(fd, &key, &info); - if (err < 0) { - fprintf(stderr, "failed to lookup infos: %d\n", err); - return -1; - } - avg_delay = info.sum_delay / info.sum_count; - if(!ifprint){ - ifprint=1; - }else{ - char* proc_name_max = get_process_name_by_pid(info.pid_max); - char* proc_name_min = get_process_name_by_pid(info.pid_min); - printf("%02d:%02d:%02d %-15lf %-15lf %10s %15lf %15s\n", - hour, min, sec, avg_delay / 1000.0, info.max_delay / 1000.0,proc_name_max,info.min_delay / 1000.0,proc_name_min); - if (proc_name_max != NULL) { - free(proc_name_max); - } - if (proc_name_min != NULL) { - free(proc_name_min); + if(env.SCHEDULE_DELAY){ + struct sum_schedule info; + int err, fd = bpf_map__fd(sd_skel->maps.sys_schedule); + time_t now = time(NULL); + struct tm *localTime = localtime(&now); + int hour = localTime->tm_hour; + int min = localTime->tm_min; + int sec = localTime->tm_sec; + unsigned long long avg_delay; + err = bpf_map_lookup_elem(fd, &key, &info); + if (err < 0) { + fprintf(stderr, "failed to lookup infos: %d\n", err); + return -1; } + avg_delay = info.sum_delay / info.sum_count; + if (!ifprint) { + ifprint=1; + }else{ + printf("%02d:%02d:%02d %-15lf %-15lf %10s %15lf %15s\n", + hour, min, sec, avg_delay / 1000.0, info.max_delay / 1000.0,info.proc_name_max,info.min_delay / 1000.0,info.proc_name_min); + } + }else if(env.MIN_US_SET){ + struct proc_schedule info; + int key = 0; + int err, fd = bpf_map__fd(sd_skel->maps.threshold_schedule); + err = bpf_map_lookup_elem(fd, &key, &info); + if (err < 0) { + fprintf(stderr, "failed to lookup infos: %d\n", err); + return -1; + } + if (info.delay / 1000 > env.MIN_US&&info.pid!=0) { // 默认输出调度延迟大于10ms的 + if (!entry_exists(info.pid, info.proc_name, info.delay / 1000)) { + printf("%-10d %-16s %15lld\n", info.pid, info.proc_name, info.delay / 1000); + add_entry(info.pid, info.proc_name, info.delay / 1000); + } + } } return 0; } + static int mq_event(void *ctx, void *data,unsigned long data_sz) { time_t now = time(NULL);// 获取当前时间 @@ -698,7 +734,7 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to create ring buffer\n"); goto sc_delay_cleanup; } - }else if(env.SCHEDULE_DELAY){ + }else if(env.SCHEDULE_DELAY||env.MIN_US_SET){ sd_skel = schedule_delay_bpf__open(); if (!sd_skel) { fprintf(stderr, "Failed to open and load BPF skeleton\n"); @@ -714,7 +750,12 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to attach BPF skeleton\n"); goto schedule_cleanup; } - printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs max_proc_name min_delay/μs min_proc_name"); + if(env.MIN_US_SET){ + printf("调度延时大于%dms的进程:\n",env.MIN_US/1000); + printf("%s\n","pid COMM schedule_delay/us"); + }else{ + printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs max_proc_name min_delay/μs min_proc_name"); + } }else if (env.SAR){ /* Load and verify BPF application */ sar_skel = sar_bpf__open(); @@ -843,8 +884,8 @@ int main(int argc, char **argv) sum_preemptTime = 0; sleep(2); } - else if (env.SCHEDULE_DELAY){ - err = schedule_print(sd_skel->maps.sys_schedule); + else if (env.SCHEDULE_DELAY||env.MIN_US_SET){ + err = schedule_print(); if (err == -EINTR) { err = 0; break; @@ -852,7 +893,9 @@ int main(int argc, char **argv) if (err < 0) { break; } - sleep(1); + if(env.SCHEDULE_DELAY){ + sleep(1); + } } else if(env.MQ_DELAY){ err = ring_buffer__poll(rb, 1000 /* timeout, s */); @@ -898,4 +941,4 @@ int main(int argc, char **argv) ring_buffer__free(rb); mq_delay_bpf__destroy(mq_skel); return err < 0 ? -err : 0; -} +} \ No newline at end of file From ffb3d9a5c07a401f95d17e92e778321523923a3e Mon Sep 17 00:00:00 2001 From: albertxu216 <145351853+albertxu216@users.noreply.github.com> Date: Thu, 6 Jun 2024 15:53:42 +0800 Subject: [PATCH 09/16] Update ebpf_cpu_watcher.yml --- .github/workflows/ebpf_cpu_watcher.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ebpf_cpu_watcher.yml b/.github/workflows/ebpf_cpu_watcher.yml index 93dbf8441..8b9f5bd32 100644 --- a/.github/workflows/ebpf_cpu_watcher.yml +++ b/.github/workflows/ebpf_cpu_watcher.yml @@ -31,3 +31,9 @@ jobs: cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/ make sudo ./cpu_watcher + + - name: Run test_cpuwatcher + run: | + cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test + make + ./test_cpuwatcher From 36f5ac5fef88b523e9be2ebe6f3f7436c791010b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=99=97=E5=8D=9A?= Date: Thu, 6 Jun 2024 15:54:34 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E4=B8=BAcpywatcher=E5=A2=9E=E5=8A=A0ewma?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cpu_watcher/bpf/schedule_delay.bpf.c | 61 ++++++++++--------- .../cpu_watcher/include/cpu_watcher.h | 10 ++- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c index a51e52459..c5e2c7d22 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c @@ -23,14 +23,14 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; #define TASK_RUNNING 0x0000 -BPF_HASH(has_scheduled,struct proc_id, bool, 10240); -BPF_HASH(enter_schedule,struct proc_id, struct schedule_event, 10240); -BPF_ARRAY(sys_schedule,int,struct sum_schedule,1); - +BPF_HASH(has_scheduled,struct proc_id, bool, 10240);//记录该进程是否调度过 +BPF_HASH(enter_schedule,struct proc_id, struct schedule_event, 10240);//记录该进程上运行队列的时间 +BPF_ARRAY(sys_schedule,int,struct sum_schedule,1);//记录整个系统的调度延迟 +BPF_ARRAY(threshold_schedule,int,struct proc_schedule,10240);//记录每个进程的调度延迟 SEC("tp_btf/sched_wakeup") int BPF_PROG(sched_wakeup, struct task_struct *p) { - pid_t pid = BPF_CORE_READ(p, pid); + pid_t pid = p->pid; int cpu = bpf_get_smp_processor_id(); struct schedule_event *schedule_event; struct proc_id id= {}; @@ -56,7 +56,7 @@ int BPF_PROG(sched_wakeup, struct task_struct *p) { SEC("tp_btf/sched_wakeup_new") int BPF_PROG(sched_wakeup_new, struct task_struct *p) { - pid_t pid = BPF_CORE_READ(p, pid); + pid_t pid = p->pid; int cpu = bpf_get_smp_processor_id(); struct proc_id id= {}; u64 current_time = bpf_ktime_get_ns(); @@ -86,17 +86,17 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s struct schedule_event *schedule_event; struct sum_schedule *sum_schedule; int key = 0; - struct proc_id next_id= {}; + struct proc_id next_id = {}; u64 delay; if (prev_state == TASK_RUNNING) { - struct proc_id prev_pd= {}; + struct proc_id prev_pd = {}; prev_pd.pid = prev_pid; if (prev_pid == 0) { prev_pd.cpu_id = prev_cpu; - } + } schedule_event = bpf_map_lookup_elem(&enter_schedule, &prev_pd); if (!schedule_event) { - struct schedule_event schedule_event2 ; + struct schedule_event schedule_event2; bool issched = false; schedule_event2.pid = prev_pid; schedule_event2.count = 1; @@ -113,44 +113,49 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s next_id.cpu_id = next_cpu; } schedule_event = bpf_map_lookup_elem(&enter_schedule, &next_id); - if (!schedule_event) return 0; + if (!schedule_event) return 0; issched = bpf_map_lookup_elem(&has_scheduled, &next_id); - if (!issched) return 0; + if (!issched) return 0; if (*issched) { schedule_event->count++; } else { *issched = true; - } + } delay = current_time - schedule_event->enter_time; + struct proc_schedule proc_schedule; + proc_schedule.pid = next_pid; + proc_schedule.delay = delay; + bpf_probe_read_kernel_str(&proc_schedule.proc_name, sizeof(proc_schedule.proc_name), next->comm); + bpf_map_update_elem(&threshold_schedule, &key, &proc_schedule, BPF_ANY); sum_schedule = bpf_map_lookup_elem(&sys_schedule, &key); if (!sum_schedule) { - struct sum_schedule sum_schedule= {}; + struct sum_schedule sum_schedule = {}; sum_schedule.sum_count++; sum_schedule.sum_delay += delay; - if (delay > sum_schedule.max_delay){ + if (delay > sum_schedule.max_delay) { sum_schedule.max_delay = delay; - if(next->pid!=0){ - sum_schedule.pid_max = next->pid; + if (next->pid != 0) { + bpf_probe_read_kernel_str(&sum_schedule.proc_name_max, sizeof(sum_schedule.proc_name_max), next->comm); } - }else if (sum_schedule.min_delay == 0 || delay < sum_schedule.min_delay) + } else if (sum_schedule.min_delay == 0 || delay < sum_schedule.min_delay) { sum_schedule.min_delay = delay; - if(next->pid!=0){ - sum_schedule.pid_min = next->pid; + if (next->pid != 0) { + bpf_probe_read_kernel_str(&sum_schedule.proc_name_min, sizeof(sum_schedule.proc_name_min), next->comm); } + } bpf_map_update_elem(&sys_schedule, &key, &sum_schedule, BPF_ANY); } else { sum_schedule->sum_count++; sum_schedule->sum_delay += delay; - if (delay > sum_schedule->max_delay){ + if (delay > sum_schedule->max_delay) { sum_schedule->max_delay = delay; - if(next->pid!=0){ - sum_schedule->pid_max = next->pid; - } - }else if (sum_schedule->min_delay == 0 || delay < sum_schedule->min_delay) + bpf_probe_read_kernel_str(&sum_schedule->proc_name_max, sizeof(sum_schedule->proc_name_max), next->comm); + } else if (sum_schedule->min_delay == 0 || delay < sum_schedule->min_delay) { sum_schedule->min_delay = delay; - if(next->pid!=0){ - sum_schedule->pid_min = next->pid; + if (next->pid != 0) { + bpf_probe_read_kernel_str(&sum_schedule->proc_name_min, sizeof(sum_schedule->proc_name_min), next->comm); } + } } return 0; } @@ -175,4 +180,4 @@ int sched_process_exit(void *ctx) { bpf_map_delete_elem(&has_scheduled, &id); } return 0; -} +} \ No newline at end of file diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h index bc67e3e5b..f1c35423f 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h @@ -134,8 +134,14 @@ struct sum_schedule { unsigned long long sum_delay; unsigned long long max_delay; unsigned long long min_delay; - int pid_max; - int pid_min; + char proc_name_max[TASK_COMM_LEN]; + char proc_name_min[TASK_COMM_LEN]; +}; + +struct proc_schedule { + int pid; + unsigned long long delay; + char proc_name[TASK_COMM_LEN]; }; /*----------------------------------------------*/ From b8e1249cbc51dba84da7bc622a82b0f7a7a134be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=99=97=E5=8D=9A?= Date: Thu, 6 Jun 2024 16:02:11 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E4=B8=BAcpywatcher=E5=A2=9E=E5=8A=A0ewma?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cpu_watcher/include/cpu_watcher_helper.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h index 0d8ab4047..409d115a5 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h @@ -1,3 +1,19 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: albert_xuu@163.com zhangxy1016304@163.com zhangziheng0525@163.com + #include /*----------------------------------------------*/ /* ewma算法 */ From 0ad80967d23c292b1a14e6ccc3accb8e1b65ba36 Mon Sep 17 00:00:00 2001 From: wynyibo Date: Thu, 6 Jun 2024 16:10:17 +0800 Subject: [PATCH 12/16] update readme add mysql information --- .../Network_Subsystem/net_watcher/README.md | 122 +++++++++---- .../net_watcher/common.bpf.h | 16 +- .../Network_Subsystem/net_watcher/mysql.bpf.h | 79 ++++++++ .../net_watcher/mysql_helper.bpf.h | 171 ++++++++++++++++++ .../net_watcher/netwatcher.bpf.c | 34 +++- .../net_watcher/netwatcher.c | 167 +++++++++++++---- .../net_watcher/netwatcher.h | 23 ++- 7 files changed, 531 insertions(+), 81 deletions(-) create mode 100644 eBPF_Supermarket/Network_Subsystem/net_watcher/mysql.bpf.h create mode 100644 eBPF_Supermarket/Network_Subsystem/net_watcher/mysql_helper.bpf.h diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/README.md b/eBPF_Supermarket/Network_Subsystem/net_watcher/README.md index 533ce30a5..355cf2f18 100644 --- a/eBPF_Supermarket/Network_Subsystem/net_watcher/README.md +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/README.md @@ -34,6 +34,8 @@ netwatcher能够追踪TCP、UDP、ICMP协议数据包从应用程序发出开始 - TCP、UDP、ICMP相关信息监测：追踪TCP、UDP、ICMP协议数据包，并实现对主机接收和发送的所有相关数据包的时延数据和流量信息 - 监测TCP连接状态信息：包括三次握手以及四次挥手的状态转变和时延数据 - 丢包事件的监控：分析导致丢包的地址以及丢包原因 +- DNS协议相关信息监控：通过截取UDP包，对DNS协议包进行解析，获取事务ID、标志字段、问题部分计数、应答记录计数、授权记录计数、附加记录计数、域名等信息 +- 主机环境下对用户态mysql的分析：uprobe实现对mysql的监测，其监测内容有进程pid、进程名、sql语句、sql语句执行时间。 #### TODO - [ ] 应用层协议的支持 @@ -72,6 +74,8 @@ netwatcher能够追踪TCP、UDP、ICMP协议数据包从应用程序发出开始 - dropreason.h ：skb_drop_reason定义77种丢包原因。 - icmp.bpf.h： icmp时延具体实现细节。 - comm.bpf.h ：辅助函数、宏、BPF映射、以及内核中使用到的结构体。 +- mysql,bpf.h : 处理mysql的具体实现逻辑。 +- mysql_helper.bpf ： mysql相关数据结构。 ## 二、快速开始 ### 2.1 安装依赖 @@ -98,13 +102,25 @@ sudo make test # 测试 ```bash Usage: netwatcher [OPTION...] Watch tcp/ip in network subsystem +Usage: netwatcher [OPTION...] +Watch tcp/ip in network subsystem + -a, --all set to trace CLOSED connection + -A, --stack set to trace of stack -d, --dport=DPORT trace this destination port only + -D, --dns set to trace dns information info include Id + 事务ID、Flags 标志字段、Qd + 问题部分计数、An 应答记录计数、Ns + 授权记录计数、Ar 附加记录计数、Qr + 域名、rx 收发包 -e, --err set to trace TCP error packets -i, --http set to trace http info -I, --icmptime set to trace layer time of icmp -k, --drop_reason trace kfree -L, --timeload analysis time load + -M, --mysql set to trace mysql information info include Pid + 进程id、Comm 进程名、Size + sql语句字节大小、Sql 语句 -n, --net_filter trace ipv4 packget filter -r, --retrans set to trace extra retrans info -s, --sport=SPORT trace this source port only @@ -114,6 +130,7 @@ Watch tcp/ip in network subsystem -u, --udp trace the udp message -x, --extra set to trace extra conn info -?, --help Give this help list + --usage Give a short usage message ``` - 参数`-d`,`-s`用于指定监控某个源端口/目的端口 @@ -149,6 +166,8 @@ Watch tcp/ip in network subsystem - 指定 `-I` 参数监控ICMP协议数据包收发过程中的时延。 - 指定 `-T` 参数将捕获到的丢包事件虚拟地址转换成函数名+偏移量形式。 - 指定 `-L` 参数监测网络协议栈数据包经过各层的时延，采用指数加权移动法对异常的时延数据进行监控并发出警告信息。 +- 指定 `-D` 参数监测DNS协议包信息。截取UDP包，对DNS协议包进行解析，获取其基本指标，包含事务ID、标志字段、问题部分计数、应答记录计数、域名等相关信息。 +- 指定 `-M` 参数监测Mysql信息。实现用户态下mysql监控，获取其sql语句及sql执行耗时，单位μs。 ### 3.1 监控连接信息 `netwatcher`会将保存在内存中的连接相关信息实时地在`data/connects.log`中更新。默认情况下，为节省资源消耗，`netwatcher`会实时删除已CLOSED的TCP连接相关信息，并只会保存每个TCP连接的基本信息。 @@ -194,30 +213,34 @@ packet{sock="0xffff9d1ecb3ba300",seq="279168002",ack="629372873",mac_time="-",ip ```c sudo ./netwatcher -t -SOCK SEQ ACK MAC_TIME IP_TIME TRAN_TIME RX HTTP -0xffffa069b1271200 3401944293 1411917682 4 15 15 0 - -0xffffa06982943600 537486036 4194537197 16 14 123 1 - -0xffffa06982943600 537486036 4194537197 16 14 130 1 - -0xffffa06982946c00 3341452281 1920160519 3 13 12 0 - +SOCK Saddr Sport Daddr Dport MAC_TIME/μs IP_TIME/μs TRAN_TIME/μs RX/direction HTTP +0xffff939118a9ad00 192.168.60.136 36236 1.1.1.1 80 2 5 11 0 - +0xffff939118a9ad00 1.1.1.1 80 192.168.60.136 36236 6 10 111 1 - +0xffff93911049b600 192.168.60.136 36244 1.1.1.1 80 6 18 38 0 - +0xffff93911049b600 1.1.1.1 80 192.168.60.136 36244 14 17 263 1 - +0xffff93911049ad00 192.168.60.136 36246 1.1.1.1 80 2 4 16 0 - ``` 指定参数`-u`，查看UDP数据包处理时延并记录于`data/udp.log`中，单位为微妙。 ```c sudo ./netwatcher -u -saddr daddr sprot dprot udp_time rx len -192.168.60.2 192.168.60.136 38151 53 7 1 119 -192.168.60.2 192.168.60.136 36524 53 5 1 89 +Saddr Daddr Sprot Dprot udp_time/μs RX/direction len/byte +192.168.60.136 192.168.60.2 53643 53 2 0 39 +192.168.60.136 192.168.60.2 34272 53 3 0 42 +192.168.60.2 192.168.60.136 53 53643 12 1 230 +192.168.60.136 192.168.60.2 34442 53 1 0 40 +192.168.60.2 192.168.60.136 53 59996 2 1 190 ``` 指定参数`-I`，查看ICMP数据包处理时延，单位为微妙。 ```c sudo ./netwatcher -I -saddr daddr time flag -192.168.60.136 192.168.60.136 11 1 -192.168.60.136 192.168.60.136 5 0 -192.168.60.136 192.168.60.136 80 1 +Saddr Daddr icmp_time/μs RX/direction +192.168.60.136 192.168.60.136 11 1 +192.168.60.136 192.168.60.136 5 0 +192.168.60.136 192.168.60.136 80 1 ``` 指定参数`-n`，查看数据包在Netfilter框架（包括PRE_ROUTING、LOCAL_IN、FORWARD、LOCAL_OUT、POST_ROUTING链中处理的时间），单位为微妙，其网络数据包路径为： @@ -228,11 +251,11 @@ saddr daddr time flag ```c sudo ./netwatcher -n -saddr daddr dprot sprot PreRT L_IN FW PostRT L_OUT rx -192.168.60.136 192.168.60.1 22 51729 0 0 0 2 3 0 -192.168.60.136 192.168.60.1 22 51729 0 0 0 1 3 0 -127.0.0.1 127.0.0.1 771 10787 2 2 0 0 0 1 -127.0.0.1 127.0.0.1 771 15685 2 2 0 0 0 1 +Saddr Daddr Sprot Dprot PreRT/μs L_IN/μs FW/μs PostRT/μs L_OUT/μs RX/direction +127.0.0.53 127.0.0.1 53 60590 3 2 0 0 0 1 +127.0.0.1 127.0.0.1 55858 40327 0 0 0 2 4 0 +127.0.0.1 127.0.0.1 55858 40327 1 1 0 0 0 1 +127.0.0.1 127.0.0.1 40327 55858 0 0 0 1 5 0 ``` #### 3.2.2 监控错误数据包 @@ -249,9 +272,11 @@ packet{sock="0xffff13235ac8ac8e",seq="1318124482",ack="2468218244",reason="Inval ```c sudo ./netwatcher -i -SOCK SEQ ACK MAC_TIME IP_TIME TCP_TIME RX HTTP -0xffff9d1ecb3b9180 3705894662 522176002 - - - 0 GET / HTTP/1.1 -0xffff9d1ecb3b9180 522176002 3705894739 - - - 1 HTTP/1.1 200 OK +SOCK Saddr Sport Daddr Dport MAC_TIME/μs IP_TIME/μs TRAN_TIME/μs RX/direction HTTP +0xffff93911049d100 192.168.60.136 44152 1.1.1.1 80 0 0 0 0 0 - +0xffff93911049d100 1.1.1.1 80 192.168.60.136 44152 0 0 0 1 1 HTTP/1.1 301 Moved Permanently +0xffff9391601a6c00 192.168.60.136 44154 1.1.1.1 80 0 0 0 0 0 - +0xffff9391601a6c00 1.1.1.1 80 192.168.60.136 44154 0 0 0 1 1 HTTP/1.1 301 Moved Permanently ``` #### 3.2.3 过滤指定目的端口、源端口 @@ -266,9 +291,10 @@ sudo ./netwatcher -d 80 或者 sudo ./netwatcher -s 80 ```C sudo ./netwatcher -S -saddr daddr sport dport oldstate newstate time -192.168.60.136 1.1.1.1 41586 80 FIN_WAIT1 FIN_WAIT2 386 -192.168.60.136 1.1.1.1 41586 80 FIN_WAIT2 CLOSE 19 +Saddr Daddr Sport Dport oldstate newstate time/μs +192.168.60.136 1.1.1.1 0 80 CLOSE SYN_SENT 0 +192.168.60.136 1.1.1.1 41312 80 SYN_SENT ESTABLISHED 181270 +192.168.60.136 1.1.1.1 41312 80 ESTABLISHED FIN_WAIT1 183729 ``` #### 3.2.5 捕捉丢包及原因 @@ -277,18 +303,20 @@ saddr daddr sport dport oldstate newstate time ```c sudo ./netwatcher -k -time saddr daddr sprot dprot prot addr -reason -14:45:11 127.0.0.1 127.0.0.1 51162 36623 ipv4 ffffffff920fbf89 SKB_DROP_REASON_TCP_OLD_DATA +Time Saddr Daddr Sprot Dprot prot addr reason +13:44:03 1.1.1.1 192.168.60.136 80 49668 ipv4 ffffffff9c914464 SKB_DROP_REASON_NOT_SPECIFIED +13:44:03 1.1.1.1 192.168.60.136 80 49680 ipv4 ffffffff9c914464 SKB_DROP_REASON_NOT_SPECIFIED ``` -指定参数`-T`，可以将虚拟地址转换成函数名+偏移的形式，在此处可以捕捉发生丢包的内核函数名称。 +指定参数`-k -T`，可以将虚拟地址转换成函数名+偏移的形式，在此处可以捕捉发生丢包的内核函数名称。 ```C sudo ./netwatcher -k -T -time saddr daddr sprot dprot prot addr reason -14:48:54 192.168.60.136 192.168.60.136 45828 8090 ipv4 tcp_v4_rcv+0x84 SKB_DROP_REASON_NO_SOCKET -14:49:04 118.105.115.105 110.103.32.49 26210 24435 other unix_dgram_sendmsg+0x521 SKB_DROP_REASON_NOT_SPECIFIED +Time Saddr Daddr Sprot Dprot prot addr reason +13:44:22 1.1.1.1 192.168.60.136 80 37078 ipv4 tcp_v4_rcv+0x84 SKB_DROP_REASON_NOT_SPECIFIED +13:44:22 1.1.1.1 192.168.60.136 80 37092 ipv4 tcp_v4_rcv+0x84 SKB_DROP_REASON_NOT_SPECIFIED +13:44:24 1.1.1.1 192.168.60.136 80 37104 ipv4 tcp_v4_rcv+0x84 SKB_DROP_REASON_NOT_SPECIFIED +13:44:24 1.1.1.1 192.168.60.136 80 37118 ipv4 tcp_v4_rcv+0x84 SKB_DROP_REASON_NOT_SPECIFIED ``` #### 3.2.7 异常时延监控 @@ -297,8 +325,38 @@ time saddr daddr sprot dprot prot addr ```C sudo ./netwatcher -t -L -saddr daddr sprot dprot udp_time rx len -127.0.0.1 127.0.0.53 44530 53 1593 0 51 abnoermal data +SOCK Saddr Sport Daddr Dport MAC_TIME/μs IP_TIME/μs TRAN_TIME/μs RX/direction HTTP +0xffff939118a9c800 192.168.60.136 53788 1.1.1.1 80 2 4 20 0 - +0xffff939118a9c800 1.1.1.1 80 192.168.60.136 53788 14 22 345 1 - +0xffff9391601a7500 192.168.60.136 53790 1.1.1.1 80 3 11 31 0 - +0xffff9391601a7500 1.1.1.1 80 192.168.60.136 53790 22 37 170 1 - +0xffff9391107dec00 113.137.56.223 443 192.168.60.136 34104 11 10 1442 1 abnormal data +``` + +#### 3.2.8 DNS协议包监控 + +选择`udp_rcv`和`udp_send_skb`挂载捕获DNS收包和发包相关信息，从UDP头部开始分析并定位DNS数据部分获取其信息，头部信息存储在`query.header`，数据部分读取存储于`data`，可以获取到DNS协议包的事务ID、标志字段、问题部分计数、应答记录计数、授权记录计数、附加记录计数、域名等信息。 + +```C +sudo ./netwatcher -D +Saddr Daddr Id Flags Qd An Ns Ar Qr RX/direction +192.168.60.2 192.168.60.136 0x7894 0x8180 1 2 0 0 baidu.com 0 +127.0.0.53 0.0.0.0 0xc247 0x8180 1 2 0 1 baidu.com 1 +127.0.0.1 127.0.0.53 0x7637 0x120 1 0 0 1 contile.services.mozilla.com 1 +192.168.60.136 192.168.60.2 0x2c35 0x100 1 0 0 0 contile.services.mozilla.com 1 +``` + +#### 3.2.9 Mysql监控 + +利用uprobe和uretprobe挂载mysql-server层的命令分发处理函数`dispatch_command`，探测该函数获取进程pid、进程名comm、sql语句、sql执行耗时(μs)。 + +```C +sudo ./netwatcher -M +Pid Comm Size Sql duration/μs +1121 connection 32 select @@version_comment limit 1 295 +1121 connection 17 SELECT DATABASE() 277 +1121 connection 14 show databases 1361 +1121 connection 11 show tables 1080 ``` ### 3.3 与Prometheus连接进行可视化 diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/common.bpf.h b/eBPF_Supermarket/Network_Subsystem/net_watcher/common.bpf.h index 1ad6d1771..b5e3f4aa1 100644 --- a/eBPF_Supermarket/Network_Subsystem/net_watcher/common.bpf.h +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/common.bpf.h @@ -148,6 +148,11 @@ struct { __uint(max_entries, 256 * 1024); } netfilter_rb SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024); +} mysql_rb SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); @@ -224,12 +229,19 @@ struct { __type(value, __u64); } tcp_state SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 256*1024); + __type(key, __u32); + __type(value, __u64); +} mysql_time SEC(".maps"); + const volatile int filter_dport = 0; const volatile int filter_sport = 0; const volatile int all_conn = 0, err_packet = 0, extra_conn_info = 0, layer_time = 0, http_info = 0, retrans_info = 0, udp_info =0,net_filter = 0, - drop_reason = 0,icmp_info = 0 ,tcp_info = 0 ,dns_info = 0 ,stack_info = 0; - + drop_reason = 0,icmp_info = 0 ,tcp_info = 0 ,dns_info = 0 ,stack_info = 0,mysql_info = 0; + /* help macro */ #define FILTER \ diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql.bpf.h b/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql.bpf.h new file mode 100644 index 000000000..b2c065148 --- /dev/null +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql.bpf.h @@ -0,0 +1,79 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: blown.away@qq.com +// mysql + +#include "common.bpf.h" +#include "mysql_helper.bpf.h" +static __always_inline int __handle_mysql_start(struct pt_regs *ctx) { + // dispatch_command(THD *thd, const COM_DATA *com_data, enum + char comm[16]; + enum enum_server_command command = PT_REGS_PARM3(ctx); + union COM_DATA *com_data = (union COM_DATA *)PT_REGS_PARM2(ctx); + + pid_t pid = bpf_get_current_pid_tgid() >> 32; + void *thd = (void *)PT_REGS_PARM1(ctx); + char *sql; + u32 size = 0; + + if (command != COM_QUERY) { + return 0; + } + + u64 start_time = bpf_ktime_get_ns()/1000; + bpf_map_update_elem(&mysql_time, &pid, &start_time, BPF_ANY); + + struct mysql_query *message = + bpf_ringbuf_reserve(&mysql_rb, sizeof(*message), 0); + if (!message) { + return 0; + } + + bpf_probe_read(&message->size, sizeof(message->size), + &com_data->com_query.length); + bpf_probe_read_str(&sql, sizeof(sql), &com_data->com_query.query); + bpf_probe_read_str(&message->msql, sizeof(message->msql), sql); + bpf_printk("%s",sql); + + message->pid = pid; + bpf_get_current_comm(&message->comm, sizeof(comm)); + + bpf_ringbuf_submit(message, 0); + return 0; +} + +static __always_inline int __handle_mysql_end(struct pt_regs *ctx) { + + pid_t pid = bpf_get_current_pid_tgid() >> 32; + u64 *start_time_ptr, duration; + u64 end_time = bpf_ktime_get_ns()/1000; + start_time_ptr = bpf_map_lookup_elem(&mysql_time, &pid); + if (!start_time_ptr) { + return 0; + } + + duration = end_time - *start_time_ptr; + struct mysql_query *message = + bpf_ringbuf_reserve(&mysql_rb, sizeof(*message), 0); + if (!message) { + return 0; + } + + message->duratime = duration; + + bpf_ringbuf_submit(message, 0); + bpf_map_delete_elem(&mysql_time, &pid); + return 0; +} diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql_helper.bpf.h b/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql_helper.bpf.h new file mode 100644 index 000000000..222764e1c --- /dev/null +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/mysql_helper.bpf.h @@ -0,0 +1,171 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: blown.away@qq.com +// +// netwatcher libbpf 内核<->用户 传递信息相关结构体 + +#ifndef __MYSQL_HELPER_BPF_H +#define __MYSQL_HELPER_BPF_H + +#include "netwatcher.h" +#include "vmlinux.h" +#include +#include +#include +#include +#include +#include + +enum enum_server_command { + COM_SLEEP, + COM_QUIT, + COM_INIT_DB, + COM_QUERY, + COM_FIELD_LIST, + COM_CREATE_DB, + COM_DROP_DB, + COM_REFRESH, + COM_SHUTDOWN, + COM_STATISTICS, + COM_PROCESS_INFO, + COM_CONNECT, + COM_PROCESS_KILL, + COM_DEBUG, + COM_PING, + COM_TIME, + COM_DELAYED_INSERT, + COM_CHANGE_USER, + COM_BINLOG_DUMP, + COM_TABLE_DUMP, + COM_CONNECT_OUT, + COM_REGISTER_SLAVE, + COM_STMT_PREPARE, + COM_STMT_EXECUTE, + COM_STMT_SEND_LONG_DATA, + COM_STMT_CLOSE, + COM_STMT_RESET, + COM_SET_OPTION, + COM_STMT_FETCH, + COM_DAEMON, + COM_BINLOG_DUMP_GTID, + COM_RESET_CONNECTION, + /* don't forget to update const char *command_name[] in sql_parse.cc */ + /* Must be last */ + COM_END +}; + +typedef struct st_com_init_db_data { + const char *db_name; + unsigned long length; +} COM_INIT_DB_DATA; + +#define MYSQL_SHUTDOWN_KILLABLE_CONNECT (unsigned char)(1 << 0) +#define MYSQL_SHUTDOWN_KILLABLE_TRANS (unsigned char)(1 << 1) +#define MYSQL_SHUTDOWN_KILLABLE_LOCK_TABLE (unsigned char)(1 << 2) +#define MYSQL_SHUTDOWN_KILLABLE_UPDATE (unsigned char)(1 << 3) + +#define LOCK_MODE_MASK 0xFUL +#define LOCK_TYPE_MASK 0xF0UL + +enum mysql_enum_shutdown_level { + SHUTDOWN_DEFAULT = 0, + SHUTDOWN_WAIT_CONNECTIONS = MYSQL_SHUTDOWN_KILLABLE_CONNECT, + SHUTDOWN_WAIT_TRANSACTIONS = MYSQL_SHUTDOWN_KILLABLE_TRANS, + SHUTDOWN_WAIT_UPDATES = MYSQL_SHUTDOWN_KILLABLE_UPDATE, + SHUTDOWN_WAIT_ALL_BUFFERS = (MYSQL_SHUTDOWN_KILLABLE_UPDATE << 1), + SHUTDOWN_WAIT_CRITICAL_BUFFERS = (MYSQL_SHUTDOWN_KILLABLE_UPDATE << 1) + 1, + KILL_QUERY = 254, + KILL_CONNECTION = 255 +}; + +typedef struct st_com_refresh_data { + unsigned char options; +} COM_REFRESH_DATA; + +typedef struct st_com_shutdown_data { + enum mysql_enum_shutdown_level level; +} COM_SHUTDOWN_DATA; + +typedef struct st_com_kill_data { + unsigned long id; +} COM_KILL_DATA; + +typedef struct st_com_set_option_data { + unsigned int opt_command; +} COM_SET_OPTION_DATA; + +typedef struct st_com_stmt_execute_data { + unsigned long stmt_id; + unsigned long flags; + unsigned char *params; + unsigned long params_length; +} COM_STMT_EXECUTE_DATA; + +typedef struct st_com_stmt_fetch_data { + unsigned long stmt_id; + unsigned long num_rows; +} COM_STMT_FETCH_DATA; + +typedef struct st_com_stmt_send_long_data_data { + unsigned long stmt_id; + unsigned int param_number; + unsigned char *longdata; + unsigned long length; +} COM_STMT_SEND_LONG_DATA_DATA; + +typedef struct st_com_stmt_prepare_data { + const char *query; + unsigned int length; +} COM_STMT_PREPARE_DATA; + +typedef struct st_stmt_close_data { + unsigned int stmt_id; +} COM_STMT_CLOSE_DATA; + +typedef struct st_com_stmt_reset_data { + unsigned int stmt_id; +} COM_STMT_RESET_DATA; + +typedef struct st_com_query_data { + const char *query; + unsigned int length; +} COM_QUERY_DATA; + +typedef struct st_com_field_list_data { + unsigned char *table_name; + unsigned int table_name_length; + const unsigned char *query; + unsigned int query_length; +} COM_FIELD_LIST_DATA; + +union COM_DATA { + COM_INIT_DB_DATA com_init_db; + COM_REFRESH_DATA com_refresh; + COM_SHUTDOWN_DATA com_shutdown; + COM_KILL_DATA com_kill; + COM_SET_OPTION_DATA com_set_option; + COM_STMT_EXECUTE_DATA com_stmt_execute; + COM_STMT_FETCH_DATA com_stmt_fetch; + COM_STMT_SEND_LONG_DATA_DATA com_stmt_send_long_data; + COM_STMT_PREPARE_DATA com_stmt_prepare; + COM_STMT_CLOSE_DATA com_stmt_close; + COM_STMT_RESET_DATA com_stmt_reset; + COM_QUERY_DATA com_query; + COM_FIELD_LIST_DATA com_field_list; +}; + +/* help functions end */ + +#endif diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.bpf.c b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.bpf.c index f0fead939..5c67c4090 100644 --- a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.bpf.c +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.bpf.c @@ -28,6 +28,8 @@ #include "udp.bpf.h" +#include "mysql.bpf.h" + #include "drop.bpf.h" // accecpt an TCP connection @@ -102,7 +104,9 @@ int BPF_KPROBE(eth_type_trans, struct sk_buff *skb) { /** in only ipv4 */ SEC("kprobe/ip_rcv_core") // 跟踪记录ipv4数据包在内核中的处理时间 -int BPF_KPROBE(ip_rcv_core, struct sk_buff *skb) { return __ip_rcv_core(skb); } +int BPF_KPROBE(ip_rcv_core, struct sk_buff *skb) { + return __ip_rcv_core(skb); +} /** in only ipv6 */ SEC("kprobe/ip6_rcv_core") int BPF_KPROBE(ip6_rcv_core, struct sk_buff *skb) { @@ -111,11 +115,15 @@ int BPF_KPROBE(ip6_rcv_core, struct sk_buff *skb) { /**in only ipv4 */ // 接收数据包 SEC("kprobe/tcp_v4_rcv") // 记录数据包在tcpv4层时间戳 -int BPF_KPROBE(tcp_v4_rcv, struct sk_buff *skb) { return __tcp_v4_rcv(skb); } +int BPF_KPROBE(tcp_v4_rcv, struct sk_buff *skb) { + return __tcp_v4_rcv(skb); +} /** in only ipv6 */ SEC("kprobe/tcp_v6_rcv") // 接收tcpv6数据包 -int BPF_KPROBE(tcp_v6_rcv, struct sk_buff *skb) { return __tcp_v6_rcv(skb); } +int BPF_KPROBE(tcp_v6_rcv, struct sk_buff *skb) { + return __tcp_v6_rcv(skb); +} // v4 & v6 do_rcv to get sk and other info SEC("kprobe/tcp_v4_do_rcv") @@ -288,10 +296,14 @@ int BPF_KPROBE(ip_forward, struct sk_buff *skb) { // drop SEC("tp/skb/kfree_skb") -int tp_kfree(struct trace_event_raw_kfree_skb *ctx) { return __tp_kfree(ctx); } +int tp_kfree(struct trace_event_raw_kfree_skb *ctx) { + return __tp_kfree(ctx); +} SEC("kprobe/icmp_rcv") -int BPF_KPROBE(icmp_rcv, struct sk_buff *skb) { return __icmp_time(skb); } +int BPF_KPROBE(icmp_rcv, struct sk_buff *skb) { + return __icmp_time(skb); +} SEC("kprobe/__sock_queue_rcv_skb") int BPF_KPROBE(__sock_queue_rcv_skb, struct sock *sk, struct sk_buff *skb) { @@ -306,6 +318,16 @@ int BPF_KPROBE(icmp_reply, struct icmp_bxm *icmp_param, struct sk_buff *skb) { // tcpstate SEC("tracepoint/sock/inet_sock_set_state") int handle_set_state(struct trace_event_raw_inet_sock_set_state *ctx) { - return __handle_set_state(ctx); } + +// mysql +SEC("uprobe/_Z16dispatch_commandP3THDPK8COM_DATA19enum_server_command") +int BPF_KPROBE(query__start) { + return __handle_mysql_start(ctx); +} + +SEC("uretprobe/_Z16dispatch_commandP3THDPK8COM_DATA19enum_server_command") +int BPF_KPROBE(query__end){ + return __handle_mysql_end(ctx); +} \ No newline at end of file diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.c b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.c index e2b5c3d09..7ff8a7bed 100644 --- a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.c +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.c @@ -43,7 +43,7 @@ static int sport = 0, dport = 0; // for filter static int all_conn = 0, err_packet = 0, extra_conn_info = 0, layer_time = 0, http_info = 0, retrans_info = 0, udp_info = 0, net_filter = 0, drop_reason = 0, addr_to_func = 0, icmp_info = 0, tcp_info = 0, - time_load = 0, dns_info = 0, stack_info=0; // flag + time_load = 0, dns_info = 0, stack_info = 0, mysql_info = 0; // flag static const char *tcp_states[] = { [1] = "ESTABLISHED", [2] = "SYN_SENT", [3] = "SYN_RECV", @@ -76,6 +76,9 @@ static const struct argp_option opts[] = { "问题部分计数、An 应答记录计数、Ns 授权记录计数、Ar 附加记录计数、Qr " "域名、rx 收发包 "}, {"stack", 'A', 0, 0, "set to trace of stack "}, + {"mysql", 'M', 0, 0, + "set to trace mysql information info include Pid 进程id、Comm " + "进程名、Size sql语句字节大小、Sql 语句"}, {}}; static error_t parse_arg(int key, char *arg, struct argp_state *state) { @@ -132,6 +135,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { case 'A': stack_info = 1; break; + case 'M': + mysql_info = 1; + break; default: return ARGP_ERR_UNKNOWN; } @@ -156,6 +162,7 @@ enum MonitorMode { MODE_ICMP, MODE_TCP, MODE_DNS, + MODE_MYSQL, MODE_DEFAULT }; @@ -172,6 +179,8 @@ enum MonitorMode get_monitor_mode() { return MODE_TCP; } else if (dns_info) { return MODE_DNS; + } else if (mysql_info) { + return MODE_MYSQL; } else { return MODE_DEFAULT; } @@ -212,6 +221,38 @@ void print_logo() { pclose(lolcat_pipe); } +static const char binary_path[] = "/usr/sbin/mysqld"; +#define __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe) \ + do { \ + LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts, .func_name = #sym_name, \ + .retprobe = is_retprobe); \ + skel->links.prog_name = bpf_program__attach_uprobe_opts( \ + skel->progs.prog_name, -1, binary_path, 0, &uprobe_opts); \ + } while (false) + +#define __CHECK_PROGRAM(skel, prog_name) \ + do { \ + if (!skel->links.prog_name) { \ + perror("no program attached for " #prog_name); \ + return -errno; \ + } \ + } while (false) + +#define __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, is_retprobe) \ + do { \ + __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe); \ + __CHECK_PROGRAM(skel, prog_name); \ + } while (false) + +#define ATTACH_UPROBE(skel, sym_name, prog_name) \ + __ATTACH_UPROBE(skel, sym_name, prog_name, false) +#define ATTACH_URETPROBE(skel, sym_name, prog_name) \ + __ATTACH_UPROBE(skel, sym_name, prog_name, true) + +#define ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name) \ + __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, false) +#define ATTACH_URETPROBE_CHECKED(skel, sym_name, prog_name) \ + __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, true) struct SymbolEntry symbols[300000]; int num_symbols = 0; @@ -369,6 +410,7 @@ static void set_rodata_flags(struct netwatcher_bpf *skel) { skel->rodata->icmp_info = icmp_info; skel->rodata->dns_info = dns_info; skel->rodata->stack_info = stack_info; + skel->rodata->mysql_info = mysql_info; } static void set_disable_load(struct netwatcher_bpf *skel) { @@ -502,6 +544,10 @@ static void set_disable_load(struct netwatcher_bpf *skel) { bpf_program__set_autoload(skel->progs.icmp_reply, icmp_info ? true : false); bpf_program__set_autoload(skel->progs.handle_set_state, tcp_info ? true : false); + bpf_program__set_autoload(skel->progs.query__start, + mysql_info ? true : false); + bpf_program__set_autoload(skel->progs.query__end, + mysql_info ? true : false); } static void print_header(enum MonitorMode mode) { @@ -554,6 +600,14 @@ static void print_header(enum MonitorMode mode) { "Saddr", "Daddr", "Id", "Flags", "Qd", "An", "Ns", "Ar", "Qr", "RX/direction"); break; + case MODE_MYSQL: + printf("===============================================================" + "====================MYSQL " + "INFORMATION====================================================" + "============================\n"); + printf("%-20s %-20s %-20s %-40s %-20s \n", "Pid", "Comm", "Size", "Sql", + "duration/μs"); + break; case MODE_DEFAULT: printf("===============================================================" "=INFORMATION===================================================" @@ -702,7 +756,7 @@ static int print_conns(struct netwatcher_bpf *skel) { static int print_packet(void *ctx, void *packet_info, size_t size) { if (udp_info || net_filter || drop_reason || icmp_info || tcp_info || - dns_info) + dns_info || mysql_info) return 0; const struct pack_t *pack_info = packet_info; if (pack_info->mac_time > MAXTIME || pack_info->ip_time > MAXTIME || @@ -827,7 +881,8 @@ static int print_udp(void *ctx, void *packet_info, size_t size) { const struct udp_message *pack_info = packet_info; unsigned int saddr = pack_info->saddr; unsigned int daddr = pack_info->daddr; - if(pack_info->tran_time > MAXTIME||(daddr & 0x0000FFFF) == 0x0000007F || (saddr & 0x0000FFFF) == 0x0000007F) + if (pack_info->tran_time > MAXTIME || (daddr & 0x0000FFFF) == 0x0000007F || + (saddr & 0x0000FFFF) == 0x0000007F) return 0; printf("%-20s %-20s %-20u %-20u %-20llu %-20d %-20d", inet_ntop(AF_INET, &saddr, s_str, sizeof(s_str)), @@ -1012,55 +1067,80 @@ static int print_dns(void *ctx, void *packet_info, size_t size) { print_domain_name((const unsigned char *)pack_info->data, domain_name); - printf("%-20s %-20s %-#12x %-#12x %-5x %-5x %-5x %-5x %-47s %-10d\n", - s_str, d_str, pack_info->id, pack_info->flags, pack_info->qdcount, + printf("%-20s %-20s %-#12x %-#12x %-5x %-5x %-5x %-5x %-47s %-10d\n", s_str, + d_str, pack_info->id, pack_info->flags, pack_info->qdcount, pack_info->ancount, pack_info->nscount, pack_info->arcount, domain_name, pack_info->rx); return 0; } +static mysql_query last_query; + +static int print_mysql(void *ctx, void *packet_info, size_t size) { + const mysql_query *pack_info = packet_info; + // 假设duratime总是0 + if (pack_info->duratime == 0) { + // 存储开始事件数据 + memcpy(&last_query, pack_info, sizeof(mysql_query)); + } else { + // 结束事件 合并 + printf("%-20d %-20s %-20u %-40s %-20llu\n", last_query.pid, + last_query.comm, last_query.size, last_query.msql, + pack_info->duratime); + // 重置 + memset(&last_query, 0, sizeof(last_query)); + } + return 0; +} static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) { return vfprintf(stderr, format, args); } -static void show_stack_trace(__u64 *stack, int stack_sz, pid_t pid) -{ +static void show_stack_trace(__u64 *stack, int stack_sz, pid_t pid) { int i; printf("-----------------------------------\n"); - for (i = 1; i < stack_sz; i++) { - if(addr_to_func) - { - struct SymbolEntry data= findfunc(stack[i]); + for (i = 1; i < stack_sz; i++) { + if (addr_to_func) { + struct SymbolEntry data = findfunc(stack[i]); char result[40]; sprintf(result, "%s+0x%llx", data.name, stack[i] - data.addr); - printf("%-10d [<%016llx>]=%s\n", i, stack[i], result); - } - else - { + printf("%-10d [<%016llx>]=%s\n", i, stack[i], result); + } else { printf("%-10d [<%016llx>]\n", i, stack[i]); } - } + } printf("-----------------------------------\n"); } -static int print_trace(void *_ctx, void *data, size_t size) -{ +static int print_trace(void *_ctx, void *data, size_t size) { struct stacktrace_event *event = data; - if (event->kstack_sz <= 0 && event->ustack_sz <= 0) - return 1; + if (event->kstack_sz <= 0 && event->ustack_sz <= 0) + return 1; - printf("COMM: %s (pid=%d) @ CPU %d\n", event->comm, event->pid, event->cpu_id); + printf("COMM: %s (pid=%d) @ CPU %d\n", event->comm, event->pid, + event->cpu_id); - if (event->kstack_sz > 0) { - printf("Kernel:\n"); - show_stack_trace(event->kstack, event->kstack_sz / sizeof(__u64), 0); - } else { - printf("No Kernel Stack\n"); - } - printf("\n"); - return 0; + if (event->kstack_sz > 0) { + printf("Kernel:\n"); + show_stack_trace(event->kstack, event->kstack_sz / sizeof(__u64), 0); + } else { + printf("No Kernel Stack\n"); + } + printf("\n"); + return 0; } + +int attach_uprobe(struct netwatcher_bpf *skel) { + ATTACH_UPROBE_CHECKED( + skel, _Z16dispatch_commandP3THDPK8COM_DATA19enum_server_command, + query__start); + ATTACH_URETPROBE_CHECKED( + skel, _Z16dispatch_commandP3THDPK8COM_DATA19enum_server_command, + query__end); + return 0; +} + int main(int argc, char **argv) { char *last_slash = strrchr(argv[0], '/'); if (last_slash) { @@ -1082,6 +1162,7 @@ int main(int argc, char **argv) { struct ring_buffer *tcp_rb = NULL; struct ring_buffer *dns_rb = NULL; struct ring_buffer *trace_rb = NULL; + struct ring_buffer *mysql_rb = NULL; struct netwatcher_bpf *skel; int err; /* Parse command line arguments */ @@ -1116,10 +1197,19 @@ int main(int argc, char **argv) { } /* Attach tracepoint handler */ - err = netwatcher_bpf__attach(skel); - if (err) { - fprintf(stderr, "Failed to attach BPF skeleton\n"); - goto cleanup; + if (mysql_info) { + err = attach_uprobe(skel); + if (err) { + fprintf(stderr, "failed to attach uprobes\n"); + + goto cleanup; + } + } else { + err = netwatcher_bpf__attach(skel); + if (err) { + fprintf(stderr, "Failed to attach BPF skeleton\n"); + goto cleanup; + } } enum MonitorMode mode = get_monitor_mode(); @@ -1169,12 +1259,20 @@ int main(int argc, char **argv) { fprintf(stderr, "Failed to create ring buffer(dns)\n"); goto cleanup; } - trace_rb =ring_buffer__new(bpf_map__fd(skel->maps.trace_rb), print_trace, NULL, NULL); + trace_rb = ring_buffer__new(bpf_map__fd(skel->maps.trace_rb), print_trace, + NULL, NULL); if (!trace_rb) { err = -1; fprintf(stderr, "Failed to create ring buffer(trace)\n"); goto cleanup; } + mysql_rb = ring_buffer__new(bpf_map__fd(skel->maps.mysql_rb), print_mysql, + NULL, NULL); + if (!mysql_rb) { + err = -1; + fprintf(stderr, "Failed to create ring buffer(trace)\n"); + goto cleanup; + } /* Set up ring buffer polling */ rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), print_packet, NULL, NULL); if (!rb) { @@ -1195,6 +1293,7 @@ int main(int argc, char **argv) { err = ring_buffer__poll(tcp_rb, 100 /* timeout, ms */); err = ring_buffer__poll(dns_rb, 100 /* timeout, ms */); err = ring_buffer__poll(trace_rb, 100 /* timeout, ms */); + err = ring_buffer__poll(mysql_rb, 100 /* timeout, ms */); print_conns(skel); sleep(1); /* Ctrl-C will cause -EINTR */ diff --git a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.h b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.h index 4a36a807e..c64d351c9 100644 --- a/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.h +++ b/eBPF_Supermarket/Network_Subsystem/net_watcher/netwatcher.h @@ -159,12 +159,21 @@ struct dns_information { #define MAX_STACK_DEPTH 128 typedef u64 stack_trace_t[MAX_STACK_DEPTH]; struct stacktrace_event { - u32 pid; - u32 cpu_id; - char comm[16]; - signed int kstack_sz; - signed int ustack_sz; - stack_trace_t kstack; - stack_trace_t ustack; + u32 pid; + u32 cpu_id; + char comm[16]; + signed int kstack_sz; + signed int ustack_sz; + stack_trace_t kstack; + stack_trace_t ustack; }; + +typedef struct mysql_query { + int pid; + char comm[20]; + u32 size; + char msql[256]; + u64 duratime; +} mysql_query; + #endif /* __NETWATCHER_H */ \ No newline at end of file From dc2443f9f6779f09b68702cb89bad6798d0a12e8 Mon Sep 17 00:00:00 2001 From: wynyibo Date: Thu, 6 Jun 2024 16:14:56 +0800 Subject: [PATCH 13/16] update --- .github/workflows/ebpf_net_watcher.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ebpf_net_watcher.yml b/.github/workflows/ebpf_net_watcher.yml index 025578510..efce13059 100644 --- a/.github/workflows/ebpf_net_watcher.yml +++ b/.github/workflows/ebpf_net_watcher.yml @@ -46,4 +46,6 @@ jobs: sudo timeout -s SIGINT 5 ./netwatcher -k -T || if [[ $? != 124 && $? != 0 ]];then exit $?;fi sudo timeout -s SIGINT 5 ./netwatcher -I || if [[ $? != 124 && $? != 0 ]];then exit $?;fi sudo timeout -s SIGINT 5 ./netwatcher -S || if [[ $? != 124 && $? != 0 ]];then exit $?;fi + sudo timeout -s SIGINT 5 ./netwatcher -D || if [[ $? != 124 && $? != 0 ]];then exit $?;fi + sudo timeout -s SIGINT 5 ./netwatcher -M || if [[ $? != 124 && $? != 0 ]];then exit $?;fi timeout-minutes: 5 From 564b28715b4bb57e9a46f767a94245558bd26cf8 Mon Sep 17 00:00:00 2001 From: Y_y_s <78297703+Monkey857@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:07:36 +0800 Subject: [PATCH 14/16] visual (#825) --- .../kvm_watcher/docs/Visualization_conf.md | 1 + .../kvm_watcher/src/kvm_watcher.c | 52 ++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/Visualization_conf.md b/eBPF_Supermarket/kvm_watcher/docs/Visualization_conf.md index 318b8755e..043259e7e 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/Visualization_conf.md +++ b/eBPF_Supermarket/kvm_watcher/docs/Visualization_conf.md @@ -110,6 +110,7 @@ make start_service #进入lmp/eBPF_Visualization/eBPF_prometheus目录，执行以下操作 #开启ebpf程序，并且向8090端口推送ebpf程序采集的数据，发送给prometheus服务端 #这里以监测vcpu调度的数据来举例： +#目前-e、-o、-i、-f -m功能的数据格式已经适配了可视化工具，请读者使用以上四种功能的数据来进行可视化展示 ./data-visual collect lmp/eBPF_Supermarket/kvm_watcher/kvm_watcher -o -p [进程号] ``` diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index daf3c22d7..1d08fb50f 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -566,6 +566,12 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } case PAGE_FAULT: { // 使用 e->page_fault_data 访问 PAGE_FAULT 特有成员 + if (env.show) { + printf("%-18.6f %-10u %-6u %-10.4f\n", timestamp_ms, + e->process.pid, e->page_fault_data.count, + NS_TO_US_WITH_DECIMAL(e->page_fault_data.delay)); + break; + } printf("%-18.6f %-15s %-10u %-12llx %-6u %-10.4f ", timestamp_ms, e->process.comm, e->process.pid, e->page_fault_data.addr, e->page_fault_data.count, @@ -670,11 +676,19 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { break; } case IRQ_INJECT: { - printf("%-18.6f %-15s %-10d %-10lld %#-10x %-10d %-10lld %-10s\n", - timestamp_ms, e->process.comm, e->process.pid, - e->irq_inject_data.delay, e->irq_inject_data.irq_nr, - e->irq_inject_data.vcpu_id, e->irq_inject_data.injections, - e->irq_inject_data.soft ? "Soft/INTn" : "IRQ"); + if (env.show) { + printf("%-18.6f %-10d %-10lld %-10d %-10d %-10lld\n", + timestamp_ms, e->process.pid, e->irq_inject_data.delay, + e->irq_inject_data.irq_nr, e->irq_inject_data.vcpu_id, + e->irq_inject_data.injections); + } else { + printf( + "%-18.6f %-15s %-10d %-10lld %#-10x %-10d %-10lld %-10s\n", + timestamp_ms, e->process.comm, e->process.pid, + e->irq_inject_data.delay, e->irq_inject_data.irq_nr, + e->irq_inject_data.vcpu_id, e->irq_inject_data.injections, + e->irq_inject_data.soft ? "Soft/INTn" : "IRQ"); + } break; } case HYPERCALL: { @@ -758,18 +772,29 @@ static int print_event_head(struct env *env) { "USERSPACE_ADDR", "SLOT_ID"); break; case PAGE_FAULT: - printf("%-18s %-15s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %s\n", - "TIME(ms)", "COMM", "PID", "GPA", "COUNT", "DELAY(us)", - "HVA", "PFN", "MEM_SLOTID", "ERROR_TYPE"); + if (env->show) { + printf("%-18s %-10s %-6s %-10s \n", "TIME(ms)", "PID", "COUNT", + "DELAY(us)"); + } else { + printf( + "%-18s %-15s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %s\n", + "TIME(ms)", "COMM", "PID", "GPA", "COUNT", "DELAY(us)", + "HVA", "PFN", "MEM_SLOTID", "ERROR_TYPE"); + } break; case IRQCHIP: printf("%-18s %-15s %-10s %-10s %-14s %-10s %-10s\n", "TIME(ms)", "COMM", "PID", "DELAY", "TYPE/PIN", "DST/VEC", "OTHERS"); break; case IRQ_INJECT: - printf("%-18s %-15s %-10s %-10s %-10s %-10s %-10s %-10s\n", - "TIME(ms)", "COMM", "PID", "DELAY", "IRQ_NR", "VCPU_ID", - "INJECTIONS", "TYPE"); + if (env->show) { + printf("%-18s %-10s %-10s %-10s %-10s %-10s \n", "TIME(ms)", + "PID", "DELAY", "IRQ_NR", "VCPU_ID", "INJECTIONS"); + } else { + printf("%-18s %-15s %-10s %-10s %-10s %-10s %-10s %-10s\n", + "TIME(ms)", "COMM", "PID", "DELAY", "IRQ_NR", "VCPU_ID", + "INJECTIONS", "TYPE"); + } break; case HYPERCALL: { printf("Waiting hypercall ... \n"); @@ -1202,9 +1227,6 @@ int main(int argc, char **argv) { struct ring_buffer *rb = NULL; struct kvm_watcher_bpf *skel; int err; - //可视化调整输出格式 - // print_logo(); - /*解析命令行参数*/ err = argp_parse(&argp, argc, argv, 0, NULL, NULL); if (err) @@ -1256,6 +1278,8 @@ int main(int argc, char **argv) { fprintf(stderr, "Invalid env parm\n"); goto cleanup; } + if (!env.show) + print_logo(); /*打印信息头*/ err = print_event_head(&env); From 1749ab211032711e20dfdb373a63e09f2f6c0615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=AE=9C=E8=90=B1?= <85030740+syxl-time@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:08:25 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E5=AF=B9=E7=94=A8=E6=88=B7=E6=80=81?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=AE=80=E5=8C=96=E4=BF=AE=E6=94=B9=20(#815)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mem_watcher/mem_watcher.c | 559 ++++++++---------- 1 file changed, 248 insertions(+), 311 deletions(-) diff --git a/eBPF_Supermarket/Memory_Subsystem/mem_watcher/mem_watcher.c b/eBPF_Supermarket/Memory_Subsystem/mem_watcher/mem_watcher.c index 5156eb2b1..8f596b55d 100644 --- a/eBPF_Supermarket/Memory_Subsystem/mem_watcher/mem_watcher.c +++ b/eBPF_Supermarket/Memory_Subsystem/mem_watcher/mem_watcher.c @@ -47,6 +47,7 @@ static size_t g_stacks_size = 0; static struct blaze_symbolizer *symbolizer; static int attach_pid; +pid_t own_pid; static char binary_path[128] = { 0 }; struct allocation { @@ -57,6 +58,8 @@ struct allocation { static struct allocation *allocs; +static volatile bool exiting = false; + #define __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe) \ do { \ LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts, \ @@ -90,6 +93,48 @@ static struct allocation *allocs; #define ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name) __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, false) #define ATTACH_URETPROBE_CHECKED(skel, sym_name, prog_name) __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, true) +#define PROCESS_SKEL(skel, func) \ + skel = func##_bpf__open(); \ + if (!skel) { \ + fprintf(stderr, "Failed to open and load BPF skeleton\n"); \ + return 1; \ + } \ + process_##func(skel) + +#define POLL_RING_BUFFER(rb, timeout, err) \ + while (!exiting) { \ + err = ring_buffer__poll(rb, timeout); \ + if (err == -EINTR) { \ + err = 0; \ + break; \ + } \ + if (err < 0) { \ + printf("Error polling perf buffer: %d\n", err); \ + break; \ + } \ + } + +#define LOAD_AND_ATTACH_SKELETON(skel, event) \ + do { \ + skel->bss->user_pid = own_pid; \ + err = event##_bpf__load(skel); \ + if (err) { \ + fprintf(stderr, "Failed to load and verify BPF skeleton\n"); \ + goto event##_cleanup; \ + } \ + \ + err = event##_bpf__attach(skel); \ + if (err) { \ + fprintf(stderr, "Failed to attach BPF skeleton\n"); \ + goto event##_cleanup; \ + } \ + \ + rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event_##event, NULL, NULL); \ + if (!rb) { \ + fprintf(stderr, "Failed to create ring buffer\n"); \ + goto event##_cleanup; \ + } \ + } while(0) static struct env { int time; @@ -148,43 +193,23 @@ static const struct argp_option opts[] = { }; static error_t parse_arg(int key, char *arg, struct argp_state *state) { - switch (key) { - case 't': - env.time = strtol(arg, NULL, 10); - if (env.time) - alarm(env.time); - break; - case 'a': - env.paf = true; - break; - case 'p': - env.pr = true; - break; - case 'r': - env.procstat = true; - break; - case 's': - env.sysstat = true; - break; - case 'n': - env.part2 = true; - break; - case 'h': - argp_state_help(state, stderr, ARGP_HELP_STD_HELP); - break; - case 'P': - env.choose_pid = strtol(arg, NULL, 10); - break; - case 'R': - env.rss = true; - break; - case 'l': - env.memleak = true; - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; + switch (key) { + case 't': + env.time = strtol(arg, NULL, 10); + if (env.time) alarm(env.time); + break; + case 'a': env.paf = true; break; + case 'p': env.pr = true; break; + case 'r': env.procstat = true; break; + case 's': env.sysstat = true; break; + case 'n': env.part2 = true; break; + case 'h': argp_state_help(state, stderr, ARGP_HELP_STD_HELP); break; + case 'P': env.choose_pid = strtol(arg, NULL, 10); break; + case 'R': env.rss = true; break; + case 'l': env.memleak = true; break; + default: return ARGP_ERR_UNKNOWN; + } + return 0; } static const struct argp argp = { @@ -193,6 +218,80 @@ static const struct argp argp = { .doc = argp_program_doc, }; +// Function prototypes +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args); +static void sig_handler(int sig); +static void setup_signals(void); +static void disable_kernel_tracepoints(struct memleak_bpf *skel); +static void print_frame(const char *name, uintptr_t input_addr, uintptr_t addr, uint64_t offset, const blaze_symbolize_code_info *code_info); +static void show_stack_trace(__u64 *stack, int stack_sz, pid_t pid); +static int print_outstanding_allocs(struct memleak_bpf *skel); +static int print_outstanding_combined_allocs(struct memleak_bpf *skel, pid_t pid); +static int handle_event_paf(void *ctx, void *data, size_t data_sz); +static int handle_event_pr(void *ctx, void *data, size_t data_sz); +static int handle_event_procstat(void *ctx, void *data, size_t data_sz); +static int handle_event_sysstat(void *ctx, void *data, size_t data_sz); +static int attach_uprobes(struct memleak_bpf *skel); +static int process_paf(struct paf_bpf *skel_paf); +static int process_pr(struct pr_bpf *skel_pr); +static int process_procstat(struct procstat_bpf *skel_procstat); +static int process_sysstat(struct sysstat_bpf *skel_sysstat); +static int process_memleak(struct memleak_bpf *skel_memleak, struct env); + +// Main function +int main(int argc, char **argv) { + int err; + struct paf_bpf *skel_paf; + struct pr_bpf *skel_pr; + struct procstat_bpf *skel_procstat; + struct sysstat_bpf *skel_sysstat; + struct memleak_bpf *skel_memleak; + + err = argp_parse(&argp, argc, argv, 0, NULL, NULL); + if (err) + return err; + + own_pid = getpid(); + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + libbpf_set_print(libbpf_print_fn); + + setup_signals(); + + if (env.paf) { + PROCESS_SKEL(skel_paf, paf); + } else if (env.pr) { + PROCESS_SKEL(skel_pr, pr); + } else if (env.procstat) { + PROCESS_SKEL(skel_procstat, procstat); + } else if (env.sysstat) { + PROCESS_SKEL(skel_sysstat, sysstat); + } else if (env.memleak) { + if (env.choose_pid != 0) { + printf("用户态内存泄漏\n"); + env.kernel_trace = false; + attach_pid = env.choose_pid; + } + else + attach_pid = 0; + + strcpy(binary_path, "/lib/x86_64-linux-gnu/libc.so.6"); + + allocs = calloc(ALLOCS_MAX_ENTRIES, sizeof(*allocs)); + + /* Set up libbpf errors and debug info callback */ + libbpf_set_print(libbpf_print_fn); + + /* Load and verify BPF application */ + skel_memleak = memleak_bpf__open(); + if (!skel_memleak) { + fprintf(stderr, "Failed to open BPF skeleton\n"); + return 1; + } + process_memleak(skel_memleak, env); + } + return 0; +} + int alloc_size_compare(const void *a, const void *b) { const struct allocation *x = (struct allocation *)a; @@ -437,13 +536,17 @@ static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va return vfprintf(stderr, format, args); } -static volatile bool exiting = false; - static void sig_handler(int sig) { exiting = true; exit(EXIT_SUCCESS); } +static void setup_signals(void) { + signal(SIGINT, sig_handler); + signal(SIGTERM, sig_handler); + signal(SIGALRM, sig_handler); +} + /* static char* flags(int flag) { @@ -551,8 +654,6 @@ static int handle_event_sysstat(void *ctx, void *data, size_t data_sz) { return 0; } -pid_t own_pid; - int attach_uprobes(struct memleak_bpf *skel) { ATTACH_UPROBE_CHECKED(skel, malloc, malloc_enter); ATTACH_URETPROBE_CHECKED(skel, malloc, malloc_exit); @@ -593,315 +694,151 @@ int attach_uprobes(struct memleak_bpf *skel) { return 0; } -int main(int argc, char **argv) { - struct ring_buffer *rb = NULL; - struct paf_bpf *skel_paf; - struct pr_bpf *skel_pr; - struct procstat_bpf *skel_procstat; - struct sysstat_bpf *skel_sysstat; - struct memleak_bpf *skel; - - int err, i; - LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts); - - err = argp_parse(&argp, argc, argv, 0, NULL, NULL); - if (err) - return err; - own_pid = getpid(); - libbpf_set_strict_mode(LIBBPF_STRICT_ALL); - - /* Set up libbpf errors and debug info callback */ - libbpf_set_print(libbpf_print_fn); - - /* Cleaner handling of Ctrl-C */ - signal(SIGINT, sig_handler); - signal(SIGTERM, sig_handler); - signal(SIGALRM, sig_handler); - - if (env.paf) { - /* Load and verify BPF application */ - skel_paf = paf_bpf__open(); - if (!skel_paf) { - fprintf(stderr, "Failed to open and load BPF skeleton\n"); - return 1; - } +// Functions to process different BPF programs +static int process_paf(struct paf_bpf *skel_paf) { + int err; + struct ring_buffer *rb; - skel_paf->bss->user_pid = own_pid; + LOAD_AND_ATTACH_SKELETON(skel_paf, paf); - /* Load & verify BPF programs */ - err = paf_bpf__load(skel_paf); - if (err) { - fprintf(stderr, "Failed to load and verify BPF skeleton\n"); - goto paf_cleanup; - } + printf("%-8s %-8s %-8s %-8s %-8s\n", "MIN", "LOW", "HIGH", "PRESENT", "FLAG"); - /* Attach tracepoints */ - err = paf_bpf__attach(skel_paf); - if (err) { - fprintf(stderr, "Failed to attach BPF skeleton\n"); - goto paf_cleanup; - } + POLL_RING_BUFFER(rb, 1000, err); - /* Set up ring buffer polling */ - rb = ring_buffer__new(bpf_map__fd(skel_paf->maps.rb), handle_event_paf, NULL, NULL); - if (!rb) { - err = -1; - fprintf(stderr, "Failed to create ring buffer\n"); - goto paf_cleanup; - } +paf_cleanup: + ring_buffer__free(rb); + paf_bpf__destroy(skel_paf); + return err; +} - /* Process events */ - printf("%-8s %-8s %-8s %-8s %-8s\n", "MIN", "LOW", "HIGH", "PRESENT", "FLAG"); - } - else if (env.pr) { - /* Load and verify BPF application */ - skel_pr = pr_bpf__open(); - if (!skel_pr) { - fprintf(stderr, "Failed to open and load BPF skeleton\n"); - return 1; - } +static int process_pr(struct pr_bpf *skel_pr) { + int err; + struct ring_buffer *rb; - skel_pr->bss->user_pid = own_pid; + LOAD_AND_ATTACH_SKELETON(skel_pr, pr); - /* Load & verify BPF programs */ - err = pr_bpf__load(skel_pr); - if (err) { - fprintf(stderr, "Failed to load and verify BPF skeleton\n"); - goto pr_cleanup; - } + printf("%-8s %-8s %-8s %-8s %-8s\n", "RECLAIM", "RECLAIMED", "UNQUEUE", "CONGESTED", "WRITEBACK"); - /* Attach tracepoints */ - err = pr_bpf__attach(skel_pr); - if (err) { - fprintf(stderr, "Failed to attach BPF skeleton\n"); - goto pr_cleanup; - } + POLL_RING_BUFFER(rb, 1000, err); - /* Set up ring buffer polling */ - rb = ring_buffer__new(bpf_map__fd(skel_pr->maps.rb), handle_event_pr, NULL, NULL); - if (!rb) { - err = -1; - fprintf(stderr, "Failed to create ring buffer\n"); - goto pr_cleanup; - } +pr_cleanup: + ring_buffer__free(rb); + pr_bpf__destroy(skel_pr); + return err; +} - /* Process events */ - printf("%-8s %-8s %-8s %-8s %-8s\n", "RECLAIM", "RECLAIMED", "UNQUEUE", "CONGESTED", "WRITEBACK"); - } +static int process_procstat(struct procstat_bpf *skel_procstat) { + int err; + struct ring_buffer *rb; - else if (env.procstat) { - /* Load and verify BPF application */ - skel_procstat = procstat_bpf__open(); - if (!skel_procstat) { - fprintf(stderr, "Failed to open and load BPF skeleton\n"); - return 1; - } + LOAD_AND_ATTACH_SKELETON(skel_procstat, procstat); - skel_procstat->bss->user_pid = own_pid; + if (env.rss) { + printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "TIME", "PID", "VMSIZE", "VMDATA", "VMSTK", "VMPTE", "VMSWAP"); + } else { + printf("%-8s %-8s %-8s %-8s %-8s %-8s\n", "TIME", "PID", "SIZE", "RSSANON", "RSSFILE", "RSSSHMEM"); + } - /* Load & verify BPF programs */ - err = procstat_bpf__load(skel_procstat); - if (err) { - fprintf(stderr, "Failed to load and verify BPF skeleton\n"); - goto procstat_cleanup; - } + POLL_RING_BUFFER(rb, 1000, err); - /* Attach tracepoints */ - err = procstat_bpf__attach(skel_procstat); - if (err) { - fprintf(stderr, "Failed to attach BPF skeleton\n"); - goto procstat_cleanup; - } +procstat_cleanup: + ring_buffer__free(rb); + procstat_bpf__destroy(skel_procstat); + return err; +} - /* Set up ring buffer polling */ - rb = ring_buffer__new(bpf_map__fd(skel_procstat->maps.rb), handle_event_procstat, NULL, NULL); - if (!rb) { - err = -1; - fprintf(stderr, "Failed to create ring buffer\n"); - goto procstat_cleanup; - } +static int process_sysstat(struct sysstat_bpf *skel_sysstat) { + int err; + struct ring_buffer *rb; - /* Process events */ - if (env.rss == true) { - printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "TIME", "PID", "VMSIZE", "VMDATA", "VMSTK", "VMPTE", "VMSWAP"); - } - else { - printf("%-8s %-8s %-8s %-8s %-8s %-8s\n", "TIME", "PID", "SIZE", "RSSANON", "RSSFILE", "RSSSHMEM"); - } - } + LOAD_AND_ATTACH_SKELETON(skel_sysstat, sysstat); - else if (env.sysstat) { - /* Load and verify BPF application */ - skel_sysstat = sysstat_bpf__open(); - if (!skel_sysstat) { - fprintf(stderr, "Failed to open and load BPF skeleton\n"); - return 1; - } + if (env.part2) { + printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "KRECLM", "SLAB", "SRECLM", "SUNRECL", "NFSUNSTB", "WRITEBACKTMP", "KMAP", "UNMAP", "PAGE"); + } else { + printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "TIME", "PID", "CPU", "MEM", "READ", "WRITE", "IOWAIT", "SWAP"); + } + POLL_RING_BUFFER(rb, 1000, err); - skel_sysstat->bss->user_pid = own_pid; +sysstat_cleanup: + ring_buffer__free(rb); + sysstat_bpf__destroy(skel_sysstat); + return err; +} - /* Load & verify BPF programs */ - err = sysstat_bpf__load(skel_sysstat); - if (err) { - fprintf(stderr, "Failed to load and verify BPF skeleton\n"); - goto sysstat_cleanup; - } +static int process_memleak(struct memleak_bpf *skel_memleak, struct env env) { + skel_memleak->rodata->stack_flags = env.kernel_trace ? KERN_STACKID_FLAGS : USER_STACKID_FLAGS; - /* Attach tracepoints */ - err = sysstat_bpf__attach(skel_sysstat); - if (err) { - fprintf(stderr, "Failed to attach BPF skeleton\n"); - goto sysstat_cleanup; - } + bpf_map__set_value_size(skel_memleak->maps.stack_traces, perf_max_stack_depth * sizeof(__u64)); + bpf_map__set_max_entries(skel_memleak->maps.stack_traces, stack_map_max_entries); - /* Set up ring buffer polling */ - rb = ring_buffer__new(bpf_map__fd(skel_sysstat->maps.rb), handle_event_sysstat, NULL, NULL); - if (!rb) { - err = -1; - fprintf(stderr, "Failed to create ring buffer\n"); - goto sysstat_cleanup; - } + if (!env.kernel_trace) + disable_kernel_tracepoints(skel_memleak); - /* Process events */ - if (env.part2 == true) { - printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "KRECLM", "SLAB", "SRECLM", "SUNRECLMA", "UNSTABLE", "WRITEBK_T", "ANONHUGE", "SHMEMHUGE", "PMDMAPP"); - } - else { - printf("%-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s %-8s\n", "ACTIVE", "INACTVE", "ANON_ACT", "ANON_INA", "FILE_ACT", "FILE_INA", "UNEVICT", "DIRTY", "WRITEBK", "ANONPAG", "MAP", "SHMEM"); - } + int err = memleak_bpf__load(skel_memleak); + if (err) { + fprintf(stderr, "Failed to load BPF skeleton\n"); + goto memleak_cleanup; } - else if (env.memleak) { - if (env.choose_pid != 0) { - printf("用户态内存泄漏\n"); - env.kernel_trace = false; - attach_pid = env.choose_pid; - } - else - attach_pid = 0; - - strcpy(binary_path, "/lib/x86_64-linux-gnu/libc.so.6"); - - allocs = calloc(ALLOCS_MAX_ENTRIES, sizeof(*allocs)); - - /* Set up libbpf errors and debug info callback */ - libbpf_set_print(libbpf_print_fn); - - /* Load and verify BPF application */ - skel = memleak_bpf__open(); - if (!skel) { - fprintf(stderr, "Failed to open BPF skeleton\n"); - return 1; - } - //skel->rodata->stack_flags = KERN_STACKID_FLAGS; - skel->rodata->stack_flags = env.kernel_trace ? KERN_STACKID_FLAGS : USER_STACKID_FLAGS; - - bpf_map__set_value_size(skel->maps.stack_traces, perf_max_stack_depth * sizeof(__u64)); - bpf_map__set_max_entries(skel->maps.stack_traces, stack_map_max_entries); - - if (!env.kernel_trace) - disable_kernel_tracepoints(skel); - - /* Load & verify BPF programs */ - err = memleak_bpf__load(skel); + if (!env.kernel_trace) { + err = attach_uprobes(skel_memleak); if (err) { - fprintf(stderr, "Failed to load BPF skeleton\n"); + fprintf(stderr, "Failed to attach uprobes\n"); goto memleak_cleanup; } + } - if (!env.kernel_trace) { - err = attach_uprobes(skel); - if (err) { - fprintf(stderr, "failed to attach uprobes\n"); - - goto memleak_cleanup; - } - } - - /* Let libbpf perform auto-attach for uprobe_sub/uretprobe_sub - * NOTICE: we provide path and symbol info in SEC for BPF programs - */ - err = memleak_bpf__attach(skel); - if (err) { - fprintf(stderr, "Failed to auto-attach BPF skeleton: %d\n", err); - goto memleak_cleanup; - } + err = memleak_bpf__attach(skel_memleak); + if (err) { + fprintf(stderr, "Failed to auto-attach BPF skeleton: %d\n", err); + goto memleak_cleanup; + } - g_stacks_size = perf_max_stack_depth * sizeof(*g_stacks); - g_stacks = (__u64 *)malloc(g_stacks_size); - memset(g_stacks, 0, g_stacks_size); + g_stacks_size = perf_max_stack_depth * sizeof(*g_stacks); + g_stacks = (__u64 *)malloc(g_stacks_size); + if (!g_stacks) { + fprintf(stderr, "Failed to allocate memory\n"); + err = -1; + goto memleak_cleanup; + } + memset(g_stacks, 0, g_stacks_size); - symbolizer = blaze_symbolizer_new(); - if (!symbolizer) { - fprintf(stderr, "Fail to create a symbolizer\n"); - err = -1; - goto memleak_cleanup; - } + symbolizer = blaze_symbolizer_new(); + if (!symbolizer) { + fprintf(stderr, "Fail to create a symbolizer\n"); + err = -1; + goto memleak_cleanup; + } - for (i = 0;; i++) { - /* trigger our BPF programs */ - if (!env.kernel_trace) - print_outstanding_combined_allocs(skel, attach_pid); - else - print_outstanding_allocs(skel); + for (;;) { + if (!env.kernel_trace) + print_outstanding_combined_allocs(skel_memleak, attach_pid); + else + print_outstanding_allocs(skel_memleak); - sleep(1); - } + sleep(1); } while (!exiting) { - if (env.paf || env.pr || env.procstat || env.sysstat) { - err = ring_buffer__poll(rb, 1000 /* timeout, ms */); - /* Ctrl-C will cause -EINTR */ - if (err == -EINTR) { - err = 0; - break; - } - if (err < 0) { - printf("Error polling perf buffer: %d\n", err); - break; - } - } - else if (env.memleak) { - /* Ctrl-C will cause -EINTR */ - if (err == -EINTR) { - err = 0; - break; - } - if (err < 0) { - printf("Error polling perf buffer: %d\n", err); - break; - } + /* Ctrl-C will cause -EINTR */ + if (err == -EINTR) { + err = 0; + break; } - else { - printf("请输入要使用的功能...\n"); + if (err < 0) { + printf("Error polling perf buffer: %d\n", err); break; } } -paf_cleanup: - ring_buffer__free(rb); - paf_bpf__destroy(skel_paf); - return err < 0 ? -err : 0; - -pr_cleanup: - ring_buffer__free(rb); - pr_bpf__destroy(skel_pr); - return err < 0 ? -err : 0; - -procstat_cleanup: - ring_buffer__free(rb); - procstat_bpf__destroy(skel_procstat); - return err < 0 ? -err : 0; - -sysstat_cleanup: - ring_buffer__free(rb); - sysstat_bpf__destroy(skel_sysstat); - return err < 0 ? -err : 0; - memleak_cleanup: - memleak_bpf__destroy(skel); - blaze_symbolizer_free(symbolizer); - free(g_stacks); - return err < 0 ? -err : 0; + memleak_bpf__destroy(skel_memleak); + if (symbolizer) + blaze_symbolizer_free(symbolizer); + if (g_stacks) + free(g_stacks); + if (allocs) + free(allocs); + return err; } \ No newline at end of file From 2df6e9d283a6672b1e5b64f89ba45b96e4dd96fa Mon Sep 17 00:00:00 2001 From: vvzxy <145555693+vvzxy@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:09:06 +0800 Subject: [PATCH 16/16] =?UTF-8?q?cpu=5Fwatcher=EF=BC=9Aschedule=5Fdelay?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=98=88=E5=80=BC=E9=80=89=E9=A1=B9&&?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0controller=E5=8A=9F=E8=83=BD=20(#813)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cpu_watcher：schedule_delay增加阈值选项&&workflow增加测试流程 * . * cpu_watcher：增加controller功能 --- .github/workflows/ebpf_cpu_watcher.yml | 3 +- .../CPU_Subsystem/cpu_watcher/Makefile | 13 +- .../cpu_watcher/bpf/cs_delay.bpf.c | 3 +- .../cpu_watcher/bpf/mq_delay.bpf.c | 3 +- .../cpu_watcher/bpf/schedule_delay.bpf.c | 20 ++ .../CPU_Subsystem/cpu_watcher/controller.c | 250 ++++++++++++++++++ .../CPU_Subsystem/cpu_watcher/cpu_watcher.c | 196 +++++++++++--- .../cpu_watcher/include/cpu_watcher.h | 41 ++- .../cpu_watcher/include/cpu_watcher_helper.h | 152 ++++++++++- .../cpu_watcher/test/test_cpuwatcher.c | 5 +- 10 files changed, 641 insertions(+), 45 deletions(-) create mode 100644 eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c diff --git a/.github/workflows/ebpf_cpu_watcher.yml b/.github/workflows/ebpf_cpu_watcher.yml index 8b9f5bd32..41364689f 100644 --- a/.github/workflows/ebpf_cpu_watcher.yml +++ b/.github/workflows/ebpf_cpu_watcher.yml @@ -31,8 +31,9 @@ jobs: cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/ make sudo ./cpu_watcher + + - name: Run test_cpuwatcher - - name: Run test_cpuwatcher run: | cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test make diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile index 8d20c7c63..bcc7c1236 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile @@ -44,6 +44,7 @@ ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) APPS =cs_delay sar sc_delay preempt schedule_delay mq_delay TARGETS=cpu_watcher +CONTROLLER := controller SRC_DIR = ./include @@ -81,12 +82,12 @@ $(call allow-override,CC,$(CROSS_COMPILE)cc) $(call allow-override,LD,$(CROSS_COMPILE)ld) .PHONY: all -all: $(TARGETS) +all: $(CONTROLLER) $(TARGETS) .PHONY: clean clean: $(call msg,CLEAN) - $(Q)rm -rf $(OUTPUT) $(TARGETS) + $(Q)rm -rf $(OUTPUT) $(TARGETS) $(CONTROLLER) $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT): $(call msg,MKDIR,$@) @@ -132,11 +133,19 @@ $(OUTPUT)/%.o: $(SRC_DIR)/%.c | $(OUTPUT) $(call msg,CC,$@) $(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ +$(OUTPUT)/%.o: $(CONTROLLER).c | $(OUTPUT) + $(call msg,CC,$@) + $(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ + $(OUTPUT)/$(TARGETS).o: $(TARGETS).c $(APPS) | $(OUTPUT) $(call msg,CC,$@) $(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@ # Build application binary +$(CONTROLLER): %: $(OUTPUT)/%.o $(COMMON_OBJ) $(LIBBPF_OBJ) | $(OUTPUT) + $(call msg,BINARY,$@) + $(Q)$(CC) $^ $(ALL_LDFLAGS) -lstdc++ -lelf -lz -o $@ + $(TARGETS): %: $(OUTPUT)/%.o $(COMMON_OBJ) $(LIBBPF_OBJ) | $(OUTPUT) $(call msg,BINARY,$@) $(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lstdc++ -lelf -lz -o $@ diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c index 3434d6a8a..4770a3c15 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c @@ -21,9 +21,10 @@ #include "cpu_watcher.h" char LICENSE[] SEC("license") = "Dual BSD/GPL"; - +const int ctrl_key = 0; //记录时间戳； BPF_ARRAY(start,int,u64,1); +BPF_ARRAY(cs_ctrl_map,int,struct cs_ctrl,1); struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c index becb4c27c..b86631002 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c @@ -23,10 +23,11 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; - +const int ctrl_key = 0; BPF_HASH(send_msg1,pid_t,struct send_events,1024);//记录pid->u_msg_ptr的关系；do_mq_timedsend入参 BPF_HASH(send_msg2,u64,struct send_events,1024);//记录msg->time的关系； BPF_HASH(rcv_msg1,pid_t,struct rcv_events,1024);//记录pid->u_msg_ptr的关系；do_mq_timedsend入参 +BPF_ARRAY(mq_ctrl_map,int,struct mq_ctrl,1); struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c index c5e2c7d22..596158c6e 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c @@ -23,13 +23,19 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; #define TASK_RUNNING 0x0000 +const int ctrl_key = 0; BPF_HASH(has_scheduled,struct proc_id, bool, 10240);//记录该进程是否调度过 BPF_HASH(enter_schedule,struct proc_id, struct schedule_event, 10240);//记录该进程上运行队列的时间 BPF_ARRAY(sys_schedule,int,struct sum_schedule,1);//记录整个系统的调度延迟 BPF_ARRAY(threshold_schedule,int,struct proc_schedule,10240);//记录每个进程的调度延迟 +BPF_ARRAY(schedule_ctrl_map,int,struct schedule_ctrl,1); SEC("tp_btf/sched_wakeup") int BPF_PROG(sched_wakeup, struct task_struct *p) { + struct schedule_ctrl *sched_ctrl; + sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); + if(!sched_ctrl || !sched_ctrl->schedule_func) + return 0; pid_t pid = p->pid; int cpu = bpf_get_smp_processor_id(); struct schedule_event *schedule_event; @@ -56,6 +62,10 @@ int BPF_PROG(sched_wakeup, struct task_struct *p) { SEC("tp_btf/sched_wakeup_new") int BPF_PROG(sched_wakeup_new, struct task_struct *p) { + struct schedule_ctrl *sched_ctrl; + sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); + if(!sched_ctrl || !sched_ctrl->schedule_func) + return 0; pid_t pid = p->pid; int cpu = bpf_get_smp_processor_id(); struct proc_id id= {}; @@ -76,6 +86,11 @@ int BPF_PROG(sched_wakeup_new, struct task_struct *p) { SEC("tp_btf/sched_switch") int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next) { + struct schedule_ctrl *sched_ctrl; + sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); + if(!sched_ctrl || !sched_ctrl->schedule_func) + return 0; + u64 current_time = bpf_ktime_get_ns(); pid_t prev_pid = prev->pid; unsigned int prev_state = prev->__state; @@ -162,6 +177,11 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s SEC("tracepoint/sched/sched_process_exit") int sched_process_exit(void *ctx) { + struct schedule_ctrl *sched_ctrl; + sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); + if(!sched_ctrl || !sched_ctrl->schedule_func) + return 0; + struct task_struct *p = (struct task_struct *)bpf_get_current_task(); pid_t pid = BPF_CORE_READ(p, pid); int cpu = bpf_get_smp_processor_id(); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c new file mode 100644 index 000000000..b02a986ce --- /dev/null +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c @@ -0,0 +1,250 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: albert_xuu@163.com zhangxy1016304@163.com zhangziheng0525@163.com +// +// used to control the execution of proc_image tool +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cpu_watcher_helper.h" + +static struct env { + // 1代表activate；2代表unactivate；3代表finish + int usemode; + bool SAR; + bool CS_DELAY; + bool SYSCALL_DELAY; + bool MIN_US_SET; + int MIN_US; + bool PREEMPT; + bool SCHEDULE_DELAY; + bool MQ_DELAY; + int freq; +} env = { + .usemode = 0, + .SAR = false, + .CS_DELAY = false, + .SYSCALL_DELAY = false, + .MIN_US_SET = false, + .MIN_US = 10000, + .PREEMPT = false, + .SCHEDULE_DELAY = false, + .MQ_DELAY = false, + .freq = 99, +}; + +const char argp_program_doc[] ="Trace process to get cpu watcher.\n"; + +static const struct argp_option opts[] = { + { "activate", 'a', NULL, 0, "Set startup policy of proc_image tool" }, + { "unactivate", 'u', NULL, 0, "Initialize to the original unactivated state" }, + { "finish", 'f', NULL, 0, "Finish to run eBPF tool" }, + {"libbpf_sar", 's', 0, 0, "Print sar_info (the data of cpu)" }, + {"cs_delay", 'c', 0, 0, "Print cs_delay (the data of cpu)" }, + {"syscall_delay", 'S', 0, 0, "Print syscall_delay (the data of syscall)" }, + {"preempt_time", 'p', 0, 0, "Print preempt_time (the data of preempt_schedule)" }, + {"schedule_delay", 'd', 0, 0, "Print schedule_delay (the data of cpu)" }, + {"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" }, + {"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" }, + // { "pid", 'p', "PID", 0, "Process ID to trace" }, + // { "tgid", 'P', "TGID", 0, "Thread group to trace" }, + // { "cpuid", 'c', "CPUID", 0, "Set For Tracing per-CPU Process(other processes don't need to set this parameter)" }, + // { "time", 't', "TIME-SEC", 0, "Max Running Time(0 for infinite)" }, + // { "myproc", 'm', NULL, 0, "Trace the process of the tool itself (not tracked by default)" }, + // { "resource", 'r', NULL, 0, "Collects resource usage information about processes" }, + // { "keytime", 'k', "KEYTIME", 0, "Collects keytime information about processes(0:except CPU kt_info,1:all kt_info,any 0 or 1 when deactivated)" }, + // { "lock", 'l', NULL, 0, "Collects lock information about processes" }, + // { "syscall", 's', "SYSCALLS", 0, "Collects syscall sequence (1~50) information about processes(any 1~50 when deactivated)" }, + // { "schedule", 'S', NULL, 0, "Collects schedule information about processes (trace tool process)" }, + { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case 'a': + env.usemode = 1; + break; + case 'u': + env.usemode = 2; + break; + case 'f': + env.usemode = 3; + break; + case 's': + env.SAR = true; + break; + case 'c': + env.CS_DELAY = true; + break; + case 'S': + env.SYSCALL_DELAY = true; + break; + case 'p': + env.PREEMPT = true; + break; + case 'd': + env.SCHEDULE_DELAY = true; + break; + case 'e': + env.MIN_US_SET = true; + if (arg) { + env.MIN_US = strtol(arg, NULL, 10); + if (env.MIN_US <= 0) { + fprintf(stderr, "Invalid value for min_us: %d\n", env.MIN_US); + argp_usage(state); + } + } else { + env.MIN_US = 10000; + } + break; + case 'm': + env.MQ_DELAY = true; + break; + case 'h': + argp_state_help(state, stderr, ARGP_HELP_STD_HELP); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +int deactivate_mode(){ + int err; + + if(env.SAR){ + struct sar_ctrl sar_ctrl = {false,0}; + err = update_sar_ctrl_map(sar_ctrl); + if(err < 0) return err; + } + if(env.CS_DELAY){ + struct cs_ctrl cs_ctrl = {false,0}; + err = update_cs_ctrl_map(cs_ctrl); + if(err < 0) return err; + } + if(env.SYSCALL_DELAY){ + struct sc_ctrl sc_ctrl = {false,0}; + err = update_sc_ctrl_map(sc_ctrl); + if(err < 0) return err; + } + if(env.PREEMPT){ + struct preempt_ctrl preempt_ctrl = {false,0}; + err = update_preempt_ctrl_map(preempt_ctrl); + if(err < 0) return err; + } + if(env.SCHEDULE_DELAY){ + struct schedule_ctrl schedule_ctrl = {false,false,10000,0}; + err = update_schedule_ctrl_map(schedule_ctrl); + if(err < 0) return err; + } + if(env.MQ_DELAY){ + struct mq_ctrl mq_ctrl = {false,0}; + err = update_mq_ctrl_map(mq_ctrl); + if(err < 0) return err; + } + return 0; +} + +static void sig_handler(int signo) +{ + deactivate_mode(); +} + +int main(int argc, char **argv) +{ + int err; + static const struct argp argp = { + .options = opts, + .parser = parse_arg, + .doc = argp_program_doc, + }; + + err = argp_parse(&argp, argc, argv, 0, NULL, NULL); + if (err) + return err; + + signal(SIGALRM,sig_handler); + signal(SIGINT,sig_handler); + signal(SIGTERM,sig_handler); + + if(env.usemode == 1){ // activate mode + if(env.SAR){ + struct sar_ctrl sar_ctrl = {true,SAR_WACTHER}; + err = update_sar_ctrl_map(sar_ctrl); + if(err < 0) return err; + } + + if(env.CS_DELAY){ + struct cs_ctrl cs_ctrl = {true,CS_WACTHER}; + err = update_cs_ctrl_map(cs_ctrl); + if(err < 0) return err; + } + + if(env.SYSCALL_DELAY){ + struct sc_ctrl sc_ctrl = {true,SC_WACTHER}; + err = update_sc_ctrl_map(sc_ctrl); + if(err < 0) return err; + } + + if(env.PREEMPT){ + struct preempt_ctrl preempt_ctrl = {true,PREEMPT_WACTHER}; + err = update_preempt_ctrl_map(preempt_ctrl); + if(err < 0) return err; + } + + if(env.SCHEDULE_DELAY){ + /* + *1.未设置env.MIN_US_SET时, prev_watcher = SCHEDULE_WACTHER + 0;输出方式为schedule输出 + *2.已设置env.MIN_US_SET时, prev_watcher = SCHEDULE_WACTHER + 1;输出方式为-e输出 + */ + struct schedule_ctrl schedule_ctrl = {true,env.MIN_US_SET,env.MIN_US,SCHEDULE_WACTHER+env.MIN_US_SET}; + err = update_schedule_ctrl_map(schedule_ctrl); + if(err < 0) return err; + } + + if(env.MQ_DELAY){ + struct mq_ctrl mq_ctrl = {true,MQ_WACTHER}; + err = update_mq_ctrl_map(mq_ctrl); + if(err < 0) return err; + } + }else if(env.usemode == 2){ // deactivate mode + err = deactivate_mode(); + if(err<0){ + fprintf(stderr, "Failed to deactivate\n"); + return err; + } + }else if(env.usemode == 3){ // finish mode + const char *command = "pkill cpu_watcher"; + int status = system(command); + if (status == -1) { + perror("system"); + } + }else{ + // 输出help信息 + printf("Please enter the usage mode(activate/deactivate/finish) before selecting the function\n"); + argp_help(&argp, stderr, ARGP_HELP_LONG, argv[0]); + } + + return 0; +} diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index 8605fc18a..23114d93c 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -28,7 +28,6 @@ #include #include #include -#include "cpu_watcher.h" #include "cpu_watcher_helper.h" #include "sar.skel.h" #include "cs_delay.skel.h" @@ -96,6 +95,15 @@ struct preempt_bpf *preempt_skel; struct schedule_delay_bpf *sd_skel; struct mq_delay_bpf *mq_skel; +static int csmap_fd; +static int sarmap_fd; +static int scmap_fd; +static int preemptmap_fd; +static int schedulemap_fd; +static int mqmap_fd; + +//static int prev_watcher = 0;//上一个使用的工具，用于在切换使用功能时，打印不用功能的表头； + u64 softirq = 0; u64 irqtime = 0; u64 idle = 0; @@ -128,7 +136,6 @@ static const struct argp_option opts[] = { {"syscall_delay", 'S', 0, 0, "Print syscall_delay (the data of syscall)" }, {"preempt_time", 'p', 0, 0, "Print preempt_time (the data of preempt_schedule)" }, {"schedule_delay", 'd', 0, 0, "Print schedule_delay (the data of cpu)" }, - {"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" }, {"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" }, {"ewma", 'E',0,0,"dynamic filte the data"}, {"cycle", 'T',"CYCLE",0,"Periods of the ewma"}, @@ -164,18 +171,6 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'd': env.SCHEDULE_DELAY = true; break; - case 'e': - env.MIN_US_SET = true; - if (arg) { - env.MIN_US = strtol(arg, NULL, 10); - if (env.MIN_US <= 0) { - fprintf(stderr, "Invalid value for min_us: %d\n", env.MIN_US); - argp_usage(state); - } - } else { - env.MIN_US = 10000; - } - break; case 'm': env.MQ_DELAY = true; break; @@ -495,22 +490,39 @@ static void histogram() struct ewma_info ewma_syscall_delay = {}; static int syscall_delay_print(void *ctx, void *data,unsigned long data_sz) { + int err,key = 0; + struct sc_ctrl sc_ctrl ={}; + + err = bpf_map_lookup_elem(scmap_fd,&key,&sc_ctrl); + if (err < 0) { + fprintf(stderr, "failed to lookup infos: %d\n", err); + return -1; + } + if(!sc_ctrl.sc_func) return 0; + const struct syscall_events *e = data; + if(e->delay<0||e->delay>1000000) return 0; + time_t now = time(NULL);// 获取当前时间 + struct tm *localTime = localtime(&now);// 将时间转换为本地时间结构 + if(env.EWMA==0){ - printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", - e->pid,e->comm,e->syscall_id,e->delay); + printf("%02d:%02d:%02d %-8u %-15lld %-15lld\n", + localTime->tm_hour, localTime->tm_min, localTime->tm_sec, + e->pid,e->syscall_id,e->delay); } else{ ewma_syscall_delay.cycle = env.cycle; if(dynamic_filter(&ewma_syscall_delay,e->delay)){ - printf("pid: %-8u comm: %-10s syscall_id: %-8lld delay: %-8lld\n", - e->pid,e->comm,e->syscall_id,e->delay); + printf("%02d:%02d:%02d %-8u %-15lld %-15lld\n", + localTime->tm_hour, localTime->tm_min, localTime->tm_sec, + e->pid,e->syscall_id,e->delay); } } return 0; } + //抢占时间输出 static int preempt_print(void *ctx, void *data, unsigned long data_sz) { @@ -555,8 +567,36 @@ void add_entry(int pid, const char *comm, long long delay) { } static int schedule_print() { - int key = 0; - if(env.SCHEDULE_DELAY){ + + int err,key = 0; + struct schedule_ctrl sd_ctrl = {}; + err = bpf_map_lookup_elem(schedulemap_fd,&key,&sd_ctrl); + if (err < 0) { + fprintf(stderr, "failed to lookup infos: %d\n", err); + return -1; + } + if(!sd_ctrl.schedule_func) return 0; + + if(sd_ctrl.prev_watcher == SCHEDULE_WACTHER ){ + printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs max_proc_name min_delay/μs min_proc_name"); + sd_ctrl.prev_watcher = SCHEDULE_WACTHER + 9;//打印表头功能关 + err = bpf_map_update_elem(schedulemap_fd, &key, &sd_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + } + } + else if(sd_ctrl.prev_watcher == SCHEDULE_WACTHER +1){ + // printf("sd_ctrl.prev_watcher = %d\n",sd_ctrl.prev_watcher); + printf("调度延时大于%dms的进程:\n",sd_ctrl.min_us/1000); + printf("%s\n","pid COMM schedule_delay/us"); + sd_ctrl.prev_watcher = SCHEDULE_WACTHER + 9;//打印表头功能关. + err = bpf_map_update_elem(schedulemap_fd, &key, &sd_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + } + } + + if(!sd_ctrl.min_us_set){ struct sum_schedule info; int err, fd = bpf_map__fd(sd_skel->maps.sys_schedule); time_t now = time(NULL); @@ -577,7 +617,8 @@ static int schedule_print() printf("%02d:%02d:%02d %-15lf %-15lf %10s %15lf %15s\n", hour, min, sec, avg_delay / 1000.0, info.max_delay / 1000.0,info.proc_name_max,info.min_delay / 1000.0,info.proc_name_min); } - }else if(env.MIN_US_SET){ + } + else{ struct proc_schedule info; int key = 0; int err, fd = bpf_map__fd(sd_skel->maps.threshold_schedule); @@ -586,13 +627,14 @@ static int schedule_print() fprintf(stderr, "failed to lookup infos: %d\n", err); return -1; } - if (info.delay / 1000 > env.MIN_US&&info.pid!=0) { // 默认输出调度延迟大于10ms的 + if (info.delay / 1000>sd_ctrl.min_us&&info.pid!=0) { if (!entry_exists(info.pid, info.proc_name, info.delay / 1000)) { printf("%-10d %-16s %15lld\n", info.pid, info.proc_name, info.delay / 1000); add_entry(info.pid, info.proc_name, info.delay / 1000); } } } + return 0; } @@ -627,6 +669,13 @@ static int mq_event(void *ctx, void *data,unsigned long data_sz) int main(int argc, char **argv) { struct ring_buffer *rb = NULL; + struct bpf_map *cs_ctrl_map = NULL; + struct bpf_map *sar_ctrl_map = NULL; + struct bpf_map *sc_ctrl_map = NULL; + struct bpf_map *preempt_ctrl_map = NULL; + struct bpf_map *schedule_ctrl_map = NULL; + struct bpf_map *mq_ctrl_map = NULL; + int key = 0; int err; err = argp_parse(&argp, argc, argv, 0, NULL, NULL); if (err) @@ -668,6 +717,19 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to load and verify BPF skeleton\n"); goto cs_delay_cleanup; } + + err = common_pin_map(&cs_ctrl_map,cs_skel->obj,"cs_ctrl_map",cs_ctrl_path); + if(err < 0){ + goto cs_delay_cleanup; + } + csmap_fd = bpf_map__fd(cs_ctrl_map); + struct cs_ctrl init_value = {false,CS_WACTHER}; + err = bpf_map_update_elem(csmap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto cs_delay_cleanup; + } + /* Attach tracepoints */ err = cs_delay_bpf__attach(cs_skel); if (err) @@ -694,6 +756,17 @@ int main(int argc, char **argv) goto preempt_cleanup; } + err = common_pin_map(&preempt_ctrl_map,preempt_skel->obj,"preempt_ctrl_map",preempt_ctrl_path); + if(err < 0){ + goto preempt_cleanup; + } + preemptmap_fd = bpf_map__fd(preempt_ctrl_map); + struct preempt_ctrl init_value = {false,PREEMPT_WACTHER}; + err = bpf_map_update_elem(preemptmap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto preempt_cleanup; + } err = preempt_bpf__attach(preempt_skel); if (err) { fprintf(stderr, "Failed to attach BPF skeleton\n"); @@ -721,6 +794,17 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to load and verify BPF skeleton\n"); goto sc_delay_cleanup; } + err = common_pin_map(&sc_ctrl_map,sc_skel->obj,"sc_ctrl_map",sc_ctrl_path); + if(err < 0){ + goto sc_delay_cleanup; + } + scmap_fd = bpf_map__fd(sc_ctrl_map); + struct sc_ctrl init_value = {false,SC_WACTHER}; + err = bpf_map_update_elem(scmap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto sc_delay_cleanup; + } /* Attach tracepoints */ err = sc_delay_bpf__attach(sc_skel); if (err) @@ -728,13 +812,18 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to attach BPF skeleton\n"); goto sc_delay_cleanup; } + printf("%-8s %-8s %-15s %-15s\n","Time","Pid","syscall_id","delay/ms"); rb = ring_buffer__new(bpf_map__fd(sc_skel->maps.rb), syscall_delay_print, NULL, NULL); //ring_buffer__new() API，允许在不使用额外选项数据结构下指定回调 if (!rb) { err = -1; fprintf(stderr, "Failed to create ring buffer\n"); goto sc_delay_cleanup; } - }else if(env.SCHEDULE_DELAY||env.MIN_US_SET){ + + + }else if(env.SCHEDULE_DELAY){ + + sd_skel = schedule_delay_bpf__open(); if (!sd_skel) { fprintf(stderr, "Failed to open and load BPF skeleton\n"); @@ -745,17 +834,23 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to load and verify BPF skeleton\n"); goto schedule_cleanup; } + err = common_pin_map(&schedule_ctrl_map,sd_skel->obj,"schedule_ctrl_map",schedule_ctrl_path); + if(err < 0){ + goto schedule_cleanup; + } + schedulemap_fd = bpf_map__fd(schedule_ctrl_map); + struct schedule_ctrl init_value = {false,false,10000,SCHEDULE_WACTHER}; + + err = bpf_map_update_elem(schedulemap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto schedule_cleanup; + } err = schedule_delay_bpf__attach(sd_skel); if (err) { fprintf(stderr, "Failed to attach BPF skeleton\n"); goto schedule_cleanup; } - if(env.MIN_US_SET){ - printf("调度延时大于%dms的进程:\n",env.MIN_US/1000); - printf("%s\n","pid COMM schedule_delay/us"); - }else{ - printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs max_proc_name min_delay/μs min_proc_name"); - } }else if (env.SAR){ /* Load and verify BPF application */ sar_skel = sar_bpf__open(); @@ -778,6 +873,18 @@ int main(int argc, char **argv) if (err) goto sar_cleanup; + err = common_pin_map(&sar_ctrl_map,sar_skel->obj,"sar_ctrl_map",sar_ctrl_path); + if(err < 0){ + goto sar_cleanup; + } + sarmap_fd = bpf_map__fd(sar_ctrl_map); + struct sar_ctrl init_value = {false,SAR_WACTHER}; + err = bpf_map_update_elem(sarmap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto sar_cleanup; + } + err = sar_bpf__attach(sar_skel); if (err) { @@ -802,6 +909,19 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to load and verify BPF skeleton\n"); goto mq_delay_cleanup; } + + err = common_pin_map(&mq_ctrl_map,mq_skel->obj,"mq_ctrl_map",mq_ctrl_path); + if(err < 0){ + goto mq_delay_cleanup; + } + mqmap_fd = bpf_map__fd(mq_ctrl_map); + struct mq_ctrl init_value = {false,MQ_WACTHER}; + err = bpf_map_update_elem(mqmap_fd, &key, &init_value, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + goto mq_delay_cleanup; + } + /* Attach tracepoints */ err = mq_delay_bpf__attach(mq_skel); if (err) @@ -854,11 +974,11 @@ int main(int argc, char **argv) printf("Error polling perf buffer: %d\n", err); break; } - time_t now = time(NULL);// 获取当前时间 - struct tm *localTime = localtime(&now);// 将时间转换为本地时间结构 - printf("\n\nTime: %02d:%02d:%02d\n",localTime->tm_hour, localTime->tm_min, localTime->tm_sec); - printf("----------------------------------------------------------------------------------------------------------\n"); - sleep(1); + // time_t now = time(NULL);// 获取当前时间 + // struct tm *localTime = localtime(&now);// 将时间转换为本地时间结构 + // printf("\n\nTime: %02d:%02d:%02d\n",localTime->tm_hour, localTime->tm_min, localTime->tm_sec); + // printf("----------------------------------------------------------------------------------------------------------\n"); + // sleep(1); } else if (env.PREEMPT) { err = ring_buffer__poll(rb, 100 /* timeout, ms */); @@ -884,7 +1004,7 @@ int main(int argc, char **argv) sum_preemptTime = 0; sleep(2); } - else if (env.SCHEDULE_DELAY||env.MIN_US_SET){ + else if (env.SCHEDULE_DELAY){ err = schedule_print(); if (err == -EINTR) { err = 0; @@ -915,29 +1035,35 @@ int main(int argc, char **argv) } cs_delay_cleanup: + bpf_map__unpin(cs_ctrl_map, cs_ctrl_path); ring_buffer__free(rb); cs_delay_bpf__destroy(cs_skel); return err < 0 ? -err : 0; sar_cleanup: + bpf_map__unpin(sar_ctrl_map, sar_ctrl_path); sar_bpf__destroy(sar_skel); return err < 0 ? -err : 0; sc_delay_cleanup: + bpf_map__unpin(sc_ctrl_map, sc_ctrl_path); ring_buffer__free(rb); sc_delay_bpf__destroy(sc_skel); return err < 0 ? -err : 0; preempt_cleanup: + bpf_map__unpin(preempt_ctrl_map, preempt_ctrl_path); ring_buffer__free(rb); preempt_bpf__destroy(preempt_skel); return err < 0 ? -err : 0; schedule_cleanup: + bpf_map__unpin(schedule_ctrl_map, schedule_ctrl_path); schedule_delay_bpf__destroy(sd_skel); return err < 0 ? -err : 0; mq_delay_cleanup: + bpf_map__unpin(mq_ctrl_map, mq_ctrl_path); ring_buffer__free(rb); mq_delay_bpf__destroy(mq_skel); return err < 0 ? -err : 0; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h index f1c35423f..00a6463ec 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h @@ -13,6 +13,8 @@ // limitations under the License. // // author: albert_xuu@163.com zhangxy1016304@163.com zhangziheng0525@163.com +#ifndef CPU_WATCHER_H +#define CPU_WATCHER_H #include #include @@ -217,4 +219,41 @@ struct idleStruct { u64 pad; unsigned int state; unsigned int cpu_id; -}; \ No newline at end of file +}; + +/*----------------------------------------------*/ +/* 控制板块 */ +/*----------------------------------------------*/ +struct sar_ctrl{ + bool sar_func; + int prev_watcher; +}; + +struct cs_ctrl{ + bool cs_func; + int prev_watcher; +}; + +struct sc_ctrl{ + bool sc_func; + int prev_watcher; +}; + +struct preempt_ctrl{ + bool preempt_func; + int prev_watcher; +}; + +struct schedule_ctrl{ + bool schedule_func; + bool min_us_set; + int min_us; + int prev_watcher; +}; + +struct mq_ctrl{ + bool mq_func; + int prev_watcher; +}; + +#endif // CPU_WATCHER_H \ No newline at end of file diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h index 409d115a5..5250aef89 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h @@ -13,8 +13,19 @@ // limitations under the License. // // author: albert_xuu@163.com zhangxy1016304@163.com zhangziheng0525@163.com +#ifndef CPU_WATCHER_HELPER_H +#define CPU_WATCHER_HELPER_H #include +#include "cpu_watcher.h" + +#define SAR_WACTHER 10 +#define CS_WACTHER 20 +#define SC_WACTHER 30 +#define PREEMPT_WACTHER 40 +#define SCHEDULE_WACTHER 50 +#define MQ_WACTHER 60 + /*----------------------------------------------*/ /* ewma算法 */ /*----------------------------------------------*/ @@ -33,7 +44,7 @@ double calculateEWMA(double previousEWMA, double dataPoint, double alpha) { } bool dynamic_filter(struct ewma_info *ewma_syscall_delay, double dataPoint) { - double alpha,ewma,threshold;; + double alpha,threshold; if(ewma_syscall_delay->cycle==0) alpha = 2.0 /(CYCLE + 1); // 计算 alpha else alpha = 2.0 /(ewma_syscall_delay->cycle + 1); @@ -51,4 +62,141 @@ bool dynamic_filter(struct ewma_info *ewma_syscall_delay, double dataPoint) { if(dataPoint >= threshold) return 1; } return 0; -} \ No newline at end of file +} + + +const char *sar_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/sar_ctrl_map"; +const char *cs_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/cs_ctrl_map"; +const char *sc_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/sc_ctrl_map"; +const char *preempt_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/preempt_ctrl_map"; +const char *schedule_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/schedule_ctrl_map"; +const char *mq_ctrl_path = "/sys/fs/bpf/cpu_watcher_map/mq_ctrl_map"; + +int common_pin_map(struct bpf_map **bpf_map, const struct bpf_object *obj, const char *map_name, const char *ctrl_path) +{ + int ret; + + *bpf_map = bpf_object__find_map_by_name(obj, map_name);//查找具有指定名称的 BPF 映射 + if (!*bpf_map) { + fprintf(stderr, "Failed to find BPF map\n"); + return -1; + } + // 用于防止上次没有成功 unpin 掉这个 map + bpf_map__unpin(*bpf_map, ctrl_path); + ret = bpf_map__pin(*bpf_map, ctrl_path); + if (ret){ + fprintf(stderr, "Failed to pin BPF map\n"); + return -1; + }//找到pin上 + + return 0; +} + +int update_sar_ctrl_map(struct sar_ctrl sar_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(sar_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open sar_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&sar_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update sar_ctrl_map elem\n"); + return err; + } + + return 0; +} + +int update_cs_ctrl_map(struct cs_ctrl cs_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(cs_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open cs_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&cs_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update cs_ctrl_map elem\n"); + return err; + } + + return 0; +} + +int update_sc_ctrl_map(struct sc_ctrl sc_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(sc_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open sc_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&sc_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update sc_ctrl_map elem\n"); + return err; + } + + return 0; +} + +int update_preempt_ctrl_map(struct preempt_ctrl preempt_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(preempt_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open preempt_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&preempt_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update preempt_ctrl_map elem\n"); + return err; + } + + return 0; +} + +int update_schedule_ctrl_map(struct schedule_ctrl schedule_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(schedule_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open schedule_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&schedule_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update schedule_ctrl_map elem\n"); + return err; + } + + return 0; +} + +int update_mq_ctrl_map(struct mq_ctrl mq_ctrl){ + int err,key = 0; + int srcmap_fd; + + srcmap_fd = bpf_obj_get(mq_ctrl_path); + if (srcmap_fd < 0) { + fprintf(stderr,"Failed to open mq_ctrl_map file\n"); + return srcmap_fd; + } + err = bpf_map_update_elem(srcmap_fd,&key,&mq_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update mq_ctrl_map elem\n"); + return err; + } + + return 0; +} +#endif // CPU_WATCHER_HELPER_H \ No newline at end of file diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test/test_cpuwatcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test/test_cpuwatcher.c index 0d5561ebd..ada9abd83 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test/test_cpuwatcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test/test_cpuwatcher.c @@ -174,7 +174,9 @@ int main(int argc, char **argv){ } if(env.preempt_test){ - /*preempt_delay的测试代码*/ + printf("PREEMPT_TEST----------------------------------------------\n"); + //PREEMPT功能测试逻辑：无限循环的线程函数，不断调用 sched_yield() 来放弃 CPU 使用权，模拟高调度负载。 + start_schedule_stress_test(10); // 创建10个线程进行调度压力测试 } if(env.schedule_test){ @@ -186,7 +188,6 @@ int main(int argc, char **argv){ printf("执行指令 sysbench --threads=32 --time=10 cpu run\n"); execve("/usr/bin/sysbench", argvv, envp); perror("execve"); - printf("\n"); }