From a558eae2ee5747aecb00503787ece626e3239e43 Mon Sep 17 00:00:00 2001 From: albertxu216 <145351853+albertxu216@users.noreply.github.com> Date: Thu, 28 Mar 2024 20:15:49 +0800 Subject: [PATCH] =?UTF-8?q?Cpu=5Fwatcher:=20=E5=A2=9E=E5=8A=A0=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E9=98=9F=E5=88=97=E5=BB=B6=E8=BF=9F=E5=8A=9F=E8=83=BD?= =?UTF-8?q?mqdelay=20(#735)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add mq_delay * Add mq_delay * Add mq_delay * add the head --- .../CPU_Subsystem/cpu_watcher/Makefile | 2 +- .../CPU_Subsystem/cpu_watcher/cpu_watcher.c | 84 ++++++- .../CPU_Subsystem/cpu_watcher/cpu_watcher.h | 46 +++- .../CPU_Subsystem/cpu_watcher/mq_delay.bpf.c | 214 ++++++++++++++++++ 4 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 eBPF_Supermarket/CPU_Subsystem/cpu_watcher/mq_delay.bpf.c diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile index a1757cf4b..efe1a9de9 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/Makefile @@ -42,7 +42,7 @@ INCLUDES := -I$(OUTPUT) -I../../../libbpf/include/uapi -I$(dir $(VMLINUX)) -I$(L CFLAGS := -g -Wall ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) -APPS =cs_delay sar sc_delay preempt schedule_delay +APPS =cs_delay sar sc_delay preempt schedule_delay mq_delay TARGETS=cpu_watcher diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index d5b890ac5..8c2beaf39 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -31,10 +31,23 @@ #include "sc_delay.skel.h" #include "preempt.skel.h" #include "schedule_delay.skel.h" +#include "mq_delay.skel.h" typedef long long unsigned int u64; typedef unsigned int u32; +struct list_head { + struct list_head *next; + struct list_head *prev; +}; +struct msg_msg { + struct list_head m_list; + long int m_type; + size_t m_ts; + struct msg_msgseg *next; + void *security; +}; + static struct env { int time; bool enable_proc; @@ -43,6 +56,7 @@ static struct env { bool SYSCALL_DELAY; bool PREEMPT; bool SCHEDULE_DELAY; + bool MQ_DELAY; int freq; } env = { .time = 0, @@ -52,6 +66,7 @@ static struct env { .SYSCALL_DELAY = false, .PREEMPT = false, .SCHEDULE_DELAY = false, + .MQ_DELAY = false, .freq = 99 }; @@ -61,6 +76,7 @@ struct sar_bpf *sar_skel; struct sc_delay_bpf *sc_skel; struct preempt_bpf *preempt_skel; struct schedule_delay_bpf *sd_skel; +struct mq_delay_bpf *mq_skel; u64 softirq = 0;//初始化softirq; u64 irqtime = 0;//初始化irq; @@ -90,6 +106,7 @@ static const struct argp_option opts[] = { {"syscall_delay", 'S', 0,0,"print syscall_delay (the data of syscall)"}, {"preempt_time", 'p', 0,0,"print preempt_time (the data of preempt_schedule)"}, {"schedule_delay", 'd', 0,0,"print schedule_delay (the data of cpu)"}, + {"mq_delay", 'm', 0,0,"print mq_delay"}, { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, {0}, }; @@ -115,6 +132,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'd': env.SCHEDULE_DELAY = true; break; + case 'm': + env.MQ_DELAY = true; + break; case 'h': argp_state_help(state, stderr, ARGP_HELP_STD_HELP); break; @@ -455,6 +475,24 @@ static int schedule_print(struct bpf_map *sys_fd) return 0; } +static int mq_event(void *ctx, void *data,unsigned long data_sz) +{ + time_t now = time(NULL);// 获取当前时间 + struct tm *localTime = localtime(&now);// 将时间转换为本地时间结构 + printf("\n\nTime: %02d:%02d:%02d\n",localTime->tm_hour, localTime->tm_min, localTime->tm_sec); + printf("-----------------------------------------------------------------------------------------------------------\n"); + const struct mq_events *e = data; + printf("Mqdes: %-8llu msg_len: %-8llu msg_prio: %-8llu\n",e->mqdes,e->msg_len,e->msg_prio); + printf("SND_PID: %-8lu SND_enter_time: %-16llu\n", + e->send_pid,e->send_enter_time); + printf("-----------------------------------------------------------------------------------------------------------\n"); + printf("RCV_PID: %-8lu RCV_enter_time: %-16llu RCV_exit_time: %-16llu\n", + e->rcv_pid,e->rcv_enter_time,e->rcv_exit_time); + printf("RCV_Delay: %-8.2fms\nDelay: %-8.2fms\n\n",(e->rcv_exit_time - e->rcv_enter_time)/1000000.0,e->delay/1000000.0); + + return 0; +} + int main(int argc, char **argv) { @@ -582,7 +620,7 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to attach BPF skeleton\n"); goto schedule_cleanup; } - printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs min_delay/μs"); + printf("%-8s %s\n", " TIME ", "avg_delay/μs max_delay/μs min_delay/μs"); }else if (env.SAR){ /* Load and verify BPF application */ sar_skel = sar_bpf__open(); @@ -613,6 +651,34 @@ int main(int argc, char **argv) } //printf(" time proc/s cswch/s runqlen irqTime/us softirq/us idle/ms kthread/us sysc/ms utime/ms sys/ms BpfCnt\n"); printf(" time proc/s cswch/s irqTime/us softirq/us idle/ms kthread/us sysc/ms utime/ms sys/ms\n"); + }else if(env.MQ_DELAY){ + /* Load and verify BPF application */ + mq_skel = mq_delay_bpf__open(); + if (!mq_skel) + { + fprintf(stderr, "Failed to open and load BPF skeleton\n"); + return 1; + } + /* Load & verify BPF programs */ + err = mq_delay_bpf__load(mq_skel); + if (err) + { + fprintf(stderr, "Failed to load and verify BPF skeleton\n"); + goto mq_delay_cleanup; + } + /* Attach tracepoints */ + err = mq_delay_bpf__attach(mq_skel); + if (err) + { + fprintf(stderr, "Failed to attach BPF skeleton\n"); + goto mq_delay_cleanup; + } + rb = ring_buffer__new(bpf_map__fd(mq_skel->maps.rb), mq_event, NULL, NULL); //ring_buffer__new() API，允许在不使用额外选项数据结构下指定回调 + if (!rb) { + err = -1; + fprintf(stderr, "Failed to create ring buffer\n"); + goto mq_delay_cleanup; + } } while (!exiting) { if(env.SAR){ @@ -692,6 +758,17 @@ int main(int argc, char **argv) } sleep(1); } + else if(env.MQ_DELAY){ + err = ring_buffer__poll(rb, 1000 /* timeout, s */); + if (err == -EINTR) { + err = 0; + break; + } + if (err < 0) { + printf("Error polling perf buffer: %d\n", err); + break; + } + } else { printf("正在开发中......\n-c 打印cs_delay:\t对内核函数schedule()的执行时长进行测试;\n-s sar工具;\n-y 打印sc_delay:\t系统调用运行延迟进行检测; \n-p 打印preempt_time:\t对抢占调度时间输出;\n"); break; @@ -720,4 +797,9 @@ int main(int argc, char **argv) schedule_cleanup: schedule_delay_bpf__destroy(sd_skel); return err < 0 ? -err : 0; + +mq_delay_cleanup: + ring_buffer__free(rb); + mq_delay_bpf__destroy(mq_skel); + return err < 0 ? -err : 0; } diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.h index e21ef66cb..4d09caabf 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.h @@ -17,10 +17,11 @@ // eBPF map for libbpf sar #include #include -//#include typedef long long unsigned int u64; typedef unsigned int u32; +typedef __kernel_mqd_t mqd_t; +#define __user #define MAX_CPU_NR 128 #define TASK_COMM_LEN 20 #define SYSCALL_MIN_TIME 1E7 @@ -142,6 +143,49 @@ struct sum_schedule{ unsigned long long max_delay; unsigned long long min_delay; }; + +/*----------------------------------------------*/ +/* mq_delay相关结构体 */ +/*----------------------------------------------*/ +struct mq_events { + int send_pid; + int rcv_pid; + mqd_t mqdes; + size_t msg_len; + unsigned int msg_prio; + + u64 send_enter_time; + u64 send_exit_time; + u64 send_delay; + + u64 rcv_enter_time; + u64 rcv_exit_time; + u64 rcv_delay; + u64 delay; +}; +struct send_events { + int send_pid; + u64 Key_msg_ptr; + + mqd_t mqdes; + size_t msg_len; + unsigned int msg_prio; + const char *u_msg_ptr; + const void *src; + u64 send_enter_time; + u64 send_exit_time; +}; +struct rcv_events { + int rcv_pid; + u64 Key_msg_ptr; + mqd_t mqdes; + size_t msg_len; + unsigned int msg_prio; + const char *u_msg_ptr; + const void *dest; + u64 rcv_enter_time; + u64 rcv_exit_time; +}; /*----------------------------------------------*/ /* cswch_args结构体 */ /*----------------------------------------------*/ diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/mq_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/mq_delay.bpf.c new file mode 100644 index 000000000..0284b1768 --- /dev/null +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/mq_delay.bpf.c @@ -0,0 +1,214 @@ +// Copyright 2024 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: albert_xuu@163.com + + +#include "vmlinux.h" +#include //包含了BPF 辅助函数 +#include +#include + +#include "cpu_watcher.h" + + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +BPF_HASH(send_msg1,pid_t,struct send_events,1024);//记录pid->u_msg_ptr的关系；do_mq_timedsend入参 +BPF_HASH(send_msg2,u64,struct send_events,1024);//记录msg->time的关系； +BPF_HASH(rcv_msg1,pid_t,struct rcv_events,1024);//记录pid->u_msg_ptr的关系；do_mq_timedsend入参 +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024); +} rb SEC(".maps"); + +// int print_send_info(struct send_events * mq_send_info,int flag){ +// bpf_printk("---------------------test----------------------------test--------------------------test--------------------------------------------test---------------------test---------------------test\n"); +// bpf_printk("send_msg_prio: %-8lu\n",mq_send_info->msg_prio); +// bpf_printk("mqdes: %-08lu send_pid: %-08lu send_enter_time: %-16lu\n",mq_send_info->mqdes,mq_send_info->send_pid,mq_send_info->send_enter_time); +// if(flag > 0){ +// bpf_printk("u_msg_ptr: 0x%08lx src: 0x%08lx\n",mq_send_info->u_msg_ptr,mq_send_info->src); +// if(flag==2) bpf_printk("Key_msg_ptr: 0x%08lx \n",mq_send_info->Key_msg_ptr); +// } +// bpf_printk("---------------------test----------------------------test--------------------------test--------------------------------------------test---------------------test---------------------test\n"); +// return 0; +// } + +// int print_rcv_info(struct rcv_events * mq_rcv_info,int flag){ +// bpf_printk("---------------------test----------------------------test--------------------------test--------------------------------------------test---------------------test---------------------test\n"); +// bpf_printk("rcv_msg_prio: %-8lu\n",mq_rcv_info->msg_prio); +// bpf_printk("mqdes: %-08lu rcv_pid: %-08lu rcv_enter_time: %-16lu\n",mq_rcv_info->mqdes,mq_rcv_info->rcv_pid,mq_rcv_info->rcv_enter_time); +// if(flag > 0){ +// bpf_printk("u_msg_ptr: 0x%08lx dest: 0x%08lx\n",mq_rcv_info->u_msg_ptr,mq_rcv_info->dest); +// if(flag==2) bpf_printk("Key_msg_ptr: 0x%08lx \n",mq_rcv_info->Key_msg_ptr); +// } +// bpf_printk("---------------------test----------------------------test--------------------------test--------------------------------------------test---------------------test---------------------test\n"); +// return 0; +// } + + +/*获取 mq_send_info -> send_time send_pid mdqes u_msg_ptr msg_len msg_prio*/ +SEC("kprobe/do_mq_timedsend") +int BPF_KPROBE(mq_timedsend,mqd_t mqdes, const char *u_msg_ptr, + size_t msg_len, unsigned int msg_prio, + struct timespec64 *ts) +{ + u64 send_enter_time = bpf_ktime_get_ns();//开始发送信息时间； + int pid = bpf_get_current_pid_tgid();//发送端pid + + /*将消息暂存至send_events结构体中*/ + struct send_events mq_send_info ={}; + mq_send_info.send_pid= pid; + mq_send_info.send_enter_time = send_enter_time; + mq_send_info.mqdes= mqdes; + mq_send_info.msg_len = msg_len; + mq_send_info.msg_prio = msg_prio; + mq_send_info.u_msg_ptr = u_msg_ptr; + + bpf_map_update_elem(&send_msg1, &pid, &mq_send_info, BPF_ANY);//pid->u_msg_ptr + return 0; +} + +/*仅获取mq_send_info -> src*/ +SEC("kprobe/load_msg") +int BPF_KPROBE(load_msg_enter,const void *src, size_t len){ + int pid = bpf_get_current_pid_tgid();//发送端pid + /*记录load入参src*/ + struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg1, &pid); + if(!mq_send_info){ + return 0; + }else{ + mq_send_info->src = src; + } + return 0; +} + +/*获取消息块作为key，并建立 message -> mq_send_info 的哈希表*/ +SEC("kretprobe/load_msg") +int BPF_KRETPROBE(load_msg_exit,void *ret){ + int pid = bpf_get_current_pid_tgid();//发送端pid + /*构建消息块结构体，作为key*/ + struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg1, &pid); + if(!mq_send_info){ + return 0; + } + + /*make key*/ + u64 Key_msg_ptr; + if(mq_send_info->u_msg_ptr == mq_send_info->src && pid == mq_send_info->send_pid){ + /*该load_msg为do_mq_timedsend调用*/ + Key_msg_ptr =(u64)ret; + mq_send_info->Key_msg_ptr = Key_msg_ptr; + } + else { + return 0; + } + /*已经获得key*/ + bpf_map_update_elem(&send_msg2, &Key_msg_ptr, mq_send_info, BPF_ANY);//key_messege->mq_send_info; + bpf_map_delete_elem(&send_msg1,&pid); + return 0; +} +/*-----------------------------------------------------------------------------发送端--------------------------------------------------------------------------------------------------------*/ +/* 分界 */ +/*-----------------------------------------------------------------------------接收端--------------------------------------------------------------------------------------------------------*/ +/*接收端*/ +SEC("kprobe/do_mq_timedreceive") +int BPF_KPROBE(mq_timedreceive_entry,mqd_t mqdes, const char __user *u_msg_ptr, + size_t msg_len, unsigned int msg_prio, + struct timespec64 *ts) +{ + u64 rcv_enter_time = bpf_ktime_get_ns(); + int pid = bpf_get_current_pid_tgid(); + + /*赋值*/ + struct rcv_events mq_rcv_info ={}; + mq_rcv_info.rcv_pid= pid; + mq_rcv_info.rcv_enter_time = rcv_enter_time; + mq_rcv_info.mqdes= mqdes; + mq_rcv_info.u_msg_ptr = u_msg_ptr; + bpf_map_update_elem(&rcv_msg1, &pid, &mq_rcv_info, BPF_ANY);//pid->u_msg_ptr + + return 0; +} + +SEC("kprobe/store_msg") +int BPF_KPROBE(store_msg,void __user *dest, struct msg_msg *msg, size_t len) +{ + int pid = bpf_get_current_pid_tgid(); + + /*make key*/ + u64 Key_msg_ptr = (u64)msg; + struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg2, &Key_msg_ptr); + if(!mq_send_info){ + return 0; + } + + struct rcv_events *mq_rcv_info = bpf_map_lookup_elem(&rcv_msg1, &pid); + if(!mq_rcv_info){ + return 0; + } + /*拿到mq_rcv_info*/ + if(dest == mq_rcv_info->u_msg_ptr && pid == mq_rcv_info->rcv_pid){ + mq_rcv_info->Key_msg_ptr = Key_msg_ptr; + mq_rcv_info->dest = dest; + mq_rcv_info->msg_prio = BPF_CORE_READ(msg,m_type); + mq_rcv_info->msg_len = BPF_CORE_READ(msg,m_ts); + }else{ + return 0; + } + return 0; +} + +SEC("kretprobe/do_mq_timedreceive") +int BPF_KRETPROBE(do_mq_timedreceive_exit,void *ret){ + u64 rcv_exit_time = bpf_ktime_get_ns(); + int pid = bpf_get_current_pid_tgid(); + u64 send_enter_time,delay; + u64 Key; + + /*获取发送端、接收端信息*/ + struct rcv_events *mq_rcv_info = bpf_map_lookup_elem(&rcv_msg1, &pid); + if(!mq_rcv_info){ + return 0; + } + Key = mq_rcv_info->Key_msg_ptr; + struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg2,&Key); + if(!mq_send_info){ + return 0; + } + + send_enter_time = mq_send_info->send_enter_time; + delay = rcv_exit_time - send_enter_time; + + /*ringbuffer传值*/ + struct mq_events *e; + e = bpf_ringbuf_reserve(&rb, sizeof(*e), 0); + if (!e) return 0; + e->send_pid = mq_send_info->send_pid; + e->rcv_pid = pid; + e->mqdes = mq_send_info->mqdes; + e->msg_len = mq_send_info->msg_len; + e->msg_prio = mq_send_info->msg_prio; + + e->send_enter_time = mq_send_info->send_enter_time; + + e->rcv_enter_time = mq_rcv_info->rcv_enter_time; + e->rcv_exit_time = rcv_exit_time; + e->delay = delay; + bpf_ringbuf_submit(e, 0); + bpf_map_delete_elem(&send_msg2, &Key);//暂时性删除 + bpf_map_delete_elem(&rcv_msg1,&pid);//删除rcv_msg1 map; + return 0; + +}