diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c index c1552cb23..79212119b 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c @@ -23,7 +23,9 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; const int ctrl_key = 0; -BPF_HASH(mutex_info_map, u64, struct mutex_info_kernel, 1024); +BPF_HASH(kmutex_info_map, u64, struct mutex_info, 1024); +BPF_HASH(umutex_info_map, u64, struct mutex_info, 1024); +BPF_HASH(trylock_map, u64, struct trylock_info, 1024); BPF_ARRAY(mu_ctrl_map, int, struct mu_ctrl, 1); struct { __uint(type, BPF_MAP_TYPE_RINGBUF); @@ -47,11 +49,11 @@ SEC("kprobe/mutex_lock") int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) { u64 lock_addr = (u64)lock; // 获取锁地址 u64 ts = bpf_ktime_get_ns(); - struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr); + struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr); if (info) { info->acquire_time = ts; // 保存锁获取时间 } else { - struct mutex_info_kernel new_info = { + struct mutex_info new_info = { .locked_total = 0, .locked_max = 0, .contended_total = 0, @@ -61,7 +63,7 @@ int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) { .ptr = lock_addr }; __builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name)); - bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY); + bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY); } return 0; } @@ -72,11 +74,11 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) { if (ret != 0) { // 成功获取锁 u64 lock_addr = (u64)lock; // 获取锁地址 u64 ts = bpf_ktime_get_ns(); - struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr); + struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr); if (info) { info->acquire_time = ts; } else { - struct mutex_info_kernel new_info = { + struct mutex_info new_info = { .locked_total = 0, .locked_max = 0, .contended_total = 0, @@ -86,7 +88,7 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) { .ptr = lock_addr }; __builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name)); - bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY); + bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY); } } return 0; @@ -121,13 +123,13 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) { e->owner_pid = 0; __builtin_memset(e->owner_name, 0, sizeof(e->owner_name)); } - struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr); + struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr); if (info) { u64 contention_start = ts; info->contended_total += (contention_start - info->acquire_time); // 更新争用时间 info->count++; // 更新争用次数 } else { - struct mutex_info_kernel new_info = { + struct mutex_info new_info = { .locked_total = 0, .locked_max = 0, .contended_total = 0, @@ -137,7 +139,7 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) { .ptr = lock_addr }; __builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name)); - bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY); + bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY); } bpf_ringbuf_submit(e, 0); return 0; @@ -148,7 +150,7 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) { u64 lock_addr = (u64)lock; u64 ts = bpf_ktime_get_ns(); pid_t pid = bpf_get_current_pid_tgid(); - struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr); + struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr); if (info) { u64 held_time = ts - info->acquire_time; // 计算锁被持有的时间 info->locked_total += held_time; // 更新锁被持有的总时间 @@ -167,27 +169,104 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) { /* 用户态互斥锁 */ /*----------------------------------------------*/ -// SEC("uprobe") -// int BPF_KPROBE(pthread_mutex_lock_init, void *__mutex){ -// } -// SEC("uprobe") -// int BPF_KPROBE(pthread_mutex_lock,void *__mutex){ - -// } +SEC("uprobe/pthread_mutex_lock") +int BPF_KPROBE(pthread_mutex_lock, void *__mutex) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + pid_t pid = pid_tgid >> 32; + u64 now = bpf_ktime_get_ns(); -// SEC("uprobe") -// int BPF_KPROBE(pthread_mutex_trylock, void *__mutex){ - -// } + struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex); + if (info) { + if (info->acquire_time > 0) { + // 如果 acquire_time 已经被设置，说明锁被争用 + info->contended_total += (now - info->acquire_time); + info->count += 1; + } + info->acquire_time = now; + info->last_owner = pid; + bpf_get_current_comm(&info->last_name, sizeof(info->last_name)); + } else { + // 初始化 mutex_info + struct mutex_info new_info = { + .locked_total = 0, + .locked_max = 0, + .contended_total = 0, + .count = 0, + .last_owner = pid, + .acquire_time = now, + .ptr = (u64)__mutex, + }; + bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name)); + bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY); + } + return 0; +} + +SEC("uprobe/__pthread_mutex_trylock") +int BPF_KPROBE(__pthread_mutex_trylock, void *__mutex) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + u64 now = bpf_ktime_get_ns(); + struct trylock_info info = { + .__mutex = __mutex, + .start_time = now, + }; + bpf_map_update_elem(&trylock_map, &pid_tgid, &info, BPF_ANY); + return 0; +} -// SEC("uprobe") -// int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){ - -// } +SEC("uretprobe/__pthread_mutex_trylock") +int BPF_KRETPROBE(ret_pthread_mutex_trylock, int ret) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + struct trylock_info *try_info = bpf_map_lookup_elem(&trylock_map, &pid_tgid); + if (!try_info) { + return 0; + } + void *__mutex = try_info->__mutex; + u64 now = bpf_ktime_get_ns(); + if (ret == 0) { + struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex); + if (info) { + if (info->acquire_time > 0) { + // 如果 acquire_time 已经被设置，说明锁被争用 + info->contended_total += (now - info->acquire_time); + info->count += 1; + } + info->acquire_time = now; + info->last_owner = pid_tgid >> 32; + bpf_get_current_comm(&info->last_name, sizeof(info->last_name)); + } else { + // 初始化 mutex_info + struct mutex_info new_info = { + .locked_total = 0, + .locked_max = 0, + .contended_total = 0, + .count = 0, + .last_owner = pid_tgid >> 32, + .acquire_time = now, + .ptr = (u64)__mutex, + }; + bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name)); + bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY); + } + } + bpf_map_delete_elem(&trylock_map, &pid_tgid); + return 0; +} + +SEC("uprobe/pthread_mutex_unlock") +int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){ + u64 now = bpf_ktime_get_ns(); + struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex); + if (info) { + u64 locked_time = now - info->acquire_time; + info->locked_total += locked_time; + if (locked_time > info->locked_max) { + info->locked_max = locked_time; + } + info->acquire_time = 0; + } + return 0; +} -// SEC("uprobe") -// int BPF_KPROBE(pthread_mutex_destroy,void *__mutex){ - -// } diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c index b507e48ff..2c319dfe0 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c @@ -41,6 +41,7 @@ static struct env { int freq; bool mutrace; bool mutex_detail; + bool umutex; } env = { .usemode = 0, .SAR = false, @@ -55,6 +56,7 @@ static struct env { .freq = 99, .mutrace = false, .mutex_detail = false, + .umutex = false, }; const char argp_program_doc[] ="Trace process to get cpu watcher.\n"; @@ -72,7 +74,8 @@ static const struct argp_option opts[] = { {"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" }, {"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" }, {"mutrace", 'x', 0, 0, "Print kernel mutex contend" }, - {"mutex_detail", 'i', 0, 0, "Print kernel mutex details" }, + {"mutex_detail", 'i', 0, 0, "Print kernel mutex details" }, + {"umutex", 'b', 0, 0, "Print user mutex details" }, { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, {}, }; @@ -126,7 +129,10 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) break; case 'i': env.mutex_detail = true; - break; + break; + case 'b': + env.umutex = true; + break; case 'h': argp_state_help(state, stderr, ARGP_HELP_STD_HELP); break; @@ -242,9 +248,16 @@ int main(int argc, char **argv) } if(env.mutrace){ - struct mu_ctrl mu_ctrl = {true,env.mutex_detail,MUTEX_WATCHER+env.mutex_detail}; - err = update_mu_ctrl_map(mu_ctrl); - if(err < 0) return err; + if (env.umutex){ + struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+2}; + err = update_mu_ctrl_map(mu_ctrl); + if(err < 0) return err; + } + else{ + struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+env.mutex_detail}; + err = update_mu_ctrl_map(mu_ctrl); + if(err < 0) return err; + } } }else if(env.usemode == 2){ // deactivate mode err = deactivate_mode(); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index 3ecc8ca4f..e50e23ab2 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -559,6 +560,18 @@ static int preempt_print(void *ctx, void *data, unsigned long data_sz) return 0; } +static int attach(struct mutrace_bpf *mu_skel) +{ + int err; + ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_lock,pthread_mutex_lock); + ATTACH_UPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,__pthread_mutex_trylock); + ATTACH_URETPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,ret_pthread_mutex_trylock); + ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_unlock,pthread_mutex_unlock); + err = mutrace_bpf__attach(mu_skel); + CHECK_ERR(err, "Failed to attach BPF skeleton"); + return 0; + +} //mutrace输出 @@ -584,8 +597,15 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) { if(err < 0){ fprintf(stderr, "Failed to update elem\n"); } + }else if (mu_ctrl.prev_watcher == MUTEX_WATCHER +2) { + printf("%s\n"," lock_ptr locked_total locked_max contended_total count last_owner last_owmer_name"); + mu_ctrl.prev_watcher = MUTEX_WATCHER + 9;//打印表头功能关 + err = bpf_map_update_elem(mumap_fd, &key, &mu_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + } } - if(!mu_ctrl.mutex_detail){ + if(!mu_ctrl.mutex_detail&& (!mu_ctrl.umutex)){ const struct mutex_contention_event *e = data; if (e->owner_pid == 0 || e->contender_pid == 0||e->owner_pid == 1) { return 0; @@ -598,10 +618,10 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) { return 0; } -static int mutex_detail() { - int fd = bpf_map__fd(mu_skel->maps.mutex_info_map); +static int kmutex_detail() { + int fd = bpf_map__fd(mu_skel->maps.kmutex_info_map); u64 key, next_key; - struct mutex_info_kernel info; + struct mutex_info info; while (bpf_map_get_next_key(fd, &key, &next_key) == 0) { int err = bpf_map_lookup_elem(fd, &next_key, &info); if (err == 0 && info.contended_total != 0) { // 添加过滤条件 @@ -613,6 +633,21 @@ static int mutex_detail() { return 0; } +static int umutex_detail() { + int fd = bpf_map__fd(mu_skel->maps.umutex_info_map); + u64 key, next_key; + struct mutex_info info; + while (bpf_map_get_next_key(fd, &key, &next_key) == 0) { + int err = bpf_map_lookup_elem(fd, &next_key, &info); + if (err == 0 && info.contended_total != 0) { // 添加过滤条件 + printf(" %15llu %15llums %15llums %15llums %15d %15d %20s\n", + next_key, info.locked_total/1000000, info.locked_max/1000000, info.contended_total/1000000, info.count, info.last_owner, info.last_name); + } + key = next_key; + } + return 0; +} + static int schedule_print() { int err,key = 0; @@ -1013,7 +1048,7 @@ int main(int argc, char **argv) goto mutrace_cleanup; } mumap_fd = bpf_map__fd(mu_ctrl_map); - struct mu_ctrl init_value = {false,false,MUTEX_WATCHER}; + struct mu_ctrl init_value = {false,false,false,MUTEX_WATCHER}; err = bpf_map_update_elem(mumap_fd, &key, &init_value, 0); if(err < 0){ @@ -1029,7 +1064,7 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to update elem\n"); goto mutrace_cleanup; } - err = mutrace_bpf__attach(mu_skel); + err = attach(mu_skel); if (err) { fprintf(stderr, "Failed to attach BPF skeleton\n"); goto mutrace_cleanup; @@ -1144,10 +1179,15 @@ int main(int argc, char **argv) break; } if(env.MUTRACE&&mu_ctrl.mutex_detail){ - err = mutex_detail(); + err = kmutex_detail(); + sleep(1); + printf("-------------------------------------------------------------\n"); + }else if(env.MUTRACE&&mu_ctrl.umutex){ + err = umutex_detail(); sleep(1); printf("-------------------------------------------------------------\n"); } + } else { printf("正在开发中......\n-c 打印cs_delay:\t对内核函数schedule()的执行时长进行测试;\n-s sar工具;\n-y 打印sc_delay:\t系统调用运行延迟进行检测; \n-p 打印preempt_time:\t对抢占调度时间输出;\n"); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h index 54a328542..7ffb2b3fa 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h @@ -158,7 +158,7 @@ struct proc_history { /*----------------------------------------------*/ /* mutrace相关结构体 */ /*----------------------------------------------*/ -struct mutex_info_kernel { +struct mutex_info { u64 locked_total;//锁被持有的总时间 u64 locked_max;//锁被持有的最长时间 u64 contended_total;//锁发生竞争的总时间 @@ -179,6 +179,11 @@ struct mutex_contention_event { int contender_prio; }; +struct trylock_info { + void *__mutex; + u64 start_time; +}; + /*----------------------------------------------*/ /* mq_delay相关结构体 */ /*----------------------------------------------*/ @@ -293,6 +298,7 @@ struct mq_ctrl{ struct mu_ctrl{ bool mu_func; bool mutex_detail; + bool umutex; int prev_watcher; }; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h index 3db69defd..08e38f0e1 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher_helper.h @@ -298,4 +298,56 @@ void add_entry(int pid, const char *comm, long long delay) { seen_count++; } } +/*----------------------------------------------*/ +/* uprobe */ +/*----------------------------------------------*/ +static const char object[] = "/usr/lib/x86_64-linux-gnu/libc.so.6"; + +#define __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe) \ + do \ + { \ + LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts, \ + .retprobe = is_retprobe, \ + .func_name = #sym_name); \ + skel->links.prog_name = bpf_program__attach_uprobe_opts( \ + skel->progs.prog_name, \ + -1, \ + object, \ + 0, \ + &uprobe_opts); \ + } while (false) + +#define __CHECK_PROGRAM(skel, prog_name) \ + do \ + { \ + if (!skel->links.prog_name) \ + { \ + fprintf(stderr, "[%s] no program attached for" #prog_name "\n", strerror(errno)); \ + return -errno; \ + } \ + } while (false) + +#define __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, is_retprobe) \ + do \ + { \ + __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe); \ + __CHECK_PROGRAM(skel, prog_name); \ + } while (false) + +#define ATTACH_UPROBE(skel, sym_name, prog_name) __ATTACH_UPROBE(skel, sym_name, prog_name, false) +#define ATTACH_URETPROBE(skel, sym_name, prog_name) __ATTACH_UPROBE(skel, sym_name, prog_name, true) + +#define ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name) __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, false) +#define ATTACH_URETPROBE_CHECKED(skel, sym_name, prog_name) __ATTACH_UPROBE_CHECKED(skel, sym_name, prog_name, true) + +#define CHECK_ERR(cond, info) \ + if (cond) \ + { \ + fprintf(stderr, "[%s]" info "\n", strerror(errno)); \ + return -1; \ + } + +#define warn(...) fprintf(stderr, __VA_ARGS__) + + #endif // CPU_WATCHER_HELPER_H \ No newline at end of file