Skip to content

Commit

Permalink
cpu_watcher：增加用户态互斥锁详细信息 (#869)
Browse files Browse the repository at this point in the history 
* cpu_watcher：schedule_delay增加阈值选项&&workflow增加测试流程

* .

* cpu_watcher：增加controller功能

* cpu_watcher:schedule_delay增加dump出调度延迟过大task的前两个task

* sar功能适配controller

* .

* proc_image：增加进程画像子功能使用说明

* cpu_watcher:增加内核态互斥锁信息细粒度采集

* Update mutrace.bpf.c

* 1

* Update mutrace.bpf.c

* Update cpu_watcher.c

* 增加输出过滤，识别被争用的锁

* 添加测试程序

* add user mutex details
  • Loading branch information
@vvzxy
vvzxy authored Jul 30, 2024
1 parent e040be4 commit 4f953cb
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 43 deletions.
139 changes: 109 additions & 30 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
char LICENSE[] SEC("license") = "Dual BSD/GPL";

const int ctrl_key = 0;
BPF_HASH(mutex_info_map, u64, struct mutex_info_kernel, 1024);
BPF_HASH(kmutex_info_map, u64, struct mutex_info, 1024);
BPF_HASH(umutex_info_map, u64, struct mutex_info, 1024);
BPF_HASH(trylock_map, u64, struct trylock_info, 1024);
BPF_ARRAY(mu_ctrl_map, int, struct mu_ctrl, 1);
struct {
__uint(type, BPF_MAP_TYPE_RINGBUF);
Expand All @@ -47,11 +49,11 @@ SEC("kprobe/mutex_lock")
int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) {
u64 lock_addr = (u64)lock; // 获取锁地址
u64 ts = bpf_ktime_get_ns();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
info->acquire_time = ts; // 保存锁获取时间
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -61,7 +63,7 @@ int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
return 0;
}
Expand All @@ -72,11 +74,11 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) {
if (ret != 0) { // 成功获取锁
u64 lock_addr = (u64)lock; // 获取锁地址
u64 ts = bpf_ktime_get_ns();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
info->acquire_time = ts;
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -86,7 +88,7 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
}
return 0;
Expand Down Expand Up @@ -121,13 +123,13 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) {
e->owner_pid = 0;
__builtin_memset(e->owner_name, 0, sizeof(e->owner_name));
}
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
u64 contention_start = ts;
info->contended_total += (contention_start - info->acquire_time); // 更新争用时间
info->count++; // 更新争用次数
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -137,7 +139,7 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
bpf_ringbuf_submit(e, 0);
return 0;
Expand All @@ -148,7 +150,7 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) {
u64 lock_addr = (u64)lock;
u64 ts = bpf_ktime_get_ns();
pid_t pid = bpf_get_current_pid_tgid();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
u64 held_time = ts - info->acquire_time; // 计算锁被持有的时间
info->locked_total += held_time; // 更新锁被持有的总时间
Expand All @@ -167,27 +169,104 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) {
/* 用户态互斥锁 */
/*----------------------------------------------*/

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_lock_init, void *__mutex){

// }

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_lock,void *__mutex){

// }
SEC("uprobe/pthread_mutex_lock")
int BPF_KPROBE(pthread_mutex_lock, void *__mutex) {
u64 pid_tgid = bpf_get_current_pid_tgid();
pid_t pid = pid_tgid >> 32;
u64 now = bpf_ktime_get_ns();

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_trylock, void *__mutex){

// }
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
if (info->acquire_time > 0) {
// 如果 acquire_time 已经被设置，说明锁被争用
info->contended_total += (now - info->acquire_time);
info->count += 1;
}
info->acquire_time = now;
info->last_owner = pid;
bpf_get_current_comm(&info->last_name, sizeof(info->last_name));
} else {
// 初始化 mutex_info
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
.count = 0,
.last_owner = pid,
.acquire_time = now,
.ptr = (u64)__mutex,
};
bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name));
bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY);
}
return 0;
}

SEC("uprobe/__pthread_mutex_trylock")
int BPF_KPROBE(__pthread_mutex_trylock, void *__mutex) {
u64 pid_tgid = bpf_get_current_pid_tgid();
u64 now = bpf_ktime_get_ns();
struct trylock_info info = {
.__mutex = __mutex,
.start_time = now,
};
bpf_map_update_elem(&trylock_map, &pid_tgid, &info, BPF_ANY);
return 0;
}

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){

// }
SEC("uretprobe/__pthread_mutex_trylock")
int BPF_KRETPROBE(ret_pthread_mutex_trylock, int ret) {
u64 pid_tgid = bpf_get_current_pid_tgid();
struct trylock_info *try_info = bpf_map_lookup_elem(&trylock_map, &pid_tgid);
if (!try_info) {
return 0;
}
void *__mutex = try_info->__mutex;
u64 now = bpf_ktime_get_ns();
if (ret == 0) {
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
if (info->acquire_time > 0) {
// 如果 acquire_time 已经被设置，说明锁被争用
info->contended_total += (now - info->acquire_time);
info->count += 1;
}
info->acquire_time = now;
info->last_owner = pid_tgid >> 32;
bpf_get_current_comm(&info->last_name, sizeof(info->last_name));
} else {
// 初始化 mutex_info
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
.count = 0,
.last_owner = pid_tgid >> 32,
.acquire_time = now,
.ptr = (u64)__mutex,
};
bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name));
bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY);
}
}
bpf_map_delete_elem(&trylock_map, &pid_tgid);
return 0;
}

SEC("uprobe/pthread_mutex_unlock")
int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){
u64 now = bpf_ktime_get_ns();
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
u64 locked_time = now - info->acquire_time;
info->locked_total += locked_time;
if (locked_time > info->locked_max) {
info->locked_max = locked_time;
}
info->acquire_time = 0;
}
return 0;
}

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_destroy,void *__mutex){

// }
23 changes: 18 additions & 5 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static struct env {
int freq;
bool mutrace;
bool mutex_detail;
bool umutex;
} env = {
.usemode = 0,
.SAR = false,
Expand All @@ -55,6 +56,7 @@ static struct env {
.freq = 99,
.mutrace = false,
.mutex_detail = false,
.umutex = false,
};

const char argp_program_doc[] ="Trace process to get cpu watcher.\n";
Expand All @@ -72,7 +74,8 @@ static const struct argp_option opts[] = {
{"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" },
{"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" },
{"mutrace", 'x', 0, 0, "Print kernel mutex contend" },
{"mutex_detail", 'i', 0, 0, "Print kernel mutex details" },
{"mutex_detail", 'i', 0, 0, "Print kernel mutex details" },
{"umutex", 'b', 0, 0, "Print user mutex details" },
{ NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" },
{},
};
Expand Down Expand Up @@ -126,7 +129,10 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
break;
case 'i':
env.mutex_detail = true;
break;
break;
case 'b':
env.umutex = true;
break;
case 'h':
argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
break;
Expand Down Expand Up @@ -242,9 +248,16 @@ int main(int argc, char **argv)
}

if(env.mutrace){
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,MUTEX_WATCHER+env.mutex_detail};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
if (env.umutex){
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+2};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
}
else{
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+env.mutex_detail};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
}
}
}else if(env.usemode == 2){ // deactivate mode
err = deactivate_mode();
Expand Down
54 changes: 47 additions & 7 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <sys/sysinfo.h>
#include <sys/select.h>
#include <unistd.h>
#include <bpf/bpf.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
Expand Down Expand Up @@ -559,6 +560,18 @@ static int preempt_print(void *ctx, void *data, unsigned long data_sz)
return 0;
}

static int attach(struct mutrace_bpf *mu_skel)
{
int err;
ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_lock,pthread_mutex_lock);
ATTACH_UPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,__pthread_mutex_trylock);
ATTACH_URETPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,ret_pthread_mutex_trylock);
ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_unlock,pthread_mutex_unlock);
err = mutrace_bpf__attach(mu_skel);
CHECK_ERR(err, "Failed to attach BPF skeleton");
return 0;

}


//mutrace输出
Expand All @@ -584,8 +597,15 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) {
if(err < 0){
fprintf(stderr, "Failed to update elem\n");
}
}else if (mu_ctrl.prev_watcher == MUTEX_WATCHER +2) {
printf("%s\n"," lock_ptr locked_total locked_max contended_total count last_owner last_owmer_name");
mu_ctrl.prev_watcher = MUTEX_WATCHER + 9;//打印表头功能关
err = bpf_map_update_elem(mumap_fd, &key, &mu_ctrl, 0);
if(err < 0){
fprintf(stderr, "Failed to update elem\n");
}
}
if(!mu_ctrl.mutex_detail){
if(!mu_ctrl.mutex_detail&& (!mu_ctrl.umutex)){
const struct mutex_contention_event *e = data;
if (e->owner_pid == 0 || e->contender_pid == 0||e->owner_pid == 1) {
return 0;
Expand All @@ -598,10 +618,10 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) {
return 0;
}

static int mutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.mutex_info_map);
static int kmutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.kmutex_info_map);
u64 key, next_key;
struct mutex_info_kernel info;
struct mutex_info info;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
int err = bpf_map_lookup_elem(fd, &next_key, &info);
if (err == 0 && info.contended_total != 0) { // 添加过滤条件
Expand All @@ -613,6 +633,21 @@ static int mutex_detail() {
return 0;
}

static int umutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.umutex_info_map);
u64 key, next_key;
struct mutex_info info;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
int err = bpf_map_lookup_elem(fd, &next_key, &info);
if (err == 0 && info.contended_total != 0) { // 添加过滤条件
printf(" %15llu %15llums %15llums %15llums %15d %15d %20s\n",
next_key, info.locked_total/1000000, info.locked_max/1000000, info.contended_total/1000000, info.count, info.last_owner, info.last_name);
}
key = next_key;
}
return 0;
}

static int schedule_print()
{
int err,key = 0;
Expand Down Expand Up @@ -1013,7 +1048,7 @@ int main(int argc, char **argv)
goto mutrace_cleanup;
}
mumap_fd = bpf_map__fd(mu_ctrl_map);
struct mu_ctrl init_value = {false,false,MUTEX_WATCHER};
struct mu_ctrl init_value = {false,false,false,MUTEX_WATCHER};

err = bpf_map_update_elem(mumap_fd, &key, &init_value, 0);
if(err < 0){
Expand All @@ -1029,7 +1064,7 @@ int main(int argc, char **argv)
fprintf(stderr, "Failed to update elem\n");
goto mutrace_cleanup;
}
err = mutrace_bpf__attach(mu_skel);
err = attach(mu_skel);
if (err) {
fprintf(stderr, "Failed to attach BPF skeleton\n");
goto mutrace_cleanup;
Expand Down Expand Up @@ -1144,10 +1179,15 @@ int main(int argc, char **argv)
break;
}
if(env.MUTRACE&&mu_ctrl.mutex_detail){
err = mutex_detail();
err = kmutex_detail();
sleep(1);
printf("-------------------------------------------------------------\n");
}else if(env.MUTRACE&&mu_ctrl.umutex){
err = umutex_detail();
sleep(1);
printf("-------------------------------------------------------------\n");
}

}
else {
printf("正在开发中......\n-c 打印cs_delay:\t对内核函数schedule()的执行时长进行测试;\n-s sar工具;\n-y 打印sc_delay:\t系统调用运行延迟进行检测; \n-p 打印preempt_time:\t对抢占调度时间输出;\n");
Expand Down
Loading

0 comments on commit 4f953cb

Please sign in to comment.