Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cpu_watcher：增加用户态互斥锁详细信息 #869

Merged
merged 23 commits into from
Jul 30, 2024
Merged

cpu_watcher：增加用户态互斥锁详细信息 #869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9dab301
cpu_watcher：schedule_delay增加阈值选项&&workflow增加测试流程
vvzxy May 31, 2024
dabb7cb
.
vvzxy May 31, 2024
faf922a
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jun 7, 2024
28286a6
cpu_watcher：增加controller功能
vvzxy Jun 7, 2024
9a1ed95
Merge branch 'develop' into develop
vvzxy Jun 7, 2024
eed150e
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jun 13, 2024
08abcb5
cpu_watcher:schedule_delay增加dump出调度延迟过大task的前两个task
vvzxy Jun 13, 2024
1f86f58
sar功能适配controller
vvzxy Jun 13, 2024
66725fc
.
vvzxy Jun 14, 2024
8646cfd
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jun 21, 2024
c7f36ef
proc_image：增加进程画像子功能使用说明
vvzxy Jun 21, 2024
12f6c67
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jun 28, 2024
796a8f1
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jul 8, 2024
26a4f07
cpu_watcher:增加内核态互斥锁信息细粒度采集
vvzxy Jul 10, 2024
3c94d95
Update mutrace.bpf.c
vvzxy Jul 10, 2024
b3d04d9
1
vvzxy Jul 10, 2024
605e3e2
Update mutrace.bpf.c
vvzxy Jul 10, 2024
29346db
Update cpu_watcher.c
vvzxy Jul 10, 2024
134eb5e
增加输出过滤，识别被争用的锁
vvzxy Jul 12, 2024
66cf7f4
Resolve merge conflicts
vvzxy Jul 12, 2024
e17cc2a
添加测试程序
vvzxy Jul 19, 2024
eb514c3
Merge branch 'linuxkerneltravel:develop' into develop
vvzxy Jul 26, 2024
7416206
add user mutex details
vvzxy Jul 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 109 additions & 30 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mutrace.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
char LICENSE[] SEC("license") = "Dual BSD/GPL";

const int ctrl_key = 0;
BPF_HASH(mutex_info_map, u64, struct mutex_info_kernel, 1024);
BPF_HASH(kmutex_info_map, u64, struct mutex_info, 1024);
BPF_HASH(umutex_info_map, u64, struct mutex_info, 1024);
BPF_HASH(trylock_map, u64, struct trylock_info, 1024);
BPF_ARRAY(mu_ctrl_map, int, struct mu_ctrl, 1);
struct {
__uint(type, BPF_MAP_TYPE_RINGBUF);
Expand All @@ -47,11 +49,11 @@ SEC("kprobe/mutex_lock")
int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) {
u64 lock_addr = (u64)lock; // 获取锁地址
u64 ts = bpf_ktime_get_ns();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
info->acquire_time = ts; // 保存锁获取时间
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -61,7 +63,7 @@ int BPF_KPROBE(trace_mutex_lock, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
return 0;
}
Expand All @@ -72,11 +74,11 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) {
if (ret != 0) { // 成功获取锁
u64 lock_addr = (u64)lock; // 获取锁地址
u64 ts = bpf_ktime_get_ns();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
info->acquire_time = ts;
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -86,7 +88,7 @@ int BPF_KPROBE(trace_mutex_trylock, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
}
return 0;
Expand Down Expand Up @@ -121,13 +123,13 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) {
e->owner_pid = 0;
__builtin_memset(e->owner_name, 0, sizeof(e->owner_name));
}
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
u64 contention_start = ts;
info->contended_total += (contention_start - info->acquire_time); // 更新争用时间
info->count++; // 更新争用次数
} else {
struct mutex_info_kernel new_info = {
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
Expand All @@ -137,7 +139,7 @@ int BPF_KPROBE(trace_mutex_lock_slowpath, struct mutex *lock) {
.ptr = lock_addr
};
__builtin_memset(new_info.last_name, 0, sizeof(new_info.last_name));
bpf_map_update_elem(&mutex_info_map, &lock_addr, &new_info, BPF_ANY);
bpf_map_update_elem(&kmutex_info_map, &lock_addr, &new_info, BPF_ANY);
}
bpf_ringbuf_submit(e, 0);
return 0;
Expand All @@ -148,7 +150,7 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) {
u64 lock_addr = (u64)lock;
u64 ts = bpf_ktime_get_ns();
pid_t pid = bpf_get_current_pid_tgid();
struct mutex_info_kernel *info = bpf_map_lookup_elem(&mutex_info_map, &lock_addr);
struct mutex_info *info = bpf_map_lookup_elem(&kmutex_info_map, &lock_addr);
if (info) {
u64 held_time = ts - info->acquire_time; // 计算锁被持有的时间
info->locked_total += held_time; // 更新锁被持有的总时间
Expand All @@ -167,27 +169,104 @@ int BPF_KPROBE(trace_mutex_unlock, struct mutex *lock) {
/* 用户态互斥锁 */
/*----------------------------------------------*/

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_lock_init, void *__mutex){

// }

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_lock,void *__mutex){

// }
SEC("uprobe/pthread_mutex_lock")
int BPF_KPROBE(pthread_mutex_lock, void *__mutex) {
u64 pid_tgid = bpf_get_current_pid_tgid();
pid_t pid = pid_tgid >> 32;
u64 now = bpf_ktime_get_ns();

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_trylock, void *__mutex){

// }
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
if (info->acquire_time > 0) {
// 如果 acquire_time 已经被设置，说明锁被争用
info->contended_total += (now - info->acquire_time);
info->count += 1;
}
info->acquire_time = now;
info->last_owner = pid;
bpf_get_current_comm(&info->last_name, sizeof(info->last_name));
} else {
// 初始化 mutex_info
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
.count = 0,
.last_owner = pid,
.acquire_time = now,
.ptr = (u64)__mutex,
};
bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name));
bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY);
}
return 0;
}

SEC("uprobe/__pthread_mutex_trylock")
int BPF_KPROBE(__pthread_mutex_trylock, void *__mutex) {
u64 pid_tgid = bpf_get_current_pid_tgid();
u64 now = bpf_ktime_get_ns();
struct trylock_info info = {
.__mutex = __mutex,
.start_time = now,
};
bpf_map_update_elem(&trylock_map, &pid_tgid, &info, BPF_ANY);
return 0;
}

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){

// }
SEC("uretprobe/__pthread_mutex_trylock")
int BPF_KRETPROBE(ret_pthread_mutex_trylock, int ret) {
u64 pid_tgid = bpf_get_current_pid_tgid();
struct trylock_info *try_info = bpf_map_lookup_elem(&trylock_map, &pid_tgid);
if (!try_info) {
return 0;
}
void *__mutex = try_info->__mutex;
u64 now = bpf_ktime_get_ns();
if (ret == 0) {
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
if (info->acquire_time > 0) {
// 如果 acquire_time 已经被设置，说明锁被争用
info->contended_total += (now - info->acquire_time);
info->count += 1;
}
info->acquire_time = now;
info->last_owner = pid_tgid >> 32;
bpf_get_current_comm(&info->last_name, sizeof(info->last_name));
} else {
// 初始化 mutex_info
struct mutex_info new_info = {
.locked_total = 0,
.locked_max = 0,
.contended_total = 0,
.count = 0,
.last_owner = pid_tgid >> 32,
.acquire_time = now,
.ptr = (u64)__mutex,
};
bpf_get_current_comm(&new_info.last_name, sizeof(new_info.last_name));
bpf_map_update_elem(&umutex_info_map, &__mutex, &new_info, BPF_ANY);
}
}
bpf_map_delete_elem(&trylock_map, &pid_tgid);
return 0;
}

SEC("uprobe/pthread_mutex_unlock")
int BPF_KPROBE(pthread_mutex_unlock, void *__mutex){
u64 now = bpf_ktime_get_ns();
struct mutex_info *info = bpf_map_lookup_elem(&umutex_info_map, &__mutex);
if (info) {
u64 locked_time = now - info->acquire_time;
info->locked_total += locked_time;
if (locked_time > info->locked_max) {
info->locked_max = locked_time;
}
info->acquire_time = 0;
}
return 0;
}

// SEC("uprobe")
// int BPF_KPROBE(pthread_mutex_destroy,void *__mutex){

// }
23 changes: 18 additions & 5 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static struct env {
int freq;
bool mutrace;
bool mutex_detail;
bool umutex;
} env = {
.usemode = 0,
.SAR = false,
Expand All @@ -55,6 +56,7 @@ static struct env {
.freq = 99,
.mutrace = false,
.mutex_detail = false,
.umutex = false,
};

const char argp_program_doc[] ="Trace process to get cpu watcher.\n";
Expand All @@ -72,7 +74,8 @@ static const struct argp_option opts[] = {
{"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" },
{"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" },
{"mutrace", 'x', 0, 0, "Print kernel mutex contend" },
{"mutex_detail", 'i', 0, 0, "Print kernel mutex details" },
{"mutex_detail", 'i', 0, 0, "Print kernel mutex details" },
{"umutex", 'b', 0, 0, "Print user mutex details" },
{ NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" },
{},
};
Expand Down Expand Up @@ -126,7 +129,10 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
break;
case 'i':
env.mutex_detail = true;
break;
break;
case 'b':
env.umutex = true;
break;
case 'h':
argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
break;
Expand Down Expand Up @@ -242,9 +248,16 @@ int main(int argc, char **argv)
}

if(env.mutrace){
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,MUTEX_WATCHER+env.mutex_detail};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
if (env.umutex){
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+2};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
}
else{
struct mu_ctrl mu_ctrl = {true,env.mutex_detail,env.umutex,MUTEX_WATCHER+env.mutex_detail};
err = update_mu_ctrl_map(mu_ctrl);
if(err < 0) return err;
}
}
}else if(env.usemode == 2){ // deactivate mode
err = deactivate_mode();
Expand Down
54 changes: 47 additions & 7 deletions eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <sys/sysinfo.h>
#include <sys/select.h>
#include <unistd.h>
#include <bpf/bpf.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
Expand Down Expand Up @@ -559,6 +560,18 @@ static int preempt_print(void *ctx, void *data, unsigned long data_sz)
return 0;
}

static int attach(struct mutrace_bpf *mu_skel)
{
int err;
ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_lock,pthread_mutex_lock);
ATTACH_UPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,__pthread_mutex_trylock);
ATTACH_URETPROBE_CHECKED(mu_skel,__pthread_mutex_trylock,ret_pthread_mutex_trylock);
ATTACH_UPROBE_CHECKED(mu_skel,pthread_mutex_unlock,pthread_mutex_unlock);
err = mutrace_bpf__attach(mu_skel);
CHECK_ERR(err, "Failed to attach BPF skeleton");
return 0;

}


//mutrace输出
Expand All @@ -584,8 +597,15 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) {
if(err < 0){
fprintf(stderr, "Failed to update elem\n");
}
}else if (mu_ctrl.prev_watcher == MUTEX_WATCHER +2) {
printf("%s\n"," lock_ptr locked_total locked_max contended_total count last_owner last_owmer_name");
mu_ctrl.prev_watcher = MUTEX_WATCHER + 9;//打印表头功能关
err = bpf_map_update_elem(mumap_fd, &key, &mu_ctrl, 0);
if(err < 0){
fprintf(stderr, "Failed to update elem\n");
}
}
if(!mu_ctrl.mutex_detail){
if(!mu_ctrl.mutex_detail&& (!mu_ctrl.umutex)){
const struct mutex_contention_event *e = data;
if (e->owner_pid == 0 || e->contender_pid == 0||e->owner_pid == 1) {
return 0;
Expand All @@ -598,10 +618,10 @@ static int mutrace_print(void *ctx, void *data, unsigned long data_sz) {
return 0;
}

static int mutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.mutex_info_map);
static int kmutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.kmutex_info_map);
u64 key, next_key;
struct mutex_info_kernel info;
struct mutex_info info;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
int err = bpf_map_lookup_elem(fd, &next_key, &info);
if (err == 0 && info.contended_total != 0) { // 添加过滤条件
Expand All @@ -613,6 +633,21 @@ static int mutex_detail() {
return 0;
}

static int umutex_detail() {
int fd = bpf_map__fd(mu_skel->maps.umutex_info_map);
u64 key, next_key;
struct mutex_info info;
while (bpf_map_get_next_key(fd, &key, &next_key) == 0) {
int err = bpf_map_lookup_elem(fd, &next_key, &info);
if (err == 0 && info.contended_total != 0) { // 添加过滤条件
printf(" %15llu %15llums %15llums %15llums %15d %15d %20s\n",
next_key, info.locked_total/1000000, info.locked_max/1000000, info.contended_total/1000000, info.count, info.last_owner, info.last_name);
}
key = next_key;
}
return 0;
}

static int schedule_print()
{
int err,key = 0;
Expand Down Expand Up @@ -1013,7 +1048,7 @@ int main(int argc, char **argv)
goto mutrace_cleanup;
}
mumap_fd = bpf_map__fd(mu_ctrl_map);
struct mu_ctrl init_value = {false,false,MUTEX_WATCHER};
struct mu_ctrl init_value = {false,false,false,MUTEX_WATCHER};

err = bpf_map_update_elem(mumap_fd, &key, &init_value, 0);
if(err < 0){
Expand All @@ -1029,7 +1064,7 @@ int main(int argc, char **argv)
fprintf(stderr, "Failed to update elem\n");
goto mutrace_cleanup;
}
err = mutrace_bpf__attach(mu_skel);
err = attach(mu_skel);
if (err) {
fprintf(stderr, "Failed to attach BPF skeleton\n");
goto mutrace_cleanup;
Expand Down Expand Up @@ -1144,10 +1179,15 @@ int main(int argc, char **argv)
break;
}
if(env.MUTRACE&&mu_ctrl.mutex_detail){
err = mutex_detail();
err = kmutex_detail();
sleep(1);
printf("-------------------------------------------------------------\n");
}else if(env.MUTRACE&&mu_ctrl.umutex){
err = umutex_detail();
sleep(1);
printf("-------------------------------------------------------------\n");
}

}
else {
printf("正在开发中......\n-c 打印cs_delay:\t对内核函数schedule()的执行时长进行测试;\n-s sar工具;\n-y 打印sc_delay:\t系统调用运行延迟进行检测; \n-p 打印preempt_time:\t对抢占调度时间输出;\n");
Expand Down
Loading
Loading