libbpf_cs_delay 可视化输出 (#575)

* add cswch & runqlen

* del vmlinux.h

* Update libbpf_sar.bpf.c

* libbpf_cs_delay可视化

* 修改Makefile

* 修改cs_delay.c

* Update libbpf.yml

.github/workflows/libbpf.yml

@@ -22,17 +22,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt install clang libelf1 libelf-dev zlib1g-dev
-          sudo apt install libbpf-dev
-          sudo apt install linux-tools-5.15.0-53-generic	
-          sudo apt install linux-cloud-tools-5.15.0-53-generic
+          sudo apt update
+          sudo apt install libbpf-dev clang llvm libelf-dev libpcap-dev gcc-multilib build-essential
+          git submodule update --init --recursive
 
       - name: Run cs_delay
         run: |
           cd eBPF_Supermarket/CPU_Subsystem/cs_delay/libbpf_cs_delay
-          # sudo bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
-          clang -g -O2 -target bpf -D__TARGET_ARCH_x86 -I/usr/include/x86_64-linux-gnu -I. -c cs_delay.bpf.c -o cs_delay.bpf.o
-          sudo bpftool gen skeleton cs_delay.bpf.o > cs_delay.skel.h
-          clang -g -O2 -Wall -I . -c cs_delay.c -o cs_delay.o
-          clang -Wall -O2 -g cs_delay.o -static -lbpf -lelf -lz -o cs_delay
-          sudo ./cs_delay
+          make cs_delay
+          sudo ./cs_delay

eBPF_Supermarket/CPU_Subsystem/cs_delay/libbpf_cs_delay/Makefile

@@ -0,0 +1,155 @@
+# Copyright 2023 The LMP Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# author: [email protected]
+#
+# compile the current folder code
+
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LIBBPF_SRC := $(abspath ../../libbpf/src)
+BPFTOOL_SRC := $(abspath ../../bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+LIBBLAZESYM_SRC := $(abspath ../../blazesym/)
+LIBBLAZESYM_INC := $(abspath $(LIBBLAZESYM_SRC)/include)
+LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/riscv64/riscv/' \
+			 | sed 's/loongarch64/loongarch/')
+VMLINUX := ../../vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../../libbpf/include/uapi -I$(dir $(VMLINUX)) -I$(LIBBLAZESYM_INC)
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = cs_delay
+
+CARGO ?= $(shell which cargo)
+ifeq ($(strip $(CARGO)),)
+BZS_APPS :=
+else
+BZS_APPS := profile
+APPS += $(BZS_APPS)
+# Required by libblazesym
+ALL_LDFLAGS += -lrt -ldl -lpthread -lm
+endif
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: $(APPS)
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS)
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+
+$(LIBBLAZESYM_SRC)/target/release/libblazesym.a::
+	$(Q)cd $(LIBBLAZESYM_SRC) && $(CARGO) build --release
+
+$(LIBBLAZESYM_OBJ): $(LIBBLAZESYM_SRC)/target/release/libblazesym.a | $(OUTPUT)
+	$(call msg,LIB, $@)
+	$(Q)cp $(LIBBLAZESYM_SRC)/target/release/libblazesym.a $@
+
+# Build BPF code
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES)		      \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+	$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+# Generate BPF skeletons
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+# Build user-space code
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
+
+$(patsubst %,$(OUTPUT)/%.o,$(BZS_APPS)): $(LIBBLAZESYM_OBJ)
+
+$(BZS_APPS): $(LIBBLAZESYM_OBJ)
+
+# Build application binary
+$(APPS): %: $(OUTPUT)/%.o $(COMMON_OBJ) $(LIBBPF_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:

eBPF_Supermarket/CPU_Subsystem/cs_delay/libbpf_cs_delay/cs_delay.bpf.c

@@ -1,29 +1,29 @@
 #include "vmlinux.h"
-#include <bpf/bpf_helpers.h>		//包含了BPF 辅助函数
+#include <bpf/bpf_helpers.h>		//包含了BPF 辅助函数
 #include <bpf/bpf_tracing.h>
 #include "cs_delay.h"
 
 char LICENSE[] SEC("license") = "Dual BSD/GPL";
 
-// 定义数组映射
+// 定义数组映射
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(max_entries, 1);
 	__type(key, int);
 	__type(value, u64);
-} start SEC(".maps");
+} start SEC(".maps");//记录时间戳；
 
 struct {
 	__uint(type, BPF_MAP_TYPE_RINGBUF);
 	__uint(max_entries, 256 * 1024);
-} rb SEC(".maps");
+} rb SEC(".maps");//环形缓冲区；
 
 SEC("kprobe/schedule")
 int BPF_KPROBE(schedule)
 {
 	u64 t1;
-	t1 = bpf_ktime_get_ns()/1000;	//bpf_ktime_get_ns返回自系统启动以来所经过的时间(以纳秒为单位)。不包括系统挂起的时间。
-	int key=0;
+	t1 = bpf_ktime_get_ns()/1000;	//bpf_ktime_get_ns返回自系统启动以来所经过的时间(以纳秒为单位)。不包括系统挂起的时间。
+	int key =0;
 	bpf_map_update_elem(&start,&key,&t1,BPF_ANY);
 
 	return 0;
@@ -40,20 +40,21 @@ int BPF_KRETPROBE(schedule_exit)
 	{
         	t1 = *val;
         	delay = t2 - t1;
+			bpf_map_delete_elem(&start, &key);
 	}else{
 		return 0;
 	}
-	bpf_map_delete_elem(&start, &key);
+
 
 	struct event *e;
 	e = bpf_ringbuf_reserve(&rb, sizeof(*e), 0);
 	if (!e)	return 0;	
 
-	e->t1=t1;
-	e->t2=t2;
-	e->delay=delay;
+	e->t1=t1;//开始时间
+	e->t2=t2;//结束时间
+	e->delay=delay;//时间间隔
 
-	/* 成功地将其提交到用户空间进行后期处理 */
+	/* 成功地将其提交到用户空间进行后期处理 */
 	bpf_ringbuf_submit(e, 0);
 
 	return 0;

eBPF_Supermarket/CPU_Subsystem/cs_delay/libbpf_cs_delay/cs_delay.c

@@ -4,6 +4,7 @@
 */ 
 
 #include <stdio.h>
+//#include <math.h>//用于对数运算
 #include <unistd.h>
 #include <sys/resource.h>
 #include <bpf/libbpf.h>
@@ -13,20 +14,89 @@
 
 static volatile bool exiting = false;
 
+int count[25]={0};//定义一个count数组，用于汇总schedul()调度时间，以log2(时间间隔)为统计依据；
+
 static void sig_handler(int sig)
 {
 	exiting = true;
 }
 
+
 static int handle_event(void *ctx, void *data,unsigned long data_sz)
 {
 	const struct event *e = data;
 	printf("t1:%lu  t2:%lu  delay:%lu\n",e->t1,e->t2,e->delay);
-
+
+	int dly=(int)(e->delay),i=0;
+	while (dly > 1){
+		dly /= 2;
+		i ++;
+	}
+	count[i]++;//记录时间间隔次数;
 	return 0;
 }
+static int print_hstgram(int i,int max,int per_len)
+{
+	int cnt=count[i];
+	if(per_len==1){
+		while(cnt>0){//打印
+			printf("*");
+			cnt--;
+		}
+	}
+	while(cnt-per_len>=0){//打印
+		printf("*");
+		cnt-=per_len;
+	}
+	printf("\n");
+	return per_len;
+}
+double pow(int n,int k)//实现pow函数
+{
+	if (k > 0)
+		return n * pow(n, k - 1);
+	else if (k == 0)
+		return 1;
+	else
+		return 1.0 / pow(n, -k);
+}
+static void histogram()
+{
+	int log10[15]={0},max=0,per_len=1;
+	for(int i=0;i<10;i++){//log10(count[i]);
+		int tmp=count[i],cnt=0;
+		while (tmp >= 10){
+			tmp /= 10;
+			cnt ++;//幂次
+		}
+		log10[cnt]++;
+	}
+
+	for(int i=0;i<10;i++){//找log10里的最大值；
+		if(max<log10[i])
+			max=i;
+	}
+
+	while(max>0){//pow(10,max);
+		per_len *=10 ;
+		max--;
+	}
+
+	printf("\n%-24s \t%-12s \t%-12s \n","cs_delay","Count","Distribution");
+	printf("%d\t=>\t%-8d \t%-12d \t|",0,1,count[0]);
+	print_hstgram(0,max,per_len);
+	printf("%d\t=>\t%-8d \t%-12d \t|",2,3,count[1]);
+	print_hstgram(1,max,per_len);
+	for(int i=2;i<20;i++){
+		printf("%d\t=>\t%-8d \t%-12d \t|",(int)pow(2,i),(int)pow(2,(i+1))-1,count[i]);
+		print_hstgram(i,max,per_len);
+	}
+	printf("per_len = %d\n",per_len);
+}
 
 
+
+
 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
 {
 	return vfprintf(stderr, format, args);
@@ -94,7 +164,10 @@ int main(int argc, char **argv)
 		exiting = true;			//使用该程序时,将该行代码注释掉 
 
 	}
-
+	/*睡眠*/
+	//sleep(99999999);
+	/*打印直方图*/
+	histogram();
 /* 卸载BPF程序 */
 cleanup:
 	ring_buffer__free(rb);