Merge pull request #730 from nanshuaibo/develop

kvm_watcher:优化代码，修改kvm msi中断挂载函数,完善内核态kvm ioctl程序
linuxkerneltravel · Mar 25, 2024 · f79bef6 · f79bef6
2 parents 31e3154 + 57f375e
commit f79bef6
Show file tree

Hide file tree

Showing 13 changed files with 160 additions and 707 deletions.
diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile
@@ -96,6 +96,6 @@ test: bpf
 
 clean:
 	cd src && rm -f *.o *.skel.h *.bpf.o
-	rm -f $(notdir $(APP))
-	rm -rf include/vmlinux.h
+	rm -rf $(notdir $(APP)) include/vmlinux.h temp
+
 
diff --git a/eBPF_Supermarket/kvm_watcher/README.md b/eBPF_Supermarket/kvm_watcher/README.md
@@ -31,14 +31,13 @@
 **安装依赖：**
 
 ```
-sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$(uname -r) linux-cloud-tools-$(uname -r)
-sudo modprobe kvm && sudo modprobe kvm-intel //加载kvm模块
+make deps
 ```
 
+
 **编译运行：**
 
 ```
-make deps
 make bpf
 sudo ./kvm_watcher [options]
 make clean
@@ -93,6 +92,8 @@ BPF program used for monitoring KVM event
 
 `-w`：记录vcpu唤醒时的相关信息
 
+`-l`：记录kvm相关ioctl系统调用命令字
+
 `-p`：指定kvm虚拟机进程pid
 
 `-t`：监控时间
@@ -114,9 +115,6 @@ BPF program used for monitoring KVM event
 │   ├── kvm_mmu.h
 │   ├── kvm_vcpu.h
 │   └── kvm_watcher.h           //公共头文件
-├── kvm_exit_bcc                //bcc版本的vm exit实现
-│   ├── kvmexit_example.txt
-│   └── kvmexit.py
 ├── Makefile                    //编译脚本
 ├── README.md
 ├── src                         

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h
@@ -54,8 +54,7 @@ struct exit {
     unsigned int vcpu_id;
 };
 
-static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int trace_kvm_exit(struct exit *ctx) {
     u32 reason;
     reason = (u32)ctx->exit_reason;
     // 如果是节能停止退出，就不采集数据

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_hypercall.h b/eBPF_Supermarket/kvm_watcher/include/kvm_hypercall.h
@@ -54,8 +54,8 @@ struct {
 } hc_count SEC(".maps");
 
 static int entry_emulate_hypercall(struct kvm_vcpu *vcpu, void *rb,
-                                   struct common_event *e, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+                                   struct common_event *e) {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
     u64 nr, a0, a1, a2, a3;
     nr = kvm_rax_read(vcpu);  // 超级调用号
     // 超级调用参数

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_ioctl.h b/eBPF_Supermarket/kvm_watcher/include/kvm_ioctl.h
@@ -21,11 +21,80 @@
 
 #include "kvm_watcher.h"
 #include "vmlinux.h"
+#include <asm-generic/ioctl.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_core_read.h>
 #include <bpf/bpf_tracing.h>
+#include <linux/version.h>
+
+#define KVMIO 0xAE
+#define KVM_CREATE_VM _IO(KVMIO, 0x01) /* returns a VM fd */
+#define KVM_CREATE_VCPU _IO(KVMIO, 0x41)
+#define KVM_GET_VCPU_EVENTS _IOR(KVMIO, 0x9f, struct kvm_vcpu_events)
+#define KVM_SET_VCPU_EVENTS _IOW(KVMIO, 0xa0, struct kvm_vcpu_events)
+#define KVM_SET_USER_MEMORY_REGION \
+    _IOW(KVMIO, 0x46, struct kvm_userspace_memory_region)
+#define KVM_TRANSLATE _IOWR(KVMIO, 0x85, struct kvm_translation)
+#define KVM_INTERRUPT _IOW(KVMIO, 0x86, struct kvm_interrupt)
+#define KVM_RUN _IO(KVMIO, 0x80)
 
 static int trace_kvm_ioctl(struct trace_event_raw_sys_enter *args) {
+    int fd = (int)args->args[0];
+    unsigned int cmd = (unsigned int)args->args[1];
+    unsigned long arg = (unsigned long)args->args[2];
+    char buf[256]; // 创建一个缓冲区来存储格式化后的字符串
+    switch (cmd) {
+        case KVM_CREATE_VM:
+            bpf_printk("KVM_CREATE_VM: fd=%d\n", fd);
+            break;
+        case KVM_CREATE_VCPU: {
+            int vcpu_id;
+            bpf_probe_read(&vcpu_id, sizeof(vcpu_id), (void *)arg);
+            bpf_printk("KVM_CREATE_VCPU: fd=%d, vcpu_id=%d\n", fd, vcpu_id);
+            break;
+        }
+        case KVM_SET_USER_MEMORY_REGION: {
+            struct kvm_userspace_memory_region region;
+            bpf_probe_read(&region, sizeof(region), (void *)arg);
+            // 打印或处理 region 数据
+            bpf_printk(
+                "KVM_SET_USER_MEMORY_REGION: fd=%d, slot=%u,flags=%u\n",
+                fd, region.slot, region.flags);
+            bpf_printk(
+                "KVM_SET_USER_MEMORY_REGION: guest_phys_addr=%llx, memory_size=%lluK,userspace_addr=%llx\n",
+                region.guest_phys_addr,
+                region.memory_size / 1024, region.userspace_addr);
+            break;
+        }
+        case KVM_GET_VCPU_EVENTS:
+        case KVM_SET_VCPU_EVENTS: {
+            struct kvm_vcpu_events events;
+            bpf_probe_read(&events, sizeof(events), (void *)arg);
+            // 打印或处理 events 数据
+            bpf_printk(
+                "KVM_SET/GET_VCPU_EVENTS: fd=%d, exception=%u, interrupt=%u\n",
+                fd, events.exception.nr, events.interrupt.nr);
+            break;
+        }
+        case KVM_TRANSLATE: {
+            struct kvm_translation tr;
+            bpf_probe_read(&tr, sizeof(tr), (void *)arg);
+            bpf_printk(
+                "KVM_TRANSLATE: fd=%d,linear_address=%llx, "
+                "physical_address=%llx\n",
+                fd, tr.linear_address, tr.physical_address);
+            break;
+        }
+        case KVM_INTERRUPT: {
+            struct kvm_interrupt irq;
+            bpf_probe_read(&irq, sizeof(irq), (void *)arg);
+            bpf_printk("KVM_INTERRUPT:fd=%d,interrupt vector:%d\n", fd,
+                       irq.irq);
+            break;
+        }
+        default:
+            break;
+    }
     return 0;
 }
 

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_irq.h b/eBPF_Supermarket/kvm_watcher/include/kvm_irq.h
@@ -38,8 +38,7 @@ struct {
     __type(value, u64);
 } irq_inject_delay SEC(".maps");
 
-static int entry_kvm_pic_set_irq(int irq, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int entry_kvm_pic_set_irq(int irq) {
     if (irq < 0 || irq >= PIC_NUM_PINS) {
         return 0;
     }
@@ -77,8 +76,7 @@ static int exit_kvm_pic_set_irq(struct kvm_pic *s, int irq, int ret, void *rb,
     return 0;
 }
 
-static int entry_kvm_ioapic_set_irq(int irq, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int entry_kvm_ioapic_set_irq(int irq) {
     if (irq < 0 || irq >= IOAPIC_NUM_PINS) {
         return 0;
     }
@@ -118,17 +116,25 @@ static int exit_kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int ret,
     return 0;
 }
 
-static int entry_kvm_set_msi_irq(struct kvm *kvm, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int entry_kvm_set_msi(struct kvm *kvm,
+                             struct kvm_kernel_irq_routing_entry *routing_entry,
+                             int level) {
+    bool x2apic_format;
+    bpf_probe_read_kernel(&x2apic_format, sizeof(bool),
+                          &kvm->arch.x2apic_format);
+    if (x2apic_format && (routing_entry->msi.address_hi & 0xff))
+        return 0;
+    if (!level)
+        return 0;
     pid_t tid = (u32)bpf_get_current_pid_tgid();
     u64 ts = bpf_ktime_get_ns();
     bpf_map_update_elem(&irq_set_delay, &tid, &ts, BPF_ANY);
     return 0;
 }
 
-static int exit_kvm_set_msi_irq(
-    struct kvm *kvm, struct kvm_kernel_irq_routing_entry *routing_entry,
-    void *rb, struct common_event *e) {
+static int exit_kvm_set_msi(struct kvm *kvm,
+                            struct kvm_kernel_irq_routing_entry *routing_entry,
+                            void *rb, struct common_event *e) {
     struct msi_msg msg = {.address_lo = routing_entry->msi.address_lo,
                           .address_hi = routing_entry->msi.address_hi,
                           .data = routing_entry->msi.data};
@@ -156,8 +162,7 @@ static int exit_kvm_set_msi_irq(
     return 0;
 }
 
-static int entry_vmx_inject_irq(struct kvm_vcpu *vcpu, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int entry_vmx_inject_irq(struct kvm_vcpu *vcpu) {
     u32 irq_nr;
     bool rei;
     bpf_probe_read_kernel(&irq_nr, sizeof(u32), &vcpu->arch.interrupt.nr);

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h
@@ -25,8 +25,6 @@
 #include <bpf/bpf_core_read.h>
 #include <bpf/bpf_tracing.h>
 
-#define PAGE_SHIFT 12
-
 struct {
     __uint(type, BPF_MAP_TYPE_HASH);
     __uint(max_entries, 8192);
@@ -50,8 +48,7 @@ struct page_fault {
     char __data[0];
 };
 
-static int trace_page_fault(struct page_fault *ctx, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int trace_page_fault(struct page_fault *ctx) {
     u64 ts = bpf_ktime_get_ns();
     u64 addr = ctx->fault_address;
     bpf_map_update_elem(&pf_delay, &addr, &ts, BPF_ANY);
@@ -101,8 +98,7 @@ static int trace_tdp_page_fault(struct kvm_vcpu *vcpu,
 }
 
 static int trace_kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
-                                    u64 error_code, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+                                    u64 error_code) {
     if (error_code & PFERR_RSVD_MASK) {
         u64 ts = bpf_ktime_get_ns();
         u64 gfn = cr2_or_gpa >> PAGE_SHIFT;
@@ -141,7 +137,7 @@ static int trace_handle_mmio_page_fault(struct mmio_page_fault *ctx, void *rb,
         bpf_map_update_elem(&pf_count, &gfn, &new_count, BPF_ANY);
     }
     e->page_fault_data.delay = delay;
-    e->page_fault_data.addr = gfn;
+    e->page_fault_data.addr = gfn << PAGE_SHIFT;
     e->page_fault_data.error_code = PFERR_RSVD_MASK;
     e->process.pid = bpf_get_current_pid_tgid() >> 32;
     bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm));

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h
@@ -55,8 +55,7 @@ struct {
 } vcpu_tid SEC(".maps");
 
 // 记录vcpu_halt的id信息
-static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu) {
     u32 tid = bpf_get_current_pid_tgid();
     u32 vcpu_id;
     bpf_probe_read_kernel(&vcpu_id, sizeof(vcpu->vcpu_id), &vcpu->vcpu_id);
@@ -65,8 +64,8 @@ static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu, pid_t vm_pid) {
 }
 // 使用kvm_vcpu_halt记录的数据，来获取vcpu的启动信息
 static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx, void *rb,
-                                 struct common_event *e, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+                                 struct common_event *e) {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
     u32 tid = bpf_get_current_pid_tgid();
     u32 *vcpu_id = bpf_map_lookup_elem(&vcpu_tid, &tid);
     if (!vcpu_id) {
@@ -87,8 +86,8 @@ static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx, void *rb,
 }
 
 static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb,
-                                  struct common_event *e, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+                                  struct common_event *e) {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
     u32 tid = bpf_get_current_pid_tgid();
     RESERVE_RINGBUF_ENTRY(rb, e);
     u64 time = bpf_ktime_get_ns();
@@ -127,8 +126,8 @@ static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu, void *rb,
 static int trace_mark_page_dirty_in_slot(struct kvm *kvm,
                                          const struct kvm_memory_slot *memslot,
                                          gfn_t gfn, void *rb,
-                                         struct common_event *e, pid_t vm_pid) {
-    CHECK_PID(vm_pid);
+                                         struct common_event *e) {
+    u32 pid = bpf_get_current_pid_tgid() >> 32;
     u32 flags;
     struct kvm_memory_slot *slot;
     bpf_probe_read_kernel(&slot, sizeof(memslot), &memslot);

diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h
@@ -22,6 +22,8 @@
 #define TASK_COMM_LEN 16
 #define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0)
 
+#define PAGE_SHIFT 12
+
 #define NS_TO_US_FACTOR 1000.0
 #define NS_TO_MS_FACTOR 1000000.0
 
@@ -66,9 +68,6 @@
         }                                         \
     } while (0)
 
-// 定义清屏宏
-#define CLEAR_SCREEN() printf("\033[2J\033[H\n")
-
 #define RING_BUFFER_TIMEOUT_MS 100
 
 #define RESERVE_RINGBUF_ENTRY(rb, e)                             \
@@ -79,10 +78,9 @@
         e = _tmp;                                                \
     } while (0)
 
-#define CHECK_PID(vm_pid)                         \
-    __u32 pid = bpf_get_current_pid_tgid() >> 32; \
-    if ((vm_pid) > 0 && pid != (vm_pid)) {        \
-        return 0;                                 \
+#define CHECK_PID(vm_pid)                                                 \
+    if ((vm_pid) > 0 && (bpf_get_current_pid_tgid() >> 32) != (vm_pid)) { \
+        return 0;                                                         \
     }
 
 struct reason_info {