diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 65a89b33b..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "eBPF_Supermarket/sidecar/libbpf"] - path = eBPF_Supermarket/sidecar/libbpf - url = https://github.com/libbpf/libbpf diff --git a/eBPF_Supermarket/sidecar/bpf/bpf.go b/eBPF_Supermarket/sidecar/bpf/bpf.go index 403c81a3c..a4a2ba61f 100644 --- a/eBPF_Supermarket/sidecar/bpf/bpf.go +++ b/eBPF_Supermarket/sidecar/bpf/bpf.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "github.com/iovisor/gobpf/bcc" ) @@ -50,8 +49,6 @@ func GetTcpFlags(f int, reversed bool) string { } } - // TODO: after verifying how it works, delete it. - res += strconv.Itoa(f) return res } diff --git a/eBPF_Supermarket/sidecar/bpf/podnet/podnet.c b/eBPF_Supermarket/sidecar/bpf/podnet/podnet.c index 6a23490ec..8ad0b1ff2 100644 --- a/eBPF_Supermarket/sidecar/bpf/podnet/podnet.c +++ b/eBPF_Supermarket/sidecar/bpf/podnet/podnet.c @@ -437,6 +437,7 @@ do_trace(void *ctx, struct sk_buff *skb, const char *func_name, void *netdev) /*FILTER_PID*/ struct event_t event = {.pid = pid, .tid = tid}; + event.ts_ns = bpf_ktime_get_ns(); union ___skb_pkt_type type = {}; if (do_trace_skb(&event, ctx, skb, netdev) < 0) @@ -446,7 +447,6 @@ do_trace(void *ctx, struct sk_buff *skb, const char *func_name, void *netdev) bpf_probe_read(&type.value, 1, ((char*)skb) + offsetof(typeof(*skb), __pkt_type_offset)); event.pkt_type = type.pkt_type; - event.ts_ns = bpf_ktime_get_ns(); bpf_strncpy(event.func_name, func_name, FUNCNAME_MAX_LEN); CALL_STACK(ctx, &event); bpf_get_current_comm(&event.task, sizeof(event.task)); @@ -619,8 +619,7 @@ int kprobe__ip_finish_output(struct pt_regs *ctx, struct net *net, struct sock * #if __BCC_iptable static int -__ipt_do_table_in(struct pt_regs *ctx, struct sk_buff *skb, - const struct nf_hook_state *state, struct xt_table *table) +__ipt_do_table_in(struct pt_regs *ctx, struct sk_buff *skb, const struct nf_hook_state *state, struct xt_table *table) { u32 pid = bpf_get_current_pid_tgid(); diff --git a/eBPF_Supermarket/sidecar/bpf/podnet/podnet.go b/eBPF_Supermarket/sidecar/bpf/podnet/podnet.go index 04f66d496..3b035ddc4 100644 --- a/eBPF_Supermarket/sidecar/bpf/podnet/podnet.go +++ b/eBPF_Supermarket/sidecar/bpf/podnet/podnet.go @@ -86,7 +86,7 @@ type bpfEventData struct { type Event struct { Time time.Duration `json:"Time,omitempty"` - NetNS uint32 + NetNS int Comm string `json:"Comm"` IfName string Pid int `json:"pid"` @@ -98,16 +98,16 @@ type Event struct { SkbAddress string L4Proto string TcpFlags string - Cpu uint8 + Cpu int Flags uint8 DestMac string - Len uint32 - Ip uint8 - TotLen uint16 - IcmpType uint8 - IcmPid uint16 - IcmpSeq uint16 + Len int + Ip int + TotLen int + IcmpType int + IcmPid int + IcmpSeq int // ipt info Hook uint32 @@ -126,53 +126,42 @@ type Event struct { } func (e Event) Print() { - fmt.Println("╔======Pod net begin====╗") - fmt.Println("e :", e) - fmt.Println("╚======Pod net end======╝") + // fmt.Println("╔======Pod net begin====╗") + fmt.Println(e) + // fmt.Println("╚======Pod net end======╝") } func getEventFromBpfEventData(bpfEvent bpfEventData) Event { return Event{ - Time: time.Duration(int64(bpfEvent.TsNs)), - Comm: strings.Trim(string(bpfEvent.Comm[:]), "\u0000"), - Pid: int(bpfEvent.Pid), - Tid: int(bpfEvent.Tid), - - FuncName: strings.Trim(string(bpfEvent.FuncName[:]), "\u0000"), - Flags: bpfEvent.Flags, - Cpu: bpfEvent.Cpu, - - // route info - IfName: strings.Trim(string(bpfEvent.IfName[:]), "\u0000"), - NetNS: bpfEvent.NetNS, - - // pkt info - DestMac: bpfEvent.DestMac.ToString(), - Len: bpfEvent.Len, - Ip: bpfEvent.Ip, - L4Proto: bpf.GetProtocolFromInt(int(bpfEvent.L4Proto)), - TotLen: bpfEvent.TotLen, - SAddr: bpfEvent.SAddr.ToString(int(bpfEvent.Ip)), - DAddr: bpfEvent.DAddr.ToString(int(bpfEvent.Ip)), - IcmpType: bpfEvent.IcmpType, - IcmPid: bpfEvent.IcmPid, - IcmpSeq: bpfEvent.IcmpSeq, - Sport: int(bpfEvent.Sport), - Dport: int(bpfEvent.Dport), - TcpFlags: bpf.GetTcpFlags(int(bpfEvent.TcpFlags), true), - - // ipt info - Hook: bpfEvent.Hook, - Pf: bpfEvent.Pf, - Verdict: bpfEvent.Verdict, - TableName: strings.Trim(string(bpfEvent.TableName[:]), "\u0000"), - IptDelay: bpfEvent.IptDelay, - - // skb info - SkbAddress: fmt.Sprintf("0x%x", bpfEvent.SkbAddress), - PktType: bpfEvent.PktType, - - // call stack + Time: time.Duration(int64(bpfEvent.TsNs)), + Comm: strings.Trim(string(bpfEvent.Comm[:]), "\u0000"), + Pid: int(bpfEvent.Pid), + Tid: int(bpfEvent.Tid), + FuncName: strings.Trim(string(bpfEvent.FuncName[:]), "\u0000"), + Flags: bpfEvent.Flags, + Cpu: int(bpfEvent.Cpu), + IfName: strings.Trim(string(bpfEvent.IfName[:]), "\u0000"), + NetNS: int(bpfEvent.NetNS), + DestMac: bpfEvent.DestMac.ToString(), + Len: int(bpfEvent.Len), + Ip: int(bpfEvent.Ip), + L4Proto: bpf.GetProtocolFromInt(int(bpfEvent.L4Proto)), + TotLen: int(bpfEvent.TotLen), + SAddr: bpfEvent.SAddr.ToString(int(bpfEvent.Ip)), + DAddr: bpfEvent.DAddr.ToString(int(bpfEvent.Ip)), + IcmpType: int(bpfEvent.IcmpType), + IcmPid: int(bpfEvent.IcmPid), + IcmpSeq: int(bpfEvent.IcmpSeq), + Sport: int(bpfEvent.Sport), + Dport: int(bpfEvent.Dport), + TcpFlags: bpf.GetTcpFlags(int(bpfEvent.TcpFlags), true), + Hook: bpfEvent.Hook, + Pf: bpfEvent.Pf, + Verdict: bpfEvent.Verdict, + TableName: strings.Trim(string(bpfEvent.TableName[:]), "\u0000"), + IptDelay: bpfEvent.IptDelay, + SkbAddress: fmt.Sprintf("0x%x", bpfEvent.SkbAddress), + PktType: bpfEvent.PktType, KernelStackId: bpfEvent.KernelStackId, KernelIp: bpfEvent.KernelIp, } @@ -224,20 +213,20 @@ func Probe(pidList []int, portList []int, protocolList []string, ch chan<- Event bpf.AttachKprobe(m, "kprobe____dev_queue_xmit", "__dev_queue_xmit") // 14 br process hooks: - bpf.AttachKprobe(m, "kprobe__br_handle_frame", "br_handle_frame") - bpf.AttachKprobe(m, "kprobe__br_handle_frame_finish", "br_handle_frame_finish") - bpf.AttachKprobe(m, "kprobe__br_nf_pre_routing", "br_nf_pre_routing") - bpf.AttachKprobe(m, "kprobe__br_nf_pre_routing_finish", "br_nf_pre_routing_finish") - bpf.AttachKprobe(m, "kprobe__br_pass_frame_up", "br_pass_frame_up") - bpf.AttachKprobe(m, "kprobe__br_netif_receive_skb", "br_netif_receive_skb") - bpf.AttachKprobe(m, "kprobe__br_forward", "br_forward") - bpf.AttachKprobe(m, "kprobe____br_forward", "__br_forward") - bpf.AttachKprobe(m, "kprobe__deliver_clone", "deliver_clone") - bpf.AttachKprobe(m, "kprobe__br_forward_finish", "br_forward_finish") - bpf.AttachKprobe(m, "kprobe__br_nf_forward_ip", "br_nf_forward_ip") - bpf.AttachKprobe(m, "kprobe__br_nf_forward_finish", "br_nf_forward_finish") - bpf.AttachKprobe(m, "kprobe__br_nf_post_routing", "br_nf_post_routing") - bpf.AttachKprobe(m, "kprobe__br_nf_dev_queue_xmit", "br_nf_dev_queue_xmit") + // bpf.AttachKprobe(m, "kprobe__br_handle_frame", "br_handle_frame") + // bpf.AttachKprobe(m, "kprobe__br_handle_frame_finish", "br_handle_frame_finish") + // bpf.AttachKprobe(m, "kprobe__br_nf_pre_routing", "br_nf_pre_routing") + // bpf.AttachKprobe(m, "kprobe__br_nf_pre_routing_finish", "br_nf_pre_routing_finish") + // bpf.AttachKprobe(m, "kprobe__br_pass_frame_up", "br_pass_frame_up") + // bpf.AttachKprobe(m, "kprobe__br_netif_receive_skb", "br_netif_receive_skb") + // bpf.AttachKprobe(m, "kprobe__br_forward", "br_forward") + // bpf.AttachKprobe(m, "kprobe____br_forward", "__br_forward") + // bpf.AttachKprobe(m, "kprobe__deliver_clone", "deliver_clone") + // bpf.AttachKprobe(m, "kprobe__br_forward_finish", "br_forward_finish") + // bpf.AttachKprobe(m, "kprobe__br_nf_forward_ip", "br_nf_forward_ip") + // bpf.AttachKprobe(m, "kprobe__br_nf_forward_finish", "br_nf_forward_finish") + // bpf.AttachKprobe(m, "kprobe__br_nf_post_routing", "br_nf_post_routing") + // bpf.AttachKprobe(m, "kprobe__br_nf_dev_queue_xmit", "br_nf_dev_queue_xmit") // 4 ip layer hooks: bpf.AttachKprobe(m, "kprobe__ip_rcv", "ip_rcv") @@ -245,9 +234,6 @@ func Probe(pidList []int, portList []int, protocolList []string, ch chan<- Event bpf.AttachKprobe(m, "kprobe__ip_output", "ip_output") bpf.AttachKprobe(m, "kprobe__ip_finish_output", "ip_finish_output") - stacks := bcc.NewTable(m.TableId("stacks"), m) - fmt.Println(stacks.Name()) - table := bcc.NewTable(m.TableId("route_event"), m) channel := make(chan []byte, 1000) @@ -272,7 +258,7 @@ func Probe(pidList []int, portList []int, protocolList []string, ch chan<- Event } goEvent := getEventFromBpfEventData(event) - fmt.Println(goEvent) + ch <- goEvent } }() diff --git a/eBPF_Supermarket/sidecar/libbpf b/eBPF_Supermarket/sidecar/libbpf deleted file mode 160000 index 29869d6ef..000000000 --- a/eBPF_Supermarket/sidecar/libbpf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 29869d6ef0893cc1c907705f919fdfd6feb7cc13 diff --git a/eBPF_Supermarket/sidecar/main.go b/eBPF_Supermarket/sidecar/main.go index a94bf3e0a..f4389fda8 100644 --- a/eBPF_Supermarket/sidecar/main.go +++ b/eBPF_Supermarket/sidecar/main.go @@ -6,7 +6,7 @@ import ( "os" "github.com/eswzy/podstat/k8s" - "github.com/eswzy/podstat/perf" + "github.com/eswzy/podstat/perf/net" "github.com/eswzy/podstat/test" "github.com/eswzy/podstat/tools" "github.com/eswzy/podstat/visualization" @@ -61,7 +61,7 @@ func main() { var sidecarPid []int var servicePid []int - var portList = []int{15006, 9080, 80, 8000} + var portList []int // {15006, 9080, 80, 8000} for i := 0; i < len(sidecarProcesses); i++ { sidecarPid = append(sidecarPid, int(sidecarProcesses[i].Pid)) @@ -69,6 +69,17 @@ func main() { for i := 0; i < len(serviceProcesses); i++ { servicePid = append(servicePid, int(serviceProcesses[i].Pid)) } + var pidList []int + pidList = append(pidList, sidecarPid...) + pidList = append(pidList, servicePid...) - perf.GetRequestOverSidecarEvent(sidecarPid, servicePid, portList, *podName) + so := net.SidecarOpt{ + SidecarPort: 8000, + ServicePort: 80, + LocalIP: "127.0.0.1", + PodIp: "UNSET", + NodeIp: "UNSET", + } + + net.GetKernelNetworkEvent(pidList, portList, so) } diff --git a/eBPF_Supermarket/sidecar/perf/net.go b/eBPF_Supermarket/sidecar/perf/net/connect.go similarity index 99% rename from eBPF_Supermarket/sidecar/perf/net.go rename to eBPF_Supermarket/sidecar/perf/net/connect.go index 272121e82..9499f1e30 100644 --- a/eBPF_Supermarket/sidecar/perf/net.go +++ b/eBPF_Supermarket/sidecar/perf/net/connect.go @@ -1,4 +1,4 @@ -package perf +package net import ( "fmt" diff --git a/eBPF_Supermarket/sidecar/perf/net/stack.go b/eBPF_Supermarket/sidecar/perf/net/stack.go new file mode 100644 index 000000000..b3884470c --- /dev/null +++ b/eBPF_Supermarket/sidecar/perf/net/stack.go @@ -0,0 +1,219 @@ +package net + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/eswzy/podstat/bpf/podnet" +) + +type SidecarOpt struct { + SidecarPort int + ServicePort int + LocalIP string + PodIp string + NodeIp string +} + +// type NetworkStackEvent podnet.Event + +type ConnectionType int32 + +const ( + EST ConnectionType = 0 + TRA ConnectionType = 1 + TER ConnectionType = 2 + ACK ConnectionType = 3 + UNKNOWN ConnectionType = 4 +) + +type DirectionType int32 + +const ( + SIDECAR DirectionType = 0 + SERVICE DirectionType = 1 + REMOTE DirectionType = 2 + POD DirectionType = 3 + HOST DirectionType = 4 + OTHER DirectionType = 5 +) + +type ConnectionEstablishment struct { + toService []podnet.Event + toSidecar []podnet.Event +} + +type DataTransfer struct { + req []podnet.Event + res []podnet.Event +} + +type ConnectionTermination struct { + toService []podnet.Event + toSidecar []podnet.Event +} + +type Acknowledgements struct { + e []podnet.Event +} + +type ConnectionOverall struct { + Cs ConnectionEstablishment + Dt DataTransfer + Ct ConnectionTermination + Ak Acknowledgements +} + +type ConnectIdType int + +// getConnectId gets identification (sidecar's random port, usually) for network event in certain connection +func (s SidecarOpt) getConnectId(sPort int, dPort int) (ConnectIdType, error) { + if sPort == s.ServicePort { + return ConnectIdType(dPort), nil + } else if dPort == s.ServicePort { + return ConnectIdType(sPort), nil + } else { + return 0, fmt.Errorf("bad match for port %d -> %d", sPort, dPort) + } +} + +// getPacketDirection gets the destination of this network event +func (s SidecarOpt) getPacketDirection(e podnet.Event) DirectionType { + if e.SAddr == s.LocalIP && e.DAddr == s.LocalIP { // in-pod network event + if e.Dport == s.ServicePort { + return SERVICE + } else if e.Sport == s.ServicePort { + return SIDECAR + } else { + return OTHER + } + } + + if s.PodIp == "UNSET" { + return OTHER + } + if e.SAddr == s.PodIp { // from pod to remote + if e.Sport == s.SidecarPort || e.Sport == s.ServicePort { + return REMOTE + } else { + return OTHER + } + } else if e.DAddr == s.PodIp { // from remote to pod + // TODO: this requires verification + if e.Dport == s.SidecarPort || e.Dport == s.ServicePort { + return POD + } else { + return OTHER + } + } + + if s.NodeIp == "UNSET" { + return OTHER + } + if e.SAddr == s.NodeIp { // from local host to remote host + return HOST + } + + return OTHER +} + +// getConnectType get TCP flag from combined tcp flags +func getConnectType(e podnet.Event) ConnectionType { + if e.TcpFlags == "ACK" { + return ACK + } else if strings.Contains(e.TcpFlags, "SYN") { + return EST + } else if strings.Contains(e.TcpFlags, "PSH") { + return TRA + } else if strings.Contains(e.TcpFlags, "FIN") { + return TER + } else { + return UNKNOWN + } +} + +// monitorLoop does monitor tasks in parallel +func monitorLoop(heap *map[ConnectIdType]ConnectionOverall, timeout time.Duration, ch <-chan bool) { + for { + select { + case <-time.Tick(timeout): // DO NOT USE THIS IN MULTI-THREAD SITUATION + for connectId, conn := range *heap { + // TODO: do analysis and visualization here + b, _ := json.Marshal(conn) + fmt.Println(string(b)) + fmt.Println(conn) + + delete(*heap, connectId) + } + case <-ch: + break + } + } +} + +// GetKernelNetworkEvent is an entrance to monitor network events +func GetKernelNetworkEvent(pidList []int, portList []int, sidecarOpt SidecarOpt) { + var protocolList []string + ch := make(chan podnet.Event, 100000) + heartBeat := make(chan bool) + allEventHeap := make(map[ConnectIdType]ConnectionOverall) + + go podnet.Probe(pidList, portList, protocolList, ch) + go monitorLoop(&allEventHeap, time.Second, heartBeat) + + for { + event := <-ch + direction := sidecarOpt.getPacketDirection(event) + connectionType := getConnectType(event) + if direction == SIDECAR || direction == SERVICE { + id, err := sidecarOpt.getConnectId(event.Sport, event.Dport) + conn, ok := allEventHeap[id] + if ok == false { + conn = ConnectionOverall{} + } + if err != nil { + fmt.Println(err) + } else { + fmt.Println("connect id:", id, "direction:", sidecarOpt.getPacketDirection(event), "type:", connectionType) + heartBeat <- true + switch connectionType { + case ACK: + conn.Ak.e = append(conn.Ak.e, event) + case EST: + if direction == SERVICE { + conn.Cs.toService = append(conn.Cs.toService, event) + } else if direction == SIDECAR { + conn.Cs.toSidecar = append(conn.Cs.toSidecar, event) + } else { + break + } + case TER: + if direction == SERVICE { + conn.Ct.toService = append(conn.Ct.toService, event) + } else if direction == SIDECAR { + conn.Ct.toSidecar = append(conn.Ct.toSidecar, event) + } else { + break + } + case TRA: + if direction == SERVICE { + conn.Dt.req = append(conn.Dt.req, event) + } else if direction == SIDECAR { + conn.Dt.res = append(conn.Dt.res, event) + } else { + break + } + default: + break + } + } + + allEventHeap[id] = conn + } else { + fmt.Println("out of pod", direction) + } + // event.Print() + } +}