Skip to content

Commit

Permalink
Merge pull request #934 from loxilb-io/egr-fix2
Browse files Browse the repository at this point in the history 
PR - #877 egress support - fixes for incluster mode
  • Loading branch information
@UltraInstinct14
UltraInstinct14 authored Jan 16, 2025
2 parents db8220e + 8197780 commit ae51e55
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 21 deletions.
23 changes: 17 additions & 6 deletions api/loxinlp/nlp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (

cmn "github.com/loxilb-io/loxilb/common"
opt "github.com/loxilb-io/loxilb/options"
"github.com/loxilb-io/loxilb/pkg/utils"
tk "github.com/loxilb-io/loxilib"
nlp "github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
Expand Down Expand Up @@ -565,6 +566,11 @@ func AddVxLANBridgeNoHook(vxlanid int, epIntfName string) int {
_, err := nlp.LinkByName(VxlanBridgeName)
if err != nil {

hwAddr, err := utils.GenerateRandomMAC()
if err != nil {
tk.LogIt(tk.LogError, "nlp: Error generating hwAddr\n")
return 403
}
EndpointInterface, err := nlp.LinkByName(epIntfName)
if err != nil {
tk.LogIt(tk.LogWarning, "nlp: Endpoint interface finding Fail\n")
Expand All @@ -577,8 +583,9 @@ func AddVxLANBridgeNoHook(vxlanid int, epIntfName string) int {
}
VxlanDev := &nlp.Vxlan{
LinkAttrs: nlp.LinkAttrs{
Name: VxlanBridgeName,
MTU: 9000, // Static Value for Vxlan in loxiLB
Name: VxlanBridgeName,
MTU: 9000, // Static Value for Vxlan in loxiLB
HardwareAddr: hwAddr,
},
SrcAddr: LocalIPs[0].IP,
VtepDevIndex: EndpointInterface.Attrs().Index,
Expand Down Expand Up @@ -1271,24 +1278,28 @@ func AddRouteNoHook(DestinationIPNet, gateway, proto string) int {
return ret
}

func GetRouteNoHook(destination string) ([]string, error) {
func GetRouteNoHook(destination string) ([]string, string, error) {
var gws []string
var src string

dst := net.ParseIP(destination)
if dst == nil {
return []string{}, errors.New("invalid destination")
return []string{}, "", errors.New("invalid destination")
}

rts, err := nlp.RouteGet(dst)
if err != nil {
return []string{}, errors.New("invalid rt destination")
return []string{}, "", errors.New("invalid rt destination")
}

for _, rt := range rts {
if src == "" {
src = rt.Src.String()
}
gws = append(gws, rt.Gw.String())
}

return gws, nil
return gws, src, nil
}

func DelRouteNoHook(DestinationIPNet string) int {
Expand Down
2 changes: 1 addition & 1 deletion loxilb-ebpf
Submodule loxilb-ebpf updated 1 files
+3 −0 kernel/loxilb_libdp.c
75 changes: 65 additions & 10 deletions pkg/loxinet/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ type CIStateH struct {
ClusterIf string
OGw []string
OGw6 []string
OSrc string
OSrc6 string
initRules bool
initRules6 bool
}

func (ci *CIStateH) BFDSessionNotify(instance string, remote string, ciState string) {
Expand Down Expand Up @@ -285,11 +289,11 @@ func CIInit(args CIKAArgs) *CIStateH {
nCIh.ClusterGw = gw.String()
nCIh.ClusterGw6 = gw6.String()

nCIh.OGw, _ = nlp.GetRouteNoHook("8.8.8.8")
nCIh.OGw6, _ = nlp.GetRouteNoHook("2001:4860:4860::8888")
nCIh.OGw, nCIh.OSrc, _ = nlp.GetRouteNoHook("8.8.8.8")
nCIh.OGw6, nCIh.OSrc6, _ = nlp.GetRouteNoHook("2001:4860:4860::8888")

tk.LogIt(tk.LogInfo, "Cluster IP address %s GW %s oGW %v\n", ip.String(), nCIh.ClusterGw, nCIh.OGw)
tk.LogIt(tk.LogInfo, "Cluster IP6 address %s GW6 %s oGw6 %v\n", ip6.String(), nCIh.ClusterGw, nCIh.OGw6)
tk.LogIt(tk.LogInfo, "Cluster IP address %s GW %s oGW %v oSrc %v \n", ip.String(), nCIh.ClusterGw, nCIh.OGw, nCIh.OSrc)
tk.LogIt(tk.LogInfo, "Cluster IP6 address %s GW6 %s oGw6 %v oSrc6 %v\n", ip6.String(), nCIh.ClusterGw, nCIh.OGw6, nCIh.OSrc6)

}

Expand Down Expand Up @@ -384,7 +388,7 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
if add {
found := false
if tk.IsNetIPv4(dest) {
gws, _ := nlp.GetRouteNoHook("8.8.8.8")
gws, _, _ := nlp.GetRouteNoHook("8.8.8.8")
for _, gw := range gws {
if gw == dest {
found = true
Expand All @@ -395,10 +399,16 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
if !found {
nlp.DelRouteNoHook("0.0.0.0/0")
nlp.AddRouteNoHook("0.0.0.0/0", dest, "static")
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"}
_, err := mh.zr.Rules.DeleteFwRule(fwarg)
if err != nil {
tk.LogIt(tk.LogError, "Failed to delete egress snat for cluster %s\n", mh.has.ClusterNet)
}

}
} else {
found = false
gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
gws, _, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
for _, gw := range gws {
if gw == dest {
found = true
Expand All @@ -408,12 +418,17 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
if !found {
nlp.DelRouteNoHook("::/0")
nlp.AddRouteNoHook("::/0", dest, "static")
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "::/0"}
_, err := mh.zr.Rules.DeleteFwRule(fwarg)
if err != nil {
tk.LogIt(tk.LogError, "Failed to delete egress snat for cluster %s\n", mh.has.ClusterNet6)
}
}
}
} else {
found := false
if tk.IsNetIPv4(dest) {
gws, _ := nlp.GetRouteNoHook("8.8.8.8")
gws, _, _ := nlp.GetRouteNoHook("8.8.8.8")
for _, gw := range gws {
if gw == dest {
found = true
Expand All @@ -422,13 +437,33 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
}
if found {
nlp.DelRouteNoHook("0.0.0.0/0")
for _, gw := range mh.has.OGw {
for i, gw := range mh.has.OGw {
nlp.AddRouteNoHook("0.0.0.0/0", gw, "static")
if i == 0 {
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"}
fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc}
_, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts)
if err != nil {
tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet, err)
}
}
}
} else if !h.initRules {
for i := range mh.has.OGw {
if i == 0 {
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"}
fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc}
_, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts)
if err != nil {
tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet, err)
}
}
}
h.initRules = true
}
} else {
found = false
gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
gws, _, _ := nlp.GetRouteNoHook("2001:4860:4860::8888")
for _, gw := range gws {
if gw == dest {
found = true
Expand All @@ -437,9 +472,29 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) {
}
if found {
nlp.DelRouteNoHook("::/0")
for _, gw := range mh.has.OGw {
for i, gw := range mh.has.OGw6 {
nlp.AddRouteNoHook("::", gw, "static")
if i == 0 {
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "0.0.0.0/0"}
fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc6}
_, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts)
if err != nil {
tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet6, err)
}
}
}
} else if !h.initRules6 {
for i := range mh.has.OGw6 {
if i == 0 {
fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "0.0.0.0/0"}
fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc6}
_, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts)
if err != nil {
tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet6, err)
}
}
}
h.initRules6 = true
}
}
}
Expand Down
13 changes: 9 additions & 4 deletions pkg/loxinet/rules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2081,10 +2081,12 @@ func (R *RuleH) AddFwRule(fwRule cmn.FwRuleArg, fwOptArgs cmn.FwOptArg) (int, er
eFw := R.tables[RtFw].eMap[rt.ruleKey()]

if eFw != nil {
if eFw.act.action.(*ruleFwOpts).opt.fwMark != fwOptArgs.Mark {
eFw.Fw2DP(DpRemove)
eFw.act.action.(*ruleFwOpts).opt.fwMark = fwOptArgs.Mark
eFw.Fw2DP(DpCreate)
if !fwOptArgs.DoSnat {
if eFw.act.action.(*ruleFwOpts).opt.fwMark != fwOptArgs.Mark {
eFw.Fw2DP(DpRemove)
eFw.act.action.(*ruleFwOpts).opt.fwMark = fwOptArgs.Mark
eFw.Fw2DP(DpCreate)
}
}
// If a FW rule already exists
return RuleExistsErr, errors.New("fwrule-exists error")
Expand Down Expand Up @@ -3107,6 +3109,9 @@ func (r *ruleEnt) Fw2DP(work DpWorkT) int {
nWork.FwVal2 = at.opt.fwMark
nWork.FwRecord = at.opt.record
nWork.OnDflt = at.opt.onDflt
if nWork.OnDflt && work == DpRemove {
r.sync = 0
}
default:
return -1
}
Expand Down
14 changes: 14 additions & 0 deletions pkg/utils/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package utils
import (
"bytes"
"context"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/binary"
Expand Down Expand Up @@ -543,3 +544,16 @@ func IPHostCIDRString(ip net.IP) string {
return ip.String() + "/128"
}
}

func GenerateRandomMAC() (net.HardwareAddr, error) {
mac := make([]byte, 6)
_, err := rand.Read(mac)
if err != nil {
return nil, err
}

mac[0] |= 0x02
mac[0] &= 0xfe

return net.HardwareAddr(mac), nil
}

0 comments on commit ae51e55

Please sign in to comment.