Skip to content

Commit

Permalink
Implement detachable state
Browse files Browse the repository at this point in the history 
We try to calculate state of EdgeNode and objects inside of it.
We construct the state based on info and metric messages from the
controller. Info and metric messages are great for debugging, but in
case of long time work we may hit the problem where state
calculation consume more and more time.

Let's slightly refactor the state logic to be able to store it locally
and reuse.

Signed-off-by: Petr Fedchenkov <[email protected]>
  • Loading branch information
@giggsoff
giggsoff committed Sep 29, 2023
1 parent ef2844a commit fdf746e
Show file tree
Hide file tree
Showing 13 changed files with 371 additions and 215 deletions.
85 changes: 57 additions & 28 deletions pkg/eve/applications.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,17 @@ type AppInstState struct {
ExternalIP string
InternalPort string
ExternalPort string
Metadata string
MemoryUsed uint32
MemoryAvail uint32
CPUUsage int
Macs []string
Volumes map[string]uint32

prevCPUNS uint64
prevCPUNSTime time.Time
deleted bool
infoTime time.Time
PrevCPUNS uint64
PrevCPUNSTime time.Time
Deleted bool
InfoTime time.Time
}

func appStateHeader() string {
Expand Down Expand Up @@ -117,7 +118,7 @@ func getPortMapping(appConfig *config.AppInstanceConfig, qemuPorts map[string]st
}

func (ctx *State) initApplications(ctrl controller.Cloud, dev *device.Ctx) error {
ctx.applications = make(map[string]*AppInstState)
ctx.Applications = make(map[string]*AppInstState)
for _, el := range dev.GetApplicationInstances() {
app, err := ctrl.GetApplicationInstanceConfig(el)
if err != nil {
Expand All @@ -144,35 +145,56 @@ func (ctx *State) initApplications(ctrl controller.Cloud, dev *device.Ctx) error
Volumes: volumes,
UUID: app.Uuidandversion.Uuid,
}
ctx.applications[app.Uuidandversion.Uuid] = appStateObj
ctx.Applications[app.Uuidandversion.Uuid] = appStateObj
}
return nil
}

func (ctx *State) applyOldStateApps(state *State) {
for stateID, stateEL := range state.Applications {
found := false
for id := range ctx.Applications {
if id != stateID {
continue
}
ctx.Applications[id] = stateEL
found = true
}
if !found {
if stateEL.Deleted {
continue
}
stateEL.AdamState = notInControllerConfig
ctx.Applications[stateID] = stateEL
}
}
}

func (ctx *State) processApplicationsByMetric(msg *metrics.ZMetricMsg) {
if appMetrics := msg.GetAm(); appMetrics != nil {
for _, appMetric := range appMetrics {
for _, el := range ctx.applications {
for _, el := range ctx.Applications {
if appMetric.AppID == el.UUID {
el.MemoryAvail = appMetric.Memory.GetAvailMem()
el.MemoryUsed = appMetric.Memory.GetUsedMem()
// if not restarted
if el.prevCPUNS < appMetric.Cpu.TotalNs {
el.CPUUsage = int(float32(appMetric.Cpu.TotalNs-el.prevCPUNS) / float32(msg.GetAtTimeStamp().AsTime().Sub(el.prevCPUNSTime).Nanoseconds()) * 100.0)
if el.PrevCPUNS < appMetric.Cpu.TotalNs {
el.CPUUsage = int(float32(appMetric.Cpu.TotalNs-el.PrevCPUNS) / float32(msg.GetAtTimeStamp().AsTime().Sub(el.PrevCPUNSTime).Nanoseconds()) * 100.0)
}
el.prevCPUNS = appMetric.Cpu.TotalNs
el.prevCPUNSTime = msg.GetAtTimeStamp().AsTime()
el.PrevCPUNS = appMetric.Cpu.TotalNs
el.PrevCPUNSTime = msg.GetAtTimeStamp().AsTime()
break
}
}
}
}
}

//nolint:cyclop
func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) {
switch im.GetZtype() {
case info.ZInfoTypes_ZiVolume:
for _, app := range ctx.applications {
for _, app := range ctx.Applications {
if len(app.Volumes) == 0 {
continue
}
Expand All @@ -188,16 +210,23 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) {
app.EVEState = fmt.Sprintf("%s (%d%%)", info.ZSwState_DOWNLOAD_STARTED.String(), int(percent)/len(app.Volumes))
}
}
case info.ZInfoTypes_ZiAppInstMetaData:
for _, app := range ctx.Applications {
if im.GetAmdinfo().Uuid == app.UUID {
app.Metadata = string(im.GetAmdinfo().Data)
break
}
}
case info.ZInfoTypes_ZiApp:
appStateObj, ok := ctx.applications[im.GetAinfo().AppID]
appStateObj, ok := ctx.Applications[im.GetAinfo().AppID]
if !ok {
appStateObj = &AppInstState{
Name: im.GetAinfo().AppName,
Image: "-",
AdamState: notInControllerConfig,
UUID: im.GetAinfo().AppID,
}
ctx.applications[im.GetAinfo().AppID] = appStateObj
ctx.Applications[im.GetAinfo().AppID] = appStateObj
}
appStateObj.EVEState = im.GetAinfo().State.String()
if len(im.GetAinfo().AppErr) > 0 {
Expand Down Expand Up @@ -227,20 +256,20 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) {
//check appStateObj not defined in adam
if appStateObj.AdamState != inControllerConfig {
if im.GetAinfo().AppID == appStateObj.UUID {
appStateObj.deleted = false //if in recent ZInfoTypes_ZiApp, then not deleted
appStateObj.Deleted = false //if in recent ZInfoTypes_ZiApp, then not deleted
}
}
if im.GetAinfo().State == info.ZSwState_INVALID {
appStateObj.deleted = true
appStateObj.Deleted = true
}
appStateObj.infoTime = im.AtTimeStamp.AsTime()
appStateObj.InfoTime = im.AtTimeStamp.AsTime()
case info.ZInfoTypes_ZiNetworkInstance: //try to find ips from NetworkInstances
for _, el := range im.GetNiinfo().IpAssignments {
// nothing to show if no IpAddress received
if len(el.IpAddress) == 0 {
continue
}
for _, appStateObj := range ctx.applications {
for _, appStateObj := range ctx.Applications {
for ind, mac := range appStateObj.Macs {
if mac == el.MacAddress {
appStateObj.InternalIP[ind] = el.IpAddress[0]
Expand All @@ -250,18 +279,18 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) {
}
case info.ZInfoTypes_ZiDevice:
for _, el := range im.GetDinfo().AppInstances {
if _, ok := ctx.applications[el.Uuid]; !ok {
if _, ok := ctx.Applications[el.Uuid]; !ok {
appStateObj := &AppInstState{
Name: el.Name,
Image: "-",
AdamState: notInControllerConfig,
EVEState: "UNKNOWN",
UUID: el.Uuid,
}
ctx.applications[el.Uuid] = appStateObj
ctx.Applications[el.Uuid] = appStateObj
}
}
for _, appStateObj := range ctx.applications {
for _, appStateObj := range ctx.Applications {
seen := false
for _, el := range im.GetDinfo().AppInstances {
if appStateObj.UUID == el.Uuid {
Expand Down Expand Up @@ -289,12 +318,12 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) {
appStateObj.ExternalIP = "127.0.0.1"
}
//check appStateObj not defined in adam
if appStateObj.AdamState != inControllerConfig && appStateObj.infoTime.Before(im.AtTimeStamp.AsTime()) {
appStateObj.deleted = true
if appStateObj.AdamState != inControllerConfig && appStateObj.InfoTime.Before(im.AtTimeStamp.AsTime()) {
appStateObj.Deleted = true
for _, el := range im.GetDinfo().AppInstances {
//if in recent ZInfoTypes_ZiDevice with timestamp after ZInfoTypes_ZiApp, than not deleted
//if in recent ZInfoTypes_ZiDevice with timestamp after ZInfoTypes_ZiApp, then not deleted
if el.Uuid == appStateObj.UUID {
appStateObj.deleted = false
appStateObj.Deleted = false
}
}
}
Expand All @@ -308,8 +337,8 @@ func (ctx *State) printPodListLines() error {
if _, err := fmt.Fprintln(w, appStateHeader()); err != nil {
return err
}
appStatesSlice := make([]*AppInstState, 0, len(ctx.Applications()))
appStatesSlice = append(appStatesSlice, ctx.Applications()...)
appStatesSlice := make([]*AppInstState, 0, len(ctx.NotDeletedApplications()))
appStatesSlice = append(appStatesSlice, ctx.NotDeletedApplications()...)
sort.SliceStable(appStatesSlice, func(i, j int) bool {
return appStatesSlice[i].Name < appStatesSlice[j].Name
})
Expand All @@ -322,7 +351,7 @@ func (ctx *State) printPodListLines() error {
}

func (ctx *State) printPodListJSON() error {
result, err := json.MarshalIndent(ctx.Applications(), "", " ")
result, err := json.MarshalIndent(ctx.NotDeletedApplications(), "", " ")
if err != nil {
return err
}
Expand Down
66 changes: 66 additions & 0 deletions pkg/eve/eve.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package eve

import (
"time"

"github.com/lf-edge/eden/pkg/controller"
"github.com/lf-edge/eden/pkg/device"
"github.com/lf-edge/eve/api/go/info"
"github.com/lf-edge/eve/api/go/metrics"
)

// NodeState describes state of edge node
type NodeState struct {
UsedMem uint32
AvailMem uint32
UsedPercentageMem float64

LastRebootTime time.Time
LastRebootReason string

// interface to ip mapping
RemoteIPs map[string][]string

LastSeen time.Time

Version string
}

func (ctx *State) initNodeState(_ controller.Cloud, _ *device.Ctx) error {
ctx.EveState = &NodeState{}
return nil
}

func (ctx *State) applyOldStateNodeState(state *State) {
ctx.EveState = state.EveState
}

func (ctx *State) processNodeStateByInfo(msg *info.ZInfoMsg) {
infoTime := msg.AtTimeStamp.AsTime()
if infoTime.After(ctx.EveState.LastSeen) {
ctx.EveState.LastSeen = infoTime
}
if deviceInfo := msg.GetDinfo(); deviceInfo != nil {
ctx.EveState.RemoteIPs = make(map[string][]string)
for _, nw := range deviceInfo.Network {
ctx.EveState.RemoteIPs[nw.LocalName] = nw.IPAddrs
}
ctx.EveState.LastRebootTime = deviceInfo.LastRebootTime.AsTime()
ctx.EveState.LastRebootReason = deviceInfo.LastRebootReason
if len(deviceInfo.SwList) > 0 {
ctx.EveState.Version = deviceInfo.SwList[0].ShortVersion
}
}
}

func (ctx *State) processNodeStateByMetric(msg *metrics.ZMetricMsg) {
metricTime := msg.AtTimeStamp.AsTime()
if metricTime.After(ctx.EveState.LastSeen) {
ctx.EveState.LastSeen = metricTime
}
if deviceMetric := msg.GetDm(); deviceMetric != nil {
ctx.EveState.AvailMem = deviceMetric.Memory.GetAvailMem()
ctx.EveState.UsedMem = deviceMetric.Memory.GetUsedMem()
ctx.EveState.UsedPercentageMem = deviceMetric.Memory.GetUsedPercentage()
}
}
42 changes: 31 additions & 11 deletions pkg/eve/networks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ type NetInstState struct {
AdamState string
EveState string
Activated bool
deleted bool
Deleted bool
}

func netInstStateHeader() string {
Expand All @@ -40,7 +40,7 @@ func (netInstStateObj *NetInstState) toString() string {
}

func (ctx *State) initNetworks(ctrl controller.Cloud, dev *device.Ctx) error {
ctx.networks = make(map[string]*NetInstState)
ctx.Networks = make(map[string]*NetInstState)
for _, el := range dev.GetNetworkInstances() {
ni, err := ctrl.GetNetworkInstanceConfig(el)
if err != nil {
Expand All @@ -55,15 +55,35 @@ func (ctx *State) initNetworks(ctrl controller.Cloud, dev *device.Ctx) error {
CIDR: ni.Ip.Subnet,
NetworkType: ni.InstType.String(),
}
ctx.networks[ni.Uuidandversion.Uuid] = netInstStateObj
ctx.Networks[ni.Uuidandversion.Uuid] = netInstStateObj
}
return nil
}

func (ctx *State) applyOldStateNetworks(state *State) {
for stateID, stateEL := range state.Networks {
found := false
for id := range ctx.Networks {
if id != stateID {
continue
}
ctx.Networks[id] = stateEL
found = true
}
if !found {
if stateEL.Deleted {
continue
}
stateEL.AdamState = notInControllerConfig
ctx.Networks[stateID] = stateEL
}
}
}

func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) {
switch im.GetZtype() {
case info.ZInfoTypes_ZiNetworkInstance:
netInstStateObj, ok := ctx.networks[im.GetNiinfo().GetNetworkID()]
netInstStateObj, ok := ctx.Networks[im.GetNiinfo().GetNetworkID()]
if !ok {
netInstStateObj = &NetInstState{
Name: im.GetNiinfo().GetDisplayname(),
Expand All @@ -73,7 +93,7 @@ func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) {
EveState: "UNKNOWN",
NetworkType: (config.ZNetworkInstType)(int32(im.GetNiinfo().InstType)).String(),
}
ctx.networks[im.GetNiinfo().GetNetworkID()] = netInstStateObj
ctx.Networks[im.GetNiinfo().GetNetworkID()] = netInstStateObj
}
netInstStateObj.EveState = im.GetNiinfo().State.String()
netInstStateObj.Activated = im.GetNiinfo().Activated
Expand All @@ -98,12 +118,12 @@ func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) {
if !netInstStateObj.Activated &&
im.GetNiinfo().State != info.ZNetworkInstanceState_ZNETINST_STATE_INIT &&
netInstStateObj.AdamState == notInControllerConfig {
netInstStateObj.deleted = true
netInstStateObj.Deleted = true
}

if im.GetNiinfo().State == info.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED &&
netInstStateObj.AdamState == notInControllerConfig {
netInstStateObj.deleted = true
netInstStateObj.Deleted = true
}
}
}
Expand All @@ -112,7 +132,7 @@ func (ctx *State) processNetworksByMetric(msg *metrics.ZMetricMsg) {
if networkMetrics := msg.GetNm(); networkMetrics != nil {
for _, networkMetric := range networkMetrics {
// XXX use [uuid] instead of loop
for _, el := range ctx.networks {
for _, el := range ctx.Networks {
if networkMetric.NetworkID == el.UUID {
el.Stats = networkMetric.GetNetworkStats().String()
break
Expand All @@ -128,8 +148,8 @@ func (ctx *State) printNetListLines() error {
if _, err := fmt.Fprintln(w, netInstStateHeader()); err != nil {
return err
}
netInstStatesSlice := make([]*NetInstState, 0, len(ctx.Networks()))
netInstStatesSlice = append(netInstStatesSlice, ctx.Networks()...)
netInstStatesSlice := make([]*NetInstState, 0, len(ctx.NotDeletedNetworks()))
netInstStatesSlice = append(netInstStatesSlice, ctx.NotDeletedNetworks()...)
sort.SliceStable(netInstStatesSlice, func(i, j int) bool {
return netInstStatesSlice[i].Name < netInstStatesSlice[j].Name
})
Expand All @@ -142,7 +162,7 @@ func (ctx *State) printNetListLines() error {
}

func (ctx *State) printNetListJSON() error {
result, err := json.MarshalIndent(ctx.Networks(), "", " ")
result, err := json.MarshalIndent(ctx.NotDeletedNetworks(), "", " ")
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit fdf746e

Please sign in to comment.