diff --git a/pkg/eve/applications.go b/pkg/eve/applications.go index 7240ce0bd..92858c6f4 100644 --- a/pkg/eve/applications.go +++ b/pkg/eve/applications.go @@ -30,16 +30,17 @@ type AppInstState struct { ExternalIP string InternalPort string ExternalPort string + Metadata string MemoryUsed uint32 MemoryAvail uint32 CPUUsage int Macs []string Volumes map[string]uint32 - prevCPUNS uint64 - prevCPUNSTime time.Time - deleted bool - infoTime time.Time + PrevCPUNS uint64 + PrevCPUNSTime time.Time + Deleted bool + InfoTime time.Time } func appStateHeader() string { @@ -117,7 +118,7 @@ func getPortMapping(appConfig *config.AppInstanceConfig, qemuPorts map[string]st } func (ctx *State) initApplications(ctrl controller.Cloud, dev *device.Ctx) error { - ctx.applications = make(map[string]*AppInstState) + ctx.Applications = make(map[string]*AppInstState) for _, el := range dev.GetApplicationInstances() { app, err := ctrl.GetApplicationInstanceConfig(el) if err != nil { @@ -144,24 +145,44 @@ func (ctx *State) initApplications(ctrl controller.Cloud, dev *device.Ctx) error Volumes: volumes, UUID: app.Uuidandversion.Uuid, } - ctx.applications[app.Uuidandversion.Uuid] = appStateObj + ctx.Applications[app.Uuidandversion.Uuid] = appStateObj } return nil } +func (ctx *State) applyOldStateApps(state *State) { + for stateID, stateEL := range state.Applications { + found := false + for id := range ctx.Applications { + if id != stateID { + continue + } + ctx.Applications[id] = stateEL + found = true + } + if !found { + if stateEL.Deleted { + continue + } + stateEL.AdamState = notInControllerConfig + ctx.Applications[stateID] = stateEL + } + } +} + func (ctx *State) processApplicationsByMetric(msg *metrics.ZMetricMsg) { if appMetrics := msg.GetAm(); appMetrics != nil { for _, appMetric := range appMetrics { - for _, el := range ctx.applications { + for _, el := range ctx.Applications { if appMetric.AppID == el.UUID { el.MemoryAvail = appMetric.Memory.GetAvailMem() el.MemoryUsed = appMetric.Memory.GetUsedMem() // if not restarted - if el.prevCPUNS < appMetric.Cpu.TotalNs { - el.CPUUsage = int(float32(appMetric.Cpu.TotalNs-el.prevCPUNS) / float32(msg.GetAtTimeStamp().AsTime().Sub(el.prevCPUNSTime).Nanoseconds()) * 100.0) + if el.PrevCPUNS < appMetric.Cpu.TotalNs { + el.CPUUsage = int(float32(appMetric.Cpu.TotalNs-el.PrevCPUNS) / float32(msg.GetAtTimeStamp().AsTime().Sub(el.PrevCPUNSTime).Nanoseconds()) * 100.0) } - el.prevCPUNS = appMetric.Cpu.TotalNs - el.prevCPUNSTime = msg.GetAtTimeStamp().AsTime() + el.PrevCPUNS = appMetric.Cpu.TotalNs + el.PrevCPUNSTime = msg.GetAtTimeStamp().AsTime() break } } @@ -169,10 +190,11 @@ func (ctx *State) processApplicationsByMetric(msg *metrics.ZMetricMsg) { } } +//nolint:cyclop func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { switch im.GetZtype() { case info.ZInfoTypes_ZiVolume: - for _, app := range ctx.applications { + for _, app := range ctx.Applications { if len(app.Volumes) == 0 { continue } @@ -188,8 +210,15 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { app.EVEState = fmt.Sprintf("%s (%d%%)", info.ZSwState_DOWNLOAD_STARTED.String(), int(percent)/len(app.Volumes)) } } + case info.ZInfoTypes_ZiAppInstMetaData: + for _, app := range ctx.Applications { + if im.GetAmdinfo().Uuid == app.UUID { + app.Metadata = string(im.GetAmdinfo().Data) + break + } + } case info.ZInfoTypes_ZiApp: - appStateObj, ok := ctx.applications[im.GetAinfo().AppID] + appStateObj, ok := ctx.Applications[im.GetAinfo().AppID] if !ok { appStateObj = &AppInstState{ Name: im.GetAinfo().AppName, @@ -197,7 +226,7 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { AdamState: notInControllerConfig, UUID: im.GetAinfo().AppID, } - ctx.applications[im.GetAinfo().AppID] = appStateObj + ctx.Applications[im.GetAinfo().AppID] = appStateObj } appStateObj.EVEState = im.GetAinfo().State.String() if len(im.GetAinfo().AppErr) > 0 { @@ -227,20 +256,20 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { //check appStateObj not defined in adam if appStateObj.AdamState != inControllerConfig { if im.GetAinfo().AppID == appStateObj.UUID { - appStateObj.deleted = false //if in recent ZInfoTypes_ZiApp, then not deleted + appStateObj.Deleted = false //if in recent ZInfoTypes_ZiApp, then not deleted } } if im.GetAinfo().State == info.ZSwState_INVALID { - appStateObj.deleted = true + appStateObj.Deleted = true } - appStateObj.infoTime = im.AtTimeStamp.AsTime() + appStateObj.InfoTime = im.AtTimeStamp.AsTime() case info.ZInfoTypes_ZiNetworkInstance: //try to find ips from NetworkInstances for _, el := range im.GetNiinfo().IpAssignments { // nothing to show if no IpAddress received if len(el.IpAddress) == 0 { continue } - for _, appStateObj := range ctx.applications { + for _, appStateObj := range ctx.Applications { for ind, mac := range appStateObj.Macs { if mac == el.MacAddress { appStateObj.InternalIP[ind] = el.IpAddress[0] @@ -250,7 +279,7 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { } case info.ZInfoTypes_ZiDevice: for _, el := range im.GetDinfo().AppInstances { - if _, ok := ctx.applications[el.Uuid]; !ok { + if _, ok := ctx.Applications[el.Uuid]; !ok { appStateObj := &AppInstState{ Name: el.Name, Image: "-", @@ -258,10 +287,10 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { EVEState: "UNKNOWN", UUID: el.Uuid, } - ctx.applications[el.Uuid] = appStateObj + ctx.Applications[el.Uuid] = appStateObj } } - for _, appStateObj := range ctx.applications { + for _, appStateObj := range ctx.Applications { seen := false for _, el := range im.GetDinfo().AppInstances { if appStateObj.UUID == el.Uuid { @@ -289,12 +318,12 @@ func (ctx *State) processApplicationsByInfo(im *info.ZInfoMsg) { appStateObj.ExternalIP = "127.0.0.1" } //check appStateObj not defined in adam - if appStateObj.AdamState != inControllerConfig && appStateObj.infoTime.Before(im.AtTimeStamp.AsTime()) { - appStateObj.deleted = true + if appStateObj.AdamState != inControllerConfig && appStateObj.InfoTime.Before(im.AtTimeStamp.AsTime()) { + appStateObj.Deleted = true for _, el := range im.GetDinfo().AppInstances { - //if in recent ZInfoTypes_ZiDevice with timestamp after ZInfoTypes_ZiApp, than not deleted + //if in recent ZInfoTypes_ZiDevice with timestamp after ZInfoTypes_ZiApp, then not deleted if el.Uuid == appStateObj.UUID { - appStateObj.deleted = false + appStateObj.Deleted = false } } } @@ -308,8 +337,8 @@ func (ctx *State) printPodListLines() error { if _, err := fmt.Fprintln(w, appStateHeader()); err != nil { return err } - appStatesSlice := make([]*AppInstState, 0, len(ctx.Applications())) - appStatesSlice = append(appStatesSlice, ctx.Applications()...) + appStatesSlice := make([]*AppInstState, 0, len(ctx.NotDeletedApplications())) + appStatesSlice = append(appStatesSlice, ctx.NotDeletedApplications()...) sort.SliceStable(appStatesSlice, func(i, j int) bool { return appStatesSlice[i].Name < appStatesSlice[j].Name }) @@ -322,7 +351,7 @@ func (ctx *State) printPodListLines() error { } func (ctx *State) printPodListJSON() error { - result, err := json.MarshalIndent(ctx.Applications(), "", " ") + result, err := json.MarshalIndent(ctx.NotDeletedApplications(), "", " ") if err != nil { return err } diff --git a/pkg/eve/eve.go b/pkg/eve/eve.go new file mode 100644 index 000000000..250623f9f --- /dev/null +++ b/pkg/eve/eve.go @@ -0,0 +1,66 @@ +package eve + +import ( + "time" + + "github.com/lf-edge/eden/pkg/controller" + "github.com/lf-edge/eden/pkg/device" + "github.com/lf-edge/eve/api/go/info" + "github.com/lf-edge/eve/api/go/metrics" +) + +// NodeState describes state of edge node +type NodeState struct { + UsedMem uint32 + AvailMem uint32 + UsedPercentageMem float64 + + LastRebootTime time.Time + LastRebootReason string + + // interface to ip mapping + RemoteIPs map[string][]string + + LastSeen time.Time + + Version string +} + +func (ctx *State) initNodeState(_ controller.Cloud, _ *device.Ctx) error { + ctx.EveState = &NodeState{} + return nil +} + +func (ctx *State) applyOldStateNodeState(state *State) { + ctx.EveState = state.EveState +} + +func (ctx *State) processNodeStateByInfo(msg *info.ZInfoMsg) { + infoTime := msg.AtTimeStamp.AsTime() + if infoTime.After(ctx.EveState.LastSeen) { + ctx.EveState.LastSeen = infoTime + } + if deviceInfo := msg.GetDinfo(); deviceInfo != nil { + ctx.EveState.RemoteIPs = make(map[string][]string) + for _, nw := range deviceInfo.Network { + ctx.EveState.RemoteIPs[nw.LocalName] = nw.IPAddrs + } + ctx.EveState.LastRebootTime = deviceInfo.LastRebootTime.AsTime() + ctx.EveState.LastRebootReason = deviceInfo.LastRebootReason + if len(deviceInfo.SwList) > 0 { + ctx.EveState.Version = deviceInfo.SwList[0].ShortVersion + } + } +} + +func (ctx *State) processNodeStateByMetric(msg *metrics.ZMetricMsg) { + metricTime := msg.AtTimeStamp.AsTime() + if metricTime.After(ctx.EveState.LastSeen) { + ctx.EveState.LastSeen = metricTime + } + if deviceMetric := msg.GetDm(); deviceMetric != nil { + ctx.EveState.AvailMem = deviceMetric.Memory.GetAvailMem() + ctx.EveState.UsedMem = deviceMetric.Memory.GetUsedMem() + ctx.EveState.UsedPercentageMem = deviceMetric.Memory.GetUsedPercentage() + } +} diff --git a/pkg/eve/networks.go b/pkg/eve/networks.go index d03904934..b9b46b76a 100644 --- a/pkg/eve/networks.go +++ b/pkg/eve/networks.go @@ -25,7 +25,7 @@ type NetInstState struct { AdamState string EveState string Activated bool - deleted bool + Deleted bool } func netInstStateHeader() string { @@ -40,7 +40,7 @@ func (netInstStateObj *NetInstState) toString() string { } func (ctx *State) initNetworks(ctrl controller.Cloud, dev *device.Ctx) error { - ctx.networks = make(map[string]*NetInstState) + ctx.Networks = make(map[string]*NetInstState) for _, el := range dev.GetNetworkInstances() { ni, err := ctrl.GetNetworkInstanceConfig(el) if err != nil { @@ -55,15 +55,35 @@ func (ctx *State) initNetworks(ctrl controller.Cloud, dev *device.Ctx) error { CIDR: ni.Ip.Subnet, NetworkType: ni.InstType.String(), } - ctx.networks[ni.Uuidandversion.Uuid] = netInstStateObj + ctx.Networks[ni.Uuidandversion.Uuid] = netInstStateObj } return nil } +func (ctx *State) applyOldStateNetworks(state *State) { + for stateID, stateEL := range state.Networks { + found := false + for id := range ctx.Networks { + if id != stateID { + continue + } + ctx.Networks[id] = stateEL + found = true + } + if !found { + if stateEL.Deleted { + continue + } + stateEL.AdamState = notInControllerConfig + ctx.Networks[stateID] = stateEL + } + } +} + func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) { switch im.GetZtype() { case info.ZInfoTypes_ZiNetworkInstance: - netInstStateObj, ok := ctx.networks[im.GetNiinfo().GetNetworkID()] + netInstStateObj, ok := ctx.Networks[im.GetNiinfo().GetNetworkID()] if !ok { netInstStateObj = &NetInstState{ Name: im.GetNiinfo().GetDisplayname(), @@ -73,7 +93,7 @@ func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) { EveState: "UNKNOWN", NetworkType: (config.ZNetworkInstType)(int32(im.GetNiinfo().InstType)).String(), } - ctx.networks[im.GetNiinfo().GetNetworkID()] = netInstStateObj + ctx.Networks[im.GetNiinfo().GetNetworkID()] = netInstStateObj } netInstStateObj.EveState = im.GetNiinfo().State.String() netInstStateObj.Activated = im.GetNiinfo().Activated @@ -98,12 +118,12 @@ func (ctx *State) processNetworksByInfo(im *info.ZInfoMsg) { if !netInstStateObj.Activated && im.GetNiinfo().State != info.ZNetworkInstanceState_ZNETINST_STATE_INIT && netInstStateObj.AdamState == notInControllerConfig { - netInstStateObj.deleted = true + netInstStateObj.Deleted = true } if im.GetNiinfo().State == info.ZNetworkInstanceState_ZNETINST_STATE_UNSPECIFIED && netInstStateObj.AdamState == notInControllerConfig { - netInstStateObj.deleted = true + netInstStateObj.Deleted = true } } } @@ -112,7 +132,7 @@ func (ctx *State) processNetworksByMetric(msg *metrics.ZMetricMsg) { if networkMetrics := msg.GetNm(); networkMetrics != nil { for _, networkMetric := range networkMetrics { // XXX use [uuid] instead of loop - for _, el := range ctx.networks { + for _, el := range ctx.Networks { if networkMetric.NetworkID == el.UUID { el.Stats = networkMetric.GetNetworkStats().String() break @@ -128,8 +148,8 @@ func (ctx *State) printNetListLines() error { if _, err := fmt.Fprintln(w, netInstStateHeader()); err != nil { return err } - netInstStatesSlice := make([]*NetInstState, 0, len(ctx.Networks())) - netInstStatesSlice = append(netInstStatesSlice, ctx.Networks()...) + netInstStatesSlice := make([]*NetInstState, 0, len(ctx.NotDeletedNetworks())) + netInstStatesSlice = append(netInstStatesSlice, ctx.NotDeletedNetworks()...) sort.SliceStable(netInstStatesSlice, func(i, j int) bool { return netInstStatesSlice[i].Name < netInstStatesSlice[j].Name }) @@ -142,7 +162,7 @@ func (ctx *State) printNetListLines() error { } func (ctx *State) printNetListJSON() error { - result, err := json.MarshalIndent(ctx.Networks(), "", " ") + result, err := json.MarshalIndent(ctx.NotDeletedNetworks(), "", " ") if err != nil { return err } diff --git a/pkg/eve/state.go b/pkg/eve/state.go index cbf447951..d00929c97 100644 --- a/pkg/eve/state.go +++ b/pkg/eve/state.go @@ -1,11 +1,17 @@ package eve import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "github.com/lf-edge/eden/pkg/controller" "github.com/lf-edge/eden/pkg/controller/einfo" "github.com/lf-edge/eden/pkg/controller/emetric" "github.com/lf-edge/eden/pkg/device" - "github.com/lf-edge/eden/pkg/projects" + "github.com/lf-edge/eden/pkg/utils" "github.com/lf-edge/eve/api/go/info" "github.com/lf-edge/eve/api/go/metrics" log "github.com/sirupsen/logrus" @@ -14,23 +20,22 @@ import ( const ( inControllerConfig = "IN_CONFIG" notInControllerConfig = "NOT_IN_CONFIG" + stateFileTemplate = "state_store_%s.json" ) -//State stores representation of EVE state -//we should assign InfoCallback and MetricCallback to update state +// State stores representation of EVE state +// we should assign InfoCallback and MetricCallback to update state type State struct { - applications map[string]*AppInstState - networks map[string]*NetInstState - volumes map[string]*VolInstState - infoAndMetrics *projects.State - device *device.Ctx + Applications map[string]*AppInstState + Networks map[string]*NetInstState + Volumes map[string]*VolInstState + EveState *NodeState + device *device.Ctx } -//Init State object with controller and device +// Init State object with controller and device func Init(ctrl controller.Cloud, dev *device.Ctx) (ctx *State) { - ctx = &State{device: dev, infoAndMetrics: projects.InitState(dev)} - ctx.applications = make(map[string]*AppInstState) - ctx.networks = make(map[string]*NetInstState) + ctx = &State{device: dev} if err := ctx.initApplications(ctrl, dev); err != nil { log.Fatalf("EVE State initApplications error: %s", err) } @@ -40,69 +45,122 @@ func Init(ctrl controller.Cloud, dev *device.Ctx) (ctx *State) { if err := ctx.initNetworks(ctrl, dev); err != nil { log.Fatalf("EVE State initNetworks error: %s", err) } + if err := ctx.initNodeState(ctrl, dev); err != nil { + log.Fatalf("EVE State initNodeState error: %s", err) + } + if err := ctx.Load(); err != nil { + log.Fatalf("EVE State Load error: %s", err) + } return } -//InfoAndMetrics returns last info and metric objects -func (ctx *State) InfoAndMetrics() *projects.State { - return ctx.infoAndMetrics +func (ctx *State) getStateFile() (string, error) { + edenDir, err := utils.DefaultEdenDir() + if err != nil { + return "", err + } + return filepath.Join(edenDir, fmt.Sprintf(stateFileTemplate, ctx.device.GetID().String())), nil +} + +// Store state into file +func (ctx *State) Store() error { + data, err := json.Marshal(ctx) + if err != nil { + return err + } + stateFile, err := ctx.getStateFile() + if err != nil { + return err + } + return os.WriteFile(stateFile, data, 0600) +} + +// Load state from file +func (ctx *State) Load() error { + stateFile, err := ctx.getStateFile() + if err != nil { + return err + } + data, err := os.ReadFile(stateFile) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var obj *State + err = json.Unmarshal(data, &obj) + if err != nil { + return err + } + ctx.applyOldStateApps(obj) + ctx.applyOldStateNetworks(obj) + ctx.applyOldStateVolumes(obj) + ctx.applyOldStateNodeState(obj) + return nil +} + +// Prepared returns true if we have enough info to work +func (ctx *State) Prepared() bool { + return ctx.EveState.LastSeen.Unix() != 0 } -//Applications extracts applications states -func (ctx *State) Applications() []*AppInstState { - v := make([]*AppInstState, 0, len(ctx.applications)) - for _, value := range ctx.applications { - if !value.deleted { +// NotDeletedApplications extracts AppInstState which are not marked as deleted +func (ctx *State) NotDeletedApplications() []*AppInstState { + v := make([]*AppInstState, 0, len(ctx.Applications)) + for _, value := range ctx.Applications { + if !value.Deleted { v = append(v, value) } } return v } -//Networks extracts networks states -func (ctx *State) Networks() []*NetInstState { - v := make([]*NetInstState, 0, len(ctx.networks)) - for _, value := range ctx.networks { - if !value.deleted { +// NotDeletedNetworks extracts NetInstState which are not marked as deleted +func (ctx *State) NotDeletedNetworks() []*NetInstState { + v := make([]*NetInstState, 0, len(ctx.Networks)) + for _, value := range ctx.Networks { + if !value.Deleted { v = append(v, value) } } return v } -//Volumes extracts volumes states -func (ctx *State) Volumes() []*VolInstState { - v := make([]*VolInstState, 0, len(ctx.volumes)) - for _, value := range ctx.volumes { - if !value.deleted { +// NotDeletedVolumes extracts VolInstState which are not marked as deleted +func (ctx *State) NotDeletedVolumes() []*VolInstState { + v := make([]*VolInstState, 0, len(ctx.Volumes)) + for _, value := range ctx.Volumes { + if !value.Deleted { v = append(v, value) } } return v } -//InfoCallback should be assigned to feed new values from info messages into state +// NodeState returns NodeState +func (ctx *State) NodeState() *NodeState { + return ctx.EveState +} + +// InfoCallback should be assigned to feed new values from info messages into state func (ctx *State) InfoCallback() einfo.HandlerFunc { return func(msg *info.ZInfoMsg) bool { ctx.processVolumesByInfo(msg) ctx.processApplicationsByInfo(msg) ctx.processNetworksByInfo(msg) - if err := ctx.infoAndMetrics.GetInfoProcessingFunction()(msg); err != nil { - log.Fatalf("EVE State GetInfoProcessingFunction error: %s", err) - } + ctx.processNodeStateByInfo(msg) return false } } -//MetricCallback should be assigned to feed new values from metric messages into state +// MetricCallback should be assigned to feed new values from metric messages into state func (ctx *State) MetricCallback() emetric.HandlerFunc { return func(msg *metrics.ZMetricMsg) bool { ctx.processVolumesByMetric(msg) ctx.processApplicationsByMetric(msg) ctx.processNetworksByMetric(msg) - if err := ctx.infoAndMetrics.GetMetricProcessingFunction()(msg); err != nil { - log.Fatalf("EVE State GetMetricProcessingFunction error: %s", err) - } + ctx.processNodeStateByMetric(msg) return false } } diff --git a/pkg/eve/volumes.go b/pkg/eve/volumes.go index 70deafafd..234f596bc 100644 --- a/pkg/eve/volumes.go +++ b/pkg/eve/volumes.go @@ -29,10 +29,10 @@ type VolInstState struct { EveState string LastError string Ref string - contentTreeID string + ContentTreeID string MountPoint string OriginType string - deleted bool + Deleted bool } func volInstStateHeader() string { @@ -51,7 +51,7 @@ func (volInstStateObj *VolInstState) toString() string { } func (ctx *State) initVolumes(ctrl controller.Cloud, dev *device.Ctx) error { - ctx.volumes = make(map[string]*VolInstState) + ctx.Volumes = make(map[string]*VolInstState) for _, el := range dev.GetVolumes() { vi, err := ctrl.GetVolume(el) if err != nil { @@ -95,19 +95,39 @@ func (ctx *State) initVolumes(ctrl controller.Cloud, dev *device.Ctx) error { MaxSize: "-", MountPoint: strings.Join(mountPoint, ";"), Ref: strings.Join(ref, ";"), - contentTreeID: contentTreeID, + ContentTreeID: contentTreeID, OriginType: vi.GetOrigin().GetType().String(), } - ctx.volumes[vi.GetUuid()] = volInstStateObj + ctx.Volumes[vi.GetUuid()] = volInstStateObj } return nil } +func (ctx *State) applyOldStateVolumes(state *State) { + for stateID, stateEL := range state.Volumes { + found := false + for id := range ctx.Volumes { + if id != stateID { + continue + } + ctx.Volumes[id] = stateEL + found = true + } + if !found { + if stateEL.Deleted { + continue + } + stateEL.AdamState = notInControllerConfig + ctx.Volumes[stateID] = stateEL + } + } +} + func (ctx *State) processVolumesByInfo(im *info.ZInfoMsg) { switch im.GetZtype() { case info.ZInfoTypes_ZiVolume: infoObject := im.GetVinfo() - volInstStateObj, ok := ctx.volumes[infoObject.GetUuid()] + volInstStateObj, ok := ctx.Volumes[infoObject.GetUuid()] if !ok { volInstStateObj = &VolInstState{ Name: infoObject.GetDisplayName(), @@ -119,9 +139,9 @@ func (ctx *State) processVolumesByInfo(im *info.ZInfoMsg) { MountPoint: "-", Ref: "-", } - ctx.volumes[infoObject.GetUuid()] = volInstStateObj + ctx.Volumes[infoObject.GetUuid()] = volInstStateObj } - volInstStateObj.deleted = + volInstStateObj.Deleted = infoObject.DisplayName == "" || infoObject.State == info.ZSwState_INVALID if volInstStateObj.VolumeType != config.Format_FmtUnknown.String() && volInstStateObj.VolumeType != config.Format_CONTAINER.String() { @@ -143,8 +163,8 @@ func (ctx *State) processVolumesByInfo(im *info.ZInfoMsg) { } case info.ZInfoTypes_ZiContentTree: infoObject := im.GetCinfo() - for _, el := range ctx.volumes { - if infoObject.Uuid == el.contentTreeID { + for _, el := range ctx.Volumes { + if infoObject.Uuid == el.ContentTreeID { el.EveState = infoObject.GetState().String() if infoObject.GetErr() != nil { el.LastError = infoObject.GetErr().String() @@ -162,7 +182,7 @@ func (ctx *State) processVolumesByInfo(im *info.ZInfoMsg) { func (ctx *State) processVolumesByMetric(msg *metrics.ZMetricMsg) { if volumeMetrics := msg.GetVm(); volumeMetrics != nil { for _, volumeMetric := range volumeMetrics { - volInstStateObj, ok := ctx.volumes[volumeMetric.GetUuid()] + volInstStateObj, ok := ctx.Volumes[volumeMetric.GetUuid()] if ok { volInstStateObj.Size = humanize.Bytes(volumeMetric.GetUsedBytes()) } @@ -175,8 +195,8 @@ func (ctx *State) printVolumeListLines() error { if _, err := fmt.Fprintln(w, volInstStateHeader()); err != nil { return err } - volInstStatesSlice := make([]*VolInstState, 0, len(ctx.Volumes())) - volInstStatesSlice = append(volInstStatesSlice, ctx.Volumes()...) + volInstStatesSlice := make([]*VolInstState, 0, len(ctx.NotDeletedVolumes())) + volInstStatesSlice = append(volInstStatesSlice, ctx.NotDeletedVolumes()...) sort.SliceStable(volInstStatesSlice, func(i, j int) bool { return volInstStatesSlice[i].Name < volInstStatesSlice[j].Name }) @@ -189,7 +209,7 @@ func (ctx *State) printVolumeListLines() error { } func (ctx *State) printVolumeListJSON() error { - result, err := json.MarshalIndent(ctx.Volumes(), "", " ") + result, err := json.MarshalIndent(ctx.NotDeletedVolumes(), "", " ") if err != nil { return err } diff --git a/pkg/openevec/eve.go b/pkg/openevec/eve.go index e063c3b4e..1962c2a73 100644 --- a/pkg/openevec/eve.go +++ b/pkg/openevec/eve.go @@ -16,7 +16,6 @@ import ( "github.com/lf-edge/eden/pkg/eve" "github.com/lf-edge/eden/pkg/utils" sdnapi "github.com/lf-edge/eden/sdn/vm/api" - "github.com/lf-edge/eve/api/go/info" log "github.com/sirupsen/logrus" ) @@ -221,25 +220,17 @@ func (openEVEC *OpenEVEC) VersionEve() error { changer := &adamChanger{} ctrl, dev, err := changer.getControllerAndDevFromConfig(openEVEC.cfg) if err != nil { - log.Debugf("getControllerAndDevFromConfig: %s", err.Error()) - fmt.Println("EVE status: undefined (no onboarded EVE)") - } else { - var lastDInfo *info.ZInfoMsg - var handleInfo = func(im *info.ZInfoMsg) bool { - if im.GetZtype() == info.ZInfoTypes_ZiDevice { - lastDInfo = im - } - return false - } - if err = ctrl.InfoLastCallback(dev.GetID(), map[string]string{"devId": dev.GetID().String()}, handleInfo); err != nil { - return fmt.Errorf("fail in get InfoLastCallback: %w", err) - } - if lastDInfo == nil { - log.Info("no info messages") - } else { - fmt.Println(lastDInfo.GetDinfo().SwList[0].ShortVersion) - } + return fmt.Errorf("getControllerAndDev: %w", err) + } + state := eve.Init(ctrl, dev) + if err := ctrl.InfoLastCallback(dev.GetID(), nil, state.InfoCallback()); err != nil { + return fmt.Errorf("fail in get InfoLastCallback: %w", err) } + if err := ctrl.MetricLastCallback(dev.GetID(), nil, state.MetricCallback()); err != nil { + return fmt.Errorf("fail in get MetricLastCallback: %w", err) + } + //nolint:forbidigo + fmt.Println(state.NodeState().Version) return nil } @@ -291,12 +282,12 @@ func (openEVEC *OpenEVEC) GetEveIP(ifName string) string { log.Error(err) return "" } - for _, nw := range networks { - if nw.LocalName == ifName { - if len(nw.IPAddrs) == 0 { + for ifNameKey, ips := range networks { + if ifNameKey == ifName { + if len(ips) == 0 { return "" } - return nw.IPAddrs[0] + return ips[0] } } return "" @@ -509,7 +500,7 @@ func (openEVEC *OpenEVEC) NewLinkEve(command, eveInterfaceName, vmName string) e return nil } -func (openEVEC *OpenEVEC) getEveNetworkInfo() (networks []*info.ZInfoNetwork, err error) { +func (openEVEC *OpenEVEC) getEveNetworkInfo() (ips map[string][]string, err error) { changer := &adamChanger{} ctrl, dev, err := changer.getControllerAndDevFromConfig(openEVEC.cfg) if err != nil { @@ -522,8 +513,5 @@ func (openEVEC *OpenEVEC) getEveNetworkInfo() (networks []*info.ZInfoNetwork, er if err = ctrl.MetricLastCallback(dev.GetID(), nil, eveState.MetricCallback()); err != nil { return nil, fmt.Errorf("MetricLastCallback failed: %w", err) } - if lastDInfo := eveState.InfoAndMetrics().GetDinfo(); lastDInfo != nil { - networks = append(networks, lastDInfo.Network...) - } - return networks, nil + return eveState.NodeState().RemoteIPs, nil } diff --git a/pkg/openevec/status.go b/pkg/openevec/status.go index b3b322aca..47a485b07 100644 --- a/pkg/openevec/status.go +++ b/pkg/openevec/status.go @@ -143,34 +143,31 @@ func (openEVEC *OpenEVEC) eveStatusRemote() error { if err = ctrl.MetricLastCallback(dev.GetID(), nil, eveState.MetricCallback()); err != nil { return fmt.Errorf("fail in get InfoLastCallback: %w", err) } - if lastDInfo := eveState.InfoAndMetrics().GetDinfo(); lastDInfo != nil { + if eveState.NodeState().LastSeen.Unix() == 0 { + fmt.Printf("%s EVE REMOTE IPs: %s\n", statusWarn(), "waiting for info...") + fmt.Printf("%s EVE memory: %s\n", statusWarn(), "waiting for info...") + } else { var ips []string - for _, nw := range lastDInfo.Network { - ips = append(ips, nw.IPAddrs...) + for _, v := range eveState.NodeState().RemoteIPs { + ips = append(ips, v...) } fmt.Printf("%s EVE REMOTE IPs: %s\n", statusOK(), strings.Join(ips, "; ")) - var lastseen = time.Unix(eveState.InfoAndMetrics().GetLastInfoTime().GetSeconds(), 0) + var lastseen = eveState.NodeState().LastSeen var timenow = time.Now().Unix() fmt.Printf("\tLast info received time: %s\n", lastseen) if (timenow - lastseen.Unix()) > 600 { fmt.Printf("\t EVE MIGHT BE DOWN OR CONNECTIVITY BETWEEN EVE AND ADAM WAS LOST\n") } - } else { - fmt.Printf("%s EVE REMOTE IPs: %s\n", statusWarn(), "waiting for info...") - } - if lastDMetric := eveState.InfoAndMetrics().GetDeviceMetrics(); lastDMetric != nil { status := statusOK() - if lastDMetric.Memory.GetUsedPercentage() >= 70 { + if eveState.NodeState().UsedPercentageMem >= 70 { status = statusWarn() } - if lastDMetric.Memory.GetUsedPercentage() >= 90 { + if eveState.NodeState().UsedPercentageMem >= 90 { status = statusBad() } fmt.Printf("%s EVE memory: %s/%s\n", status, - humanize.Bytes((uint64)(lastDMetric.Memory.GetUsedMem()*humanize.MByte)), - humanize.Bytes((uint64)(lastDMetric.Memory.GetAvailMem()*humanize.MByte))) - } else { - fmt.Printf("%s EVE memory: %s\n", statusWarn(), "waiting for info...") + humanize.Bytes((uint64)(eveState.NodeState().UsedMem*humanize.MByte)), + humanize.Bytes((uint64)(eveState.NodeState().AvailMem*humanize.MByte))) } return nil } diff --git a/pkg/projects/state.go b/pkg/projects/state.go index 3854bab1d..3f7f19b38 100644 --- a/pkg/projects/state.go +++ b/pkg/projects/state.go @@ -3,9 +3,11 @@ package projects import ( "reflect" + "github.com/lf-edge/eden/pkg/controller" "github.com/lf-edge/eden/pkg/controller/einfo" "github.com/lf-edge/eden/pkg/controller/emetric" "github.com/lf-edge/eden/pkg/device" + "github.com/lf-edge/eden/pkg/eve" "github.com/lf-edge/eden/pkg/utils" "github.com/lf-edge/eve/api/go/info" "github.com/lf-edge/eve/api/go/metrics" @@ -32,17 +34,19 @@ type infoState struct { type State struct { device *device.Ctx deviceInfo *infoState + eveState *eve.State } // InitState init State object for device -func InitState(device *device.Ctx) *State { - return &State{device: device, deviceInfo: &infoState{}} +func InitState(cloud controller.Cloud, device *device.Ctx) *State { + return &State{device: device, deviceInfo: &infoState{}, eveState: eve.Init(cloud, device)} } func (state *State) processInfo(infoMsg *info.ZInfoMsg) error { if infoMsg.DevId != state.device.GetID().String() { return nil } + state.eveState.InfoCallback()(infoMsg) state.deviceInfo.LastInfoMessageTime = infoMsg.AtTimeStamp switch infoMsg.GetZtype() { case info.ZInfoTypes_ZiDevice: @@ -118,6 +122,7 @@ func (state *State) processMetric(metricMsg *metrics.ZMetricMsg) error { if metricMsg.DevID != state.device.GetID().String() { return nil } + state.eveState.MetricCallback()(metricMsg) state.deviceInfo.AppMetrics = metricMsg.GetAm() state.deviceInfo.NetworkInstanceMetrics = metricMsg.GetNm() state.deviceInfo.VolumeMetrics = metricMsg.GetVm() @@ -225,3 +230,8 @@ func (state *State) CheckReady() bool { } return true } + +// GetEVEState returns state of edge node +func (state *State) GetEVEState() *eve.State { + return state.eveState +} diff --git a/pkg/projects/testContext.go b/pkg/projects/testContext.go index 9b9281c20..289f7cc14 100644 --- a/pkg/projects/testContext.go +++ b/pkg/projects/testContext.go @@ -340,7 +340,7 @@ func (tc *TestContext) AddProcTimer(edgeNode *device.Ctx, processFunction ProcTi func (tc *TestContext) StartTrackingState(onlyNewElements bool) { tc.states = map[*device.Ctx]*State{} for _, dev := range tc.nodes { - curState := InitState(dev) + curState := InitState(tc.cloud, dev) tc.states[dev] = curState if !onlyNewElements { //process all events from controller diff --git a/tests/app/app_test.go b/tests/app/app_test.go index 69feb32a1..c0289fc24 100644 --- a/tests/app/app_test.go +++ b/tests/app/app_test.go @@ -9,6 +9,7 @@ import ( "github.com/lf-edge/eden/pkg/controller/eapps" "github.com/lf-edge/eden/pkg/controller/types" + "github.com/lf-edge/eden/pkg/device" "github.com/lf-edge/eden/pkg/eve" "github.com/lf-edge/eden/pkg/projects" "github.com/lf-edge/eden/pkg/tests" @@ -25,10 +26,9 @@ type appState struct { // This test wait for the app's state with a timewait. var ( timewait = flag.Duration("timewait", 10*time.Minute, "Timewait for items waiting") - newitems = flag.Bool("check-new", false, "Check only new info messages") + _ = flag.Bool("check-new", false, "Check only new info messages") tc *projects.TestContext states map[string][]appState - eveState *eve.State ) // TestMain is used to provide setup and teardown for the rest of the @@ -48,9 +48,7 @@ func TestMain(m *testing.M) { tc.AddEdgeNodesFromDescription() - eveState = eve.Init(tc.GetController(), tc.GetEdgeNode()) - - tc.StartTrackingState(true) + tc.StartTrackingState(false) res := m.Run() @@ -83,11 +81,11 @@ func checkState(eveState *eve.State, state string, appNames []string) error { out := "\n" if state == "-" { foundAny := false - if eveState.InfoAndMetrics().GetDinfo() == nil { + if !eveState.Prepared() { //we need to wait for info return nil } - for _, app := range eveState.Applications() { + for _, app := range eveState.NotDeletedApplications() { if _, inSlice := utils.FindEleInSlice(appNames, app.Name); inSlice { checkAndAppendState(app.Name, app.EVEState) foundAny = true @@ -103,7 +101,7 @@ func checkState(eveState *eve.State, state string, appNames []string) error { } return fmt.Errorf(out) } - for _, app := range eveState.Applications() { + for _, app := range eveState.NotDeletedApplications() { if _, inSlice := utils.FindEleInSlice(appNames, app.Name); inSlice { checkAndAppendState(app.Name, app.EVEState) } @@ -124,10 +122,9 @@ func checkState(eveState *eve.State, state string, appNames []string) error { } // checkApp wait for info of ZInfoApp type with state -func checkApp(state string, appNames []string) projects.ProcInfoFunc { +func checkApp(edgeNode *device.Ctx, state string, appNames []string) projects.ProcInfoFunc { return func(msg *info.ZInfoMsg) error { - eveState.InfoCallback()(msg) //feed state with new info - return checkState(eveState, state, appNames) + return checkState(tc.GetState(edgeNode).GetEVEState(), state, appNames) } } @@ -156,16 +153,9 @@ func TestAppStatus(t *testing.T) { timestamp: time.Now()}} } - if !*newitems { - // observe existing info object and feed them into eveState object - if err := tc.GetController().InfoLastCallback(edgeNode.GetID(), nil, eveState.InfoCallback()); err != nil { - t.Fatal(err) - } - } - - if ready := checkState(eveState, state, apps); ready == nil { + if ready := checkState(tc.GetState(edgeNode).GetEVEState(), state, apps); ready == nil { - tc.AddProcInfo(edgeNode, checkApp(state, apps)) + tc.AddProcInfo(edgeNode, checkApp(edgeNode, state, apps)) callback := func() { t.Errorf("ASSERTION FAILED (%s): expected apps %s in %s state", time.Now().Format(time.RFC3339Nano), apps, state) @@ -177,7 +167,7 @@ func TestAppStatus(t *testing.T) { t.Errorf("\t\tstate: %s received in: %s", st.state, st.timestamp.Format(time.RFC3339Nano)) } } - for _, app := range eveState.Applications() { + for _, app := range tc.GetState(edgeNode).GetEVEState().NotDeletedApplications() { if app.Name == k { appID, err := uuid.FromString(app.UUID) if err != nil { diff --git a/tests/network/nw_test.go b/tests/network/nw_test.go index 47a8a36a7..92dcb6876 100644 --- a/tests/network/nw_test.go +++ b/tests/network/nw_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/lf-edge/eden/pkg/device" "github.com/lf-edge/eden/pkg/eve" "github.com/lf-edge/eden/pkg/projects" "github.com/lf-edge/eden/pkg/utils" @@ -21,10 +22,9 @@ type nwState struct { // This test wait for the network's state with a timewait. var ( timewait = flag.Duration("timewait", time.Minute, "Timewait for items waiting") - newitems = flag.Bool("check-new", false, "Check only new info messages") + _ = flag.Bool("check-new", false, "Check only new info messages") tc *projects.TestContext states map[string][]nwState - eveState *eve.State ) // TestMain is used to provide setup and teardown for the rest of the @@ -42,9 +42,7 @@ func TestMain(m *testing.M) { tc.AddEdgeNodesFromDescription() - eveState = eve.Init(tc.GetController(), tc.GetEdgeNode()) - - tc.StartTrackingState(true) + tc.StartTrackingState(false) res := m.Run() @@ -74,7 +72,7 @@ func checkState(eveState *eve.State, state string, netNames []string) error { out := "\n" if state == "-" { foundAny := false - for _, net := range eveState.Networks() { + for _, net := range eveState.NotDeletedNetworks() { if _, inSlice := utils.FindEleInSlice(netNames, net.Name); inSlice { checkAndAppendState(net.Name, net.EveState) foundAny = true @@ -90,7 +88,7 @@ func checkState(eveState *eve.State, state string, netNames []string) error { } return fmt.Errorf(out) } - for _, net := range eveState.Networks() { + for _, net := range eveState.NotDeletedNetworks() { if _, inSlice := utils.FindEleInSlice(netNames, net.Name); inSlice { checkAndAppendState(net.Name, net.EveState) } @@ -111,10 +109,9 @@ func checkState(eveState *eve.State, state string, netNames []string) error { } // checkNet wait for info of ZInfoApp type with state -func checkNet(state string, volNames []string) projects.ProcInfoFunc { - return func(msg *info.ZInfoMsg) error { - eveState.InfoCallback()(msg) //feed state with new info - return checkState(eveState, state, volNames) +func checkNet(edgeNode *device.Ctx, state string, volNames []string) projects.ProcInfoFunc { + return func(_ *info.ZInfoMsg) error { + return checkState(tc.GetState(edgeNode).GetEVEState(), state, volNames) } } @@ -141,17 +138,10 @@ func TestNetworkStatus(t *testing.T) { states[el] = []nwState{{state: "no info from controller", timestamp: time.Now()}} } - if !*newitems { - // observe existing info object and feed them into eveState object - if err := tc.GetController().InfoLastCallback(edgeNode.GetID(), nil, eveState.InfoCallback()); err != nil { - t.Fatal(err) - } - } - // we are done if our eveState object is in required state - if ready := checkState(eveState, state, nws); ready == nil { + if ready := checkState(tc.GetState(edgeNode).GetEVEState(), state, nws); ready == nil { - tc.AddProcInfo(edgeNode, checkNet(state, nws)) + tc.AddProcInfo(edgeNode, checkNet(edgeNode, state, nws)) callback := func() { t.Errorf("ASSERTION FAILED (%s): expected networks %s in %s state", time.Now().Format(time.RFC3339Nano), nws, state) diff --git a/tests/reboot/reboot_test.go b/tests/reboot/reboot_test.go index 419bcdd6d..9b9f1cd9b 100644 --- a/tests/reboot/reboot_test.go +++ b/tests/reboot/reboot_test.go @@ -13,8 +13,6 @@ import ( "github.com/lf-edge/eden/pkg/tests" "github.com/lf-edge/eden/pkg/utils" "github.com/lf-edge/eve/api/go/info" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" ) // This context holds all the configuration items in the same @@ -40,25 +38,23 @@ var ( tc *projects.TestContext - lastRebootTime *timestamppb.Timestamp + lastRebootTime time.Time ) func checkReboot(t *testing.T, edgeNode *device.Ctx) projects.ProcInfoFunc { - return func(im *info.ZInfoMsg) error { - if im.GetZtype() != info.ZInfoTypes_ZiDevice { - return nil - } - currentLastRebootTime := im.GetDinfo().LastRebootTime - if !proto.Equal(lastRebootTime, currentLastRebootTime) { - if im.GetDinfo().LastRebootReason == "" && - currentLastRebootTime.AsTime().Unix() == 0 { + return func(_ *info.ZInfoMsg) error { + eveState := tc.GetState(edgeNode).GetEVEState() + currentLastRebootTime := eveState.NodeState().LastRebootTime + if !lastRebootTime.Equal(currentLastRebootTime) { + if eveState.NodeState().LastRebootReason == "" && + currentLastRebootTime.Unix() == 0 { // device may not fill the info return nil } lastRebootTime = currentLastRebootTime - fmt.Printf("rebooted with reason %s at %s/n", im.GetDinfo().LastRebootReason, lastRebootTime.AsTime()) - if !strings.Contains(im.GetDinfo().LastRebootReason, "NORMAL") { - err := fmt.Errorf("abnormal reboot: %s", im.GetDinfo().LastRebootReason) + fmt.Printf("rebooted with reason %s at %s/n", eveState.NodeState().LastRebootReason, lastRebootTime) + if !strings.Contains(eveState.NodeState().LastRebootReason, "NORMAL") { + err := fmt.Errorf("abnormal reboot: %s", eveState.NodeState().LastRebootReason) if *reboot { //if we use this test to do reboot, abnormal one must errored the test t.Fatal(err) @@ -167,11 +163,13 @@ func TestReboot(t *testing.T) { t.Log(utils.AddTimestamp(fmt.Sprintf("reboot: %t", *reboot))) t.Log(utils.AddTimestamp(fmt.Sprintf("count: %d", *count))) - lastRebootTime = tc.GetState(edgeNode).GetDinfo().LastRebootTime + nodeState := tc.GetState(edgeNode).GetEVEState().NodeState() + + t.Log(utils.AddTimestamp(fmt.Sprintf("LastRebootTime: %s", nodeState.LastRebootTime))) - t.Log(utils.AddTimestamp(fmt.Sprintf("LastRebootTime: %s", lastRebootTime.AsTime()))) + t.Log(utils.AddTimestamp(fmt.Sprintf("LastRebootReason: %s", nodeState.LastRebootReason))) - t.Log(utils.AddTimestamp(fmt.Sprintf("LastRebootReason: %s", tc.GetState(edgeNode).GetDinfo().LastRebootReason))) + lastRebootTime = nodeState.LastRebootTime tc.AddProcInfo(edgeNode, checkReboot(t, edgeNode)) diff --git a/tests/volume/vol_test.go b/tests/volume/vol_test.go index 34dfa3c4f..4a80525ce 100644 --- a/tests/volume/vol_test.go +++ b/tests/volume/vol_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/lf-edge/eden/pkg/device" "github.com/lf-edge/eden/pkg/eve" "github.com/lf-edge/eden/pkg/projects" "github.com/lf-edge/eden/pkg/utils" @@ -21,10 +22,9 @@ type volState struct { // This test wait for the volume's state with a timewait. var ( timewait = flag.Duration("timewait", time.Minute, "Timewait for items waiting") - newitems = flag.Bool("check-new", false, "Check only new info messages") + _ = flag.Bool("check-new", false, "Check only new info messages") tc *projects.TestContext states map[string][]volState - eveState *eve.State ) // TestMain is used to provide setup and teardown for the rest of the @@ -42,8 +42,6 @@ func TestMain(m *testing.M) { tc.AddEdgeNodesFromDescription() - eveState = eve.Init(tc.GetController(), tc.GetEdgeNode()) - tc.StartTrackingState(false) res := m.Run() @@ -74,11 +72,11 @@ func checkState(eveState *eve.State, state string, volNames []string) error { out := "\n" if state == "-" { foundAny := false - if eveState.InfoAndMetrics().GetDinfo() == nil { + if !eveState.Prepared() { //we need to wait for info return nil } - for _, vol := range eveState.Volumes() { + for _, vol := range eveState.NotDeletedVolumes() { if _, inSlice := utils.FindEleInSlice(volNames, vol.Name); inSlice { checkAndAppendState(vol.Name, vol.EveState) foundAny = true @@ -94,7 +92,7 @@ func checkState(eveState *eve.State, state string, volNames []string) error { } return fmt.Errorf(out) } - for _, vol := range eveState.Volumes() { + for _, vol := range eveState.NotDeletedVolumes() { if _, inSlice := utils.FindEleInSlice(volNames, vol.Name); inSlice { checkAndAppendState(vol.Name, vol.EveState) } @@ -115,10 +113,9 @@ func checkState(eveState *eve.State, state string, volNames []string) error { } // checkVol wait for info of ZInfoApp type with state -func checkVol(state string, volNames []string) projects.ProcInfoFunc { - return func(msg *info.ZInfoMsg) error { - eveState.InfoCallback()(msg) //feed state with new info - return checkState(eveState, state, volNames) +func checkVol(edgeNode *device.Ctx, state string, volNames []string) projects.ProcInfoFunc { + return func(_ *info.ZInfoMsg) error { + return checkState(tc.GetState(edgeNode).GetEVEState(), state, volNames) } } @@ -148,17 +145,10 @@ func TestVolStatus(t *testing.T) { states[el] = []volState{{state: "no info from controller", timestamp: time.Now()}} } - if !*newitems { - // observe existing info object and feed them into eveState object - if err := tc.GetController().InfoLastCallback(edgeNode.GetID(), nil, eveState.InfoCallback()); err != nil { - t.Fatal(err) - } - } - // we are done if our eveState object is in required state - if ready := checkState(eveState, state, vols); ready == nil { + if ready := checkState(tc.GetState(edgeNode).GetEVEState(), state, vols); ready == nil { - tc.AddProcInfo(edgeNode, checkVol(state, vols)) + tc.AddProcInfo(edgeNode, checkVol(edgeNode, state, vols)) callback := func() { t.Errorf("ASSERTION FAILED (%s): expected volumes %s in %s state", time.Now().Format(time.RFC3339Nano), vols, state)