Skip to content

Commit

Permalink
fix(components): do not panic when there's no data collected yet (#130)
Browse files Browse the repository at this point in the history 
Signed-off-by: Gyuho Lee <[email protected]>
  • Loading branch information
@gyuho
gyuho authored Oct 21, 2024
1 parent 7b629d6 commit 871533e
Show file tree
Hide file tree
Showing 30 changed files with 300 additions and 80 deletions.
15 changes: 11 additions & 4 deletions components/accelerator/nvidia/bad-envs/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", bad_envs_id.Name)
return nil, nil
return []components.State{
{
Name: bad_envs_id.Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down Expand Up @@ -97,9 +104,9 @@ func (c *component) Output() (any, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", bad_envs_id.Name)
return nil, nil
return nil, query.ErrNoData
}
if last.Error != nil {
return nil, last.Error
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/clock-speed/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/clock/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/ecc/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/error/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/fabric-manager/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/infiniband/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
15 changes: 11 additions & 4 deletions components/accelerator/nvidia/info/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down Expand Up @@ -113,9 +120,9 @@ func (c *component) Output() (any, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return nil, query.ErrNoData
}
if last.Error != nil {
return nil, last.Error
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/memory/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/nvlink/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/peermem/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", nvidia_peermem_id.Name)
return nil, nil
return []components.State{
{
Name: nvidia_peermem_id.Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/persistence-mode/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", nvidia_persistence_mode_id.Name)
return nil, nil
return []components.State{
{
Name: nvidia_persistence_mode_id.Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/power/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/processes/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/remapped-rows/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/temperature/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/accelerator/nvidia/utilization/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
11 changes: 9 additions & 2 deletions components/containerd/pod/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
// this is the error from "ListSandboxStatus"
Expand Down
11 changes: 9 additions & 2 deletions components/cpu/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,16 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
if err != nil {
return nil, err
}
if last == nil { // no data
if last == nil && err != nil && err != query.ErrNoData { // no data
log.Logger.Debugw("nothing found in last state (no data collected yet)", "component", Name)
return nil, nil
return []components.State{
{
Name: Name,
Healthy: false,
Error: query.ErrNoData.Error(),
Reason: query.ErrNoData.Error(),
},
}, nil
}
if last.Error != nil {
return []components.State{
Expand Down
Loading

0 comments on commit 871533e

Please sign in to comment.