fix(components): scan panic error, docker-container error handling by…

… converting state.error to string Signed-off-by: Gyuho Lee <[email protected]>
leptonai · Aug 19, 2024 · 15e303d · 15e303d
1 parent 27f861b
commit 15e303d
Show file tree

Hide file tree

Showing 31 changed files with 55 additions and 62 deletions.
diff --git a/cmd/gpud/command/up.go b/cmd/gpud/command/up.go
@@ -20,7 +20,7 @@ func cmdUp(cliContext *cli.Context) (retErr error) {
 				retErr = err
 			}
 		} else {
-			fmt.Printf("visit https://localhost:15132 to view the dashboard")
+			fmt.Printf("\nvisit https://localhost:15132 to view the dashboard\n\n")
 		}
 	}()
 

diff --git a/components/accelerator/nvidia/clock-speed/component.go b/components/accelerator/nvidia/clock-speed/component.go
@@ -4,7 +4,6 @@ package clockspeed
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/clock/component.go b/components/accelerator/nvidia/clock/component.go
@@ -4,7 +4,6 @@ package clock
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/clock/component_output.go b/components/accelerator/nvidia/clock/component_output.go
@@ -115,7 +115,6 @@ func (o *Output) States() ([]components.State, error) {
 			{
 				Name:    StateNameHWSlowdown,
 				Healthy: true,
-				Error:   nil,
 				Reason:  rm,
 				ExtraInfo: map[string]string{
 					StateKeyHWSlowdownData:     string(b),

diff --git a/components/accelerator/nvidia/ecc/component.go b/components/accelerator/nvidia/ecc/component.go
@@ -4,7 +4,6 @@ package ecc
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/error/component.go b/components/accelerator/nvidia/error/component.go
@@ -3,7 +3,6 @@ package error
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -51,7 +50,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -75,7 +74,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/fabric-manager/component.go b/components/accelerator/nvidia/fabric-manager/component.go
@@ -4,7 +4,6 @@ package fabricmanager
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -68,7 +67,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -101,7 +100,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "fabric manager query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeyFabricManagerExists: fmt.Sprintf("%v", allOutput.FabricManagerExists),

diff --git a/components/accelerator/nvidia/infiniband/component.go b/components/accelerator/nvidia/infiniband/component.go
@@ -52,7 +52,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil

diff --git a/components/accelerator/nvidia/info/component.go b/components/accelerator/nvidia/info/component.go
@@ -3,7 +3,6 @@ package info
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -51,7 +50,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -75,7 +74,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/memory/component.go b/components/accelerator/nvidia/memory/component.go
@@ -4,7 +4,6 @@ package memory
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/nvlink/component.go b/components/accelerator/nvidia/nvlink/component.go
@@ -55,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil

diff --git a/components/accelerator/nvidia/peermem/component.go b/components/accelerator/nvidia/peermem/component.go
@@ -4,7 +4,6 @@ package peermem
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -52,7 +51,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -76,7 +75,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "lsmod peermem query failed with " + e,
 			})
 		}

diff --git a/components/accelerator/nvidia/power/component.go b/components/accelerator/nvidia/power/component.go
@@ -4,7 +4,6 @@ package power
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/processes/component.go b/components/accelerator/nvidia/processes/component.go
@@ -4,7 +4,6 @@ package processes
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/query/query.go b/components/accelerator/nvidia/query/query.go
@@ -326,13 +326,15 @@ func (o *Output) PrintInfo(debug bool) {
 		fmt.Printf("%s successfully checked fabric manager\n", checkMark)
 	}
 
-	if len(o.Ibstat.Errors) > 0 {
-		fmt.Printf("%s ibstat check failed with %d error(s)\n", warningSign, len(o.Ibstat.Errors))
-		for _, err := range o.Ibstat.Errors {
-			fmt.Println(err)
+	if o.IbstatExists {
+		if o.Ibstat != nil && len(o.Ibstat.Errors) > 0 {
+			fmt.Printf("%s ibstat check failed with %d error(s)\n", warningSign, len(o.Ibstat.Errors))
+			for _, err := range o.Ibstat.Errors {
+				fmt.Println(err)
+			}
+		} else {
+			fmt.Printf("%s successfully checked ibstat\n", checkMark)
 		}
-	} else {
-		fmt.Printf("%s successfully checked ibstat\n", checkMark)
 	}
 
 	if len(o.LsmodPeermemErrors) > 0 {

diff --git a/components/accelerator/nvidia/temperature/component.go b/components/accelerator/nvidia/temperature/component.go
@@ -4,7 +4,6 @@ package temperature
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/accelerator/nvidia/utilization/component.go b/components/accelerator/nvidia/utilization/component.go
@@ -4,7 +4,6 @@ package utilization
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,7 +55,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 		return []components.State{
 			{
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil
@@ -80,7 +79,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			cs = append(cs, components.State{
 				Name:    Name,
 				Healthy: false,
-				Error:   errors.New(e),
+				Error:   e,
 				Reason:  "nvidia-smi query failed with " + e,
 				ExtraInfo: map[string]string{
 					nvidia_query.StateKeySMIExists: fmt.Sprintf("%v", allOutput.SMIExists),

diff --git a/components/components.go b/components/components.go
@@ -68,7 +68,7 @@ type State struct {
 	Name      string            `json:"name,omitempty"`
 	Healthy   bool              `json:"healthy,omitempty"`
 	Reason    string            `json:"reason,omitempty"`     // a detailed and processed reason on why the component is not healthy
-	Error     error             `json:"error,omitempty"`      // the unprocessed error returned from the component
+	Error     string            `json:"error,omitempty"`      // the unprocessed error returned from the component
 	ExtraInfo map[string]string `json:"extra_info,omitempty"` // any extra information the component may want to expose
 }
 

diff --git a/components/containerd/pod/component.go b/components/containerd/pod/component.go
@@ -51,7 +51,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			{
 				Name:    Name,
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil

diff --git a/components/cpu/component.go b/components/cpu/component.go
@@ -56,7 +56,7 @@ func (c *component) States(ctx context.Context) ([]components.State, error) {
 			{
 				Name:    Name,
 				Healthy: false,
-				Error:   last.Error,
+				Error:   last.Error.Error(),
 				Reason:  "last query failed",
 			},
 		}, nil