Skip to content

Commit

Permalink
Make the internal blockers implementation internal
Browse files Browse the repository at this point in the history
This at the same time, removes the alert public package.
Alert was only used inside prometheus blocker, so it allows
to simplify the code.

Signed-off-by: Jean-Philippe Evrard <[email protected]>
  • Loading branch information
evrardjp committed Oct 19, 2024
1 parent bd23d59 commit ab24e06
Show file tree
Hide file tree
Showing 7 changed files with 194 additions and 189 deletions.
30 changes: 11 additions & 19 deletions cmd/kured/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/kubereboot/kured/internal"
"github.com/kubereboot/kured/pkg/blockers"
"github.com/kubereboot/kured/pkg/checkers"
"github.com/prometheus/client_golang/prometheus/promhttp"
"math/rand"
"net/http"
"net/url"
Expand All @@ -19,8 +15,17 @@ import (
"time"

"github.com/containrrr/shoutrrr"
"github.com/kubereboot/kured/internal"
"github.com/kubereboot/kured/pkg/blockers"
"github.com/kubereboot/kured/pkg/checkers"
"github.com/kubereboot/kured/pkg/daemonsetlock"
"github.com/kubereboot/kured/pkg/delaytick"
"github.com/kubereboot/kured/pkg/reboot"
"github.com/kubereboot/kured/pkg/taints"
"github.com/kubereboot/kured/pkg/timewindow"
papi "github.com/prometheus/client_golang/api"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
Expand All @@ -29,13 +34,6 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
kubectldrain "k8s.io/kubectl/pkg/drain"

"github.com/kubereboot/kured/pkg/alerts"
"github.com/kubereboot/kured/pkg/daemonsetlock"
"github.com/kubereboot/kured/pkg/delaytick"
"github.com/kubereboot/kured/pkg/reboot"
"github.com/kubereboot/kured/pkg/taints"
"github.com/kubereboot/kured/pkg/timewindow"
)

var (
Expand Down Expand Up @@ -609,12 +607,6 @@ func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers.
preferNoScheduleTaint.Disable()
}

// instantiate prometheus client
promClient, err := alerts.NewPromClient(papi.Config{Address: prometheusURL})
if err != nil {
log.Fatal("Unable to create prometheus client: ", err)
}

source = rand.NewSource(time.Now().UnixNano())
tick = delaytick.New(source, period)
for range tick {
Expand Down Expand Up @@ -655,10 +647,10 @@ func rebootAsRequired(nodeID string, rebooter reboot.Rebooter, checker checkers.

var blockCheckers []blockers.RebootBlocker
if prometheusURL != "" {
blockCheckers = append(blockCheckers, blockers.PrometheusBlockingChecker{PromClient: promClient, Filter: alertFilter.Regexp, FiringOnly: alertFiringOnly, FilterMatchOnly: alertFilterMatchOnly})
blockCheckers = append(blockCheckers, blockers.NewPrometheusBlockingChecker(papi.Config{Address: prometheusURL}, alertFilter.Regexp, alertFiringOnly, alertFilterMatchOnly))
}
if podSelectors != nil {
blockCheckers = append(blockCheckers, blockers.KubernetesBlockingChecker{Client: client, Nodename: nodeID, Filter: podSelectors})
blockCheckers = append(blockCheckers, blockers.NewKubernetesBlockingChecker(client, nodeID, podSelectors))
}

var rebootRequiredBlockCondition string
Expand Down
77 changes: 0 additions & 77 deletions pkg/alerts/prometheus.go

This file was deleted.

84 changes: 0 additions & 84 deletions pkg/blockers/blockers.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,5 @@
package blockers

import (
"context"
"fmt"
"github.com/kubereboot/kured/pkg/alerts"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"regexp"
)

// RebootBlocked checks that a single block Checker
// will block the reboot or not.
func RebootBlocked(blockers ...RebootBlocker) bool {
Expand All @@ -26,77 +16,3 @@ func RebootBlocked(blockers ...RebootBlocker) bool {
type RebootBlocker interface {
IsBlocked() bool
}

// PrometheusBlockingChecker contains info for connecting
// to prometheus, and can give info about whether a reboot should be blocked
type PrometheusBlockingChecker struct {
// prometheusClient to make prometheus-go-client and api config available
// into the PrometheusBlockingChecker struct
PromClient *alerts.PromClient
// regexp used to get alerts
Filter *regexp.Regexp
// bool to indicate if only firing alerts should be considered
FiringOnly bool
// bool to indicate that we're only blocking on alerts which match the filter
FilterMatchOnly bool
}

// KubernetesBlockingChecker contains info for connecting
// to k8s, and can give info about whether a reboot should be blocked
type KubernetesBlockingChecker struct {
// client used to contact kubernetes API
Client *kubernetes.Clientset
Nodename string
// lised used to filter pods (podSelector)
Filter []string
}

// IsBlocked for the prometheus will check if there are active alerts matching
// the arguments given into promclient which would actively block the reboot.
// As of today, no blocker information is shared as a return of the method,
// and the information is simply logged.
func (pb PrometheusBlockingChecker) IsBlocked() bool {
alertNames, err := pb.PromClient.ActiveAlerts(pb.Filter, pb.FiringOnly, pb.FilterMatchOnly)
if err != nil {
log.Warnf("Reboot blocked: prometheus query error: %v", err)
return true
}
count := len(alertNames)
if count > 10 {
alertNames = append(alertNames[:10], "...")
}
if count > 0 {
log.Warnf("Reboot blocked: %d active alerts: %v", count, alertNames)
return true
}
return false
}

// IsBlocked for the KubernetesBlockingChecker will check if a pod, for the node, is preventing
// the reboot. It will warn in the logs about blocking, but does not return an error.
func (kb KubernetesBlockingChecker) IsBlocked() bool {
fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.Nodename)
for _, labelSelector := range kb.Filter {
podList, err := kb.Client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
LabelSelector: labelSelector,
FieldSelector: fieldSelector,
Limit: 10})
if err != nil {
log.Warnf("Reboot blocked: pod query error: %v", err)
return true
}

if len(podList.Items) > 0 {
podNames := make([]string, 0, len(podList.Items))
for _, pod := range podList.Items {
podNames = append(podNames, pod.Name)
}
if len(podList.Continue) > 0 {
podNames = append(podNames, "...")
}
log.Warnf("Reboot blocked: matching pods: %v", podNames)
return true
}
}
return false
}
4 changes: 1 addition & 3 deletions pkg/blockers/blockers_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package blockers

import (
"github.com/kubereboot/kured/pkg/alerts"
papi "github.com/prometheus/client_golang/api"
"testing"
)
Expand All @@ -20,8 +19,7 @@ func Test_rebootBlocked(t *testing.T) {
blockingChecker := BlockingChecker{blocking: true}

// Instantiate a prometheusClient with a broken_url
promClient, _ := alerts.NewPromClient(papi.Config{Address: "broken_url"})
brokenPrometheusClient := PrometheusBlockingChecker{PromClient: promClient, Filter: nil, FiringOnly: false}
brokenPrometheusClient := NewPrometheusBlockingChecker(papi.Config{Address: "broken_url"}, nil, false, false)

type args struct {
blockers []RebootBlocker
Expand Down
61 changes: 61 additions & 0 deletions pkg/blockers/kubernetespod.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package blockers

import (
"context"
"fmt"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

// Compile-time checks to ensure the type implements the interface
var (
_ RebootBlocker = (*KubernetesBlockingChecker)(nil)
)

// KubernetesBlockingChecker contains info for connecting
// to k8s, and can give info about whether a reboot should be blocked
type KubernetesBlockingChecker struct {
// client used to contact kubernetes API
client *kubernetes.Clientset
nodeName string
// lised used to filter pods (podSelector)
filter []string
}

func NewKubernetesBlockingChecker(client *kubernetes.Clientset, nodename string, podSelectors []string) *KubernetesBlockingChecker {
return &KubernetesBlockingChecker{
client: client,
nodeName: nodename,
filter: podSelectors,
}
}

// IsBlocked for the KubernetesBlockingChecker will check if a pod, for the node, is preventing
// the reboot. It will warn in the logs about blocking, but does not return an error.
func (kb KubernetesBlockingChecker) IsBlocked() bool {
fieldSelector := fmt.Sprintf("spec.nodeName=%s,status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown", kb.nodeName)
for _, labelSelector := range kb.filter {
podList, err := kb.client.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
LabelSelector: labelSelector,
FieldSelector: fieldSelector,
Limit: 10})
if err != nil {
log.Warnf("Reboot blocked: pod query error: %v", err)
return true
}

if len(podList.Items) > 0 {
podNames := make([]string, 0, len(podList.Items))
for _, pod := range podList.Items {
podNames = append(podNames, pod.Name)
}
if len(podList.Continue) > 0 {
podNames = append(podNames, "...")
}
log.Warnf("Reboot blocked: matching pods: %v", podNames)
return true
}
}
return false
}
Loading

0 comments on commit ab24e06

Please sign in to comment.