Skip to content
This repository has been archived by the owner on Apr 8, 2022. It is now read-only.

Enhance Config (.kubewatch.yaml) for granular alerting #176

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,8 @@ handler:
url: ""
webhook:
url: ""

// Resource Config Section for generic alerting
codenio marked this conversation as resolved.
Show resolved Hide resolved
resource:
deployment: false
replicationcontroller: false
Expand All @@ -333,6 +335,19 @@ resource:
secret: false
configmap: false
ingress: false

// Events Config Section for granular alerting
event:
global:
- pod
- deployment
create:
- service
update:

delete:
- job
- service
namespace: ""

```
Expand Down Expand Up @@ -418,6 +433,31 @@ $ kubewatch resource add --rc --po --svc
$ kubewatch resource remove --rc --po --svc
```

## Events

Event config section in `.kubewatch.yaml` file can be used for granular alerting.

```
handler:
slack:
token: xoxb-xxxxx-yyyyyyy
channel: kube-watch-test

event:
global: // global alerts for all events
codenio marked this conversation as resolved.
Show resolved Hide resolved
- pod
- deployment
create: // create alerts for resource object creation
- service
update: // update alerts for resource object updation
-
delete: // delete alerts for resource object deletion
- job
- service

namespace: ""
```

# Build

### Using go
Expand Down
4 changes: 2 additions & 2 deletions cmd/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func configureResource(operation string, cmd *cobra.Command, conf *config.Config
}{
{
"svc",
&conf.Resource.Services,
&conf.Resource.Service,
},
{
"deploy",
Expand Down Expand Up @@ -173,7 +173,7 @@ func init() {
resourceConfigRemoveCmd,
)
// Add resource object flags as PersistentFlags to resourceConfigCmd
resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for services")
resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for Service")
codenio marked this conversation as resolved.
Show resolved Hide resolved
resourceConfigCmd.PersistentFlags().Bool("deploy", false, "watch for deployments")
resourceConfigCmd.PersistentFlags().Bool("po", false, "watch for pods")
resourceConfigCmd.PersistentFlags().Bool("rc", false, "watch for replication controllers")
Expand Down
1 change: 1 addition & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ supported webhooks:
logrus.Fatal(err)
}
config.CheckMissingResourceEnvvars()
config.UnmarshallConfig()
c.Run(config)
},
}
Expand Down
130 changes: 122 additions & 8 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"path/filepath"
"runtime"

"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)

Expand Down Expand Up @@ -52,7 +53,7 @@ type Resource struct {
ReplicationController bool `json:"rc"`
ReplicaSet bool `json:"rs"`
DaemonSet bool `json:"ds"`
Services bool `json:"svc"`
Service bool `json:"svc"`
Pod bool `json:"po"`
Job bool `json:"job"`
Node bool `json:"node"`
Expand All @@ -65,18 +66,24 @@ type Resource struct {
Ingress bool `json:"ing"`
}

// Event struct for granular config
type Event struct {
Global []string `json:"string,omitempty"`
Create []string `json:"create,omitempty"`
Update []string `json:"update,omitempty"`
Delete []string `json:"delete,omitempty"`
}

// Config struct contains kubewatch configuration
type Config struct {
// Handlers know how to send notifications to specific services.
Handler Handler `json:"handler"`

//Reason []string `json:"reason"`

// Resources to watch.
Resource Resource `json:"resource"`

// For watching specific namespace, leave it empty for watching all.
Resource Resource `json:"resource,omitempty"`
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that in your rebase+conflict resolutions you reverted the comments.

in #233 I added a feature that generates a "sample" configuration file annotated with doc comments:

$ kubewatch config sample

For this to work, the Config Go type (and all structs used for its value) must have meaningful comments, following the godoc rules, namely if a comment directly precedes a field declaration, it's considered a field documentation comment.

this is why the commented out //Reason.. statement has been separated from the Resource field by an empty line.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you be specific about the change to correct this

// for watching specific namespace, leave it empty for watching all.
// this config is ignored when watching namespaces
Event Event `json:"event,omitempty"`
Namespace string `json:"namespace,omitempty"`
}

Expand Down Expand Up @@ -210,6 +217,7 @@ func (c *Config) Load() error {

// CheckMissingResourceEnvvars will read the environment for equivalent config variables to set
func (c *Config) CheckMissingResourceEnvvars() {

if !c.Resource.DaemonSet && os.Getenv("KW_DAEMONSET") == "true" {
c.Resource.DaemonSet = true
}
Expand All @@ -228,8 +236,8 @@ func (c *Config) CheckMissingResourceEnvvars() {
if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" {
c.Resource.ReplicationController = true
}
if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" {
c.Resource.Services = true
if !c.Resource.Service && os.Getenv("KW_SERVICE") == "true" {
c.Resource.Service = true
}
if !c.Resource.Job && os.Getenv("KW_JOB") == "true" {
c.Resource.Job = true
Expand Down Expand Up @@ -263,6 +271,112 @@ func (c *Config) CheckMissingResourceEnvvars() {
}
}

func (c *Config) UnmarshallConfig() {

// Resource Object Config add events under global scope
if c.Resource != (Resource{}) {
logrus.Info("Configuring Resources For Global Events")
if c.Resource.DaemonSet {
c.Event.Global = append(c.Event.Global, "demonset")
}
if c.Resource.ReplicaSet {
c.Event.Global = append(c.Event.Global, "replicaset")
}
if c.Resource.Namespace {
c.Event.Global = append(c.Event.Global, "namespace")
}
if c.Resource.Deployment {
c.Event.Global = append(c.Event.Global, "deployment")
}
if c.Resource.Pod {
c.Event.Global = append(c.Event.Global, "pod")
}
if c.Resource.ReplicationController {
c.Event.Global = append(c.Event.Global, "replicationcontroller")
}
if c.Resource.Service {
c.Event.Global = append(c.Event.Global, "service")
}
if c.Resource.Job {
c.Event.Global = append(c.Event.Global, "job")
}
if c.Resource.PersistentVolume {
c.Event.Global = append(c.Event.Global, "persistentvolume")
}
if c.Resource.Secret {
c.Event.Global = append(c.Event.Global, "secret")
}
if c.Resource.ConfigMap {
c.Event.Global = append(c.Event.Global, "configmap")
}
if c.Resource.Ingress {
c.Event.Global = append(c.Event.Global, "ingress")
}
} else {
// Configured using Events Config
logrus.Info("Configuring Resources Based on Events Config")
c.configureEvents(c.Event.Global)
c.configureEvents(c.Event.Create)
c.configureEvents(c.Event.Update)
c.configureEvents(c.Event.Delete)
}
}

func (c *Config) configureEvents(s []string) {
for i := 0; i < len(s); i++ {
switch s[i] {
case "deployment":
{
c.Resource.Deployment = true
}
case "replicationcontroller":
{
c.Resource.ReplicationController = true
}
case "replicaset":
{
c.Resource.ReplicaSet = true
}
case "daemonset":
{
c.Resource.DaemonSet = true
}
case "service":
{
c.Resource.Service = true
}
case "pod":
{
c.Resource.Pod = true
}
case "job":
{
c.Resource.Job = true
}
case "persistentvolume":
{
c.Resource.PersistentVolume = true
}
case "namespace":
{
c.Resource.Namespace = true
}
case "secret":
{
c.Resource.Secret = true
}
case "configmap":
{
c.Resource.ConfigMap = true
}
case "ingress":
{
c.Resource.Ingress = true
}
}
}
}

func (c *Config) Write() error {
f, err := os.OpenFile(getConfigFile(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ var configStr = `
"replicationcontroller": "false",
"replicaset": "false",
"daemonset": "false",
"services": "false",
"Service": "false",
codenio marked this conversation as resolved.
Show resolved Hide resolved
"pod": "false",
"secret": "true",
"configmap": "true",
Expand Down
15 changes: 15 additions & 0 deletions examples/conf/kubewatch.conf.events.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
handler:
flock:
url: "https://api.flock.com/hooks/sendMessage/XXXXXXXX" # XXXXXXXX to be replaced with incomming webhooks of the flock channl
event:
global:
- pod
- deployment
create:
- service
update:
delete:
- job
- service
namespace: ""

48 changes: 47 additions & 1 deletion pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ const maxRetries = 5

var serverStartTime time.Time

// Maps for holding events config
var global map[string]uint8
var create map[string]uint8
var delete map[string]uint8
var update map[string]uint8

// Event indicate the informerEvent
type Event struct {
key string
Expand All @@ -69,6 +75,10 @@ type Controller struct {

// Start prepares watchers and run their controllers, then waits for process termination signals
func Start(conf *config.Config, eventHandler handlers.Handler) {

// loads events config into memory for granular alerting
loadEventConfig(conf)

var kubeClient kubernetes.Interface

if _, err := rest.InClusterConfig(); err != nil {
Expand Down Expand Up @@ -238,7 +248,7 @@ func Start(conf *config.Config, eventHandler handlers.Handler) {
go c.Run(stopCh)
}

if conf.Resource.Services {
if conf.Resource.Service {
informer := cache.NewSharedIndexInformer(
&cache.ListWatch{
ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {
Expand Down Expand Up @@ -696,3 +706,39 @@ func (c *Controller) processItem(newEvent Event) error {
}
return nil
}

// loadEventConfig loads event list from Event config for granular alerting
func loadEventConfig(c *config.Config) {

// Load Global events
if len(c.Event.Global) > 0 {
global = make(map[string]uint8)
for _, r := range c.Event.Global {
global[r] = 0
}
}

// Load Create events
if len(c.Event.Create) > 0 {
create = make(map[string]uint8)
for _, r := range c.Event.Create {
create[r] = 0
}
}

// Load Update events
if len(c.Event.Update) > 0 {
update = make(map[string]uint8)
for _, r := range c.Event.Update {
update[r] = 0
}
}

// Load Delete events
if len(c.Event.Delete) > 0 {
delete = make(map[string]uint8)
for _, r := range c.Event.Delete {
delete[r] = 0
}
}
}