From 7dac1096e5ac23237e24c97b71bec3cd9f81f90d Mon Sep 17 00:00:00 2001 From: aananthraj Date: Fri, 7 Jun 2019 22:37:25 +0530 Subject: [PATCH 1/2] Enhance Config for granular alerting This commit - Adds "Event" config section in the .kubewatch.yaml file for granular alerting - Makes Event config optional - Makes Resource config optional for backward compatibility - Enables configuration of alerts either using the "Resource" config or "Event" Config - Renames "Services" option to "Service" option in .kubewatch.yaml file - Populates Namespace details from events key, if the Namespace filed is empty Upon Merging - Enables the user to customize alerts based on the Event configuration provided. eg: pod - alerts on creation and deletion events can be configured. svc - alerts on deletion can be configured individually. --- README.md | 40 +++++++ cmd/resource.go | 4 +- cmd/root.go | 1 + config/config.go | 130 +++++++++++++++++++++-- config/config_test.go | 2 +- examples/conf/kubewatch.conf.events.yaml | 15 +++ pkg/controller/controller.go | 48 ++++++++- 7 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 examples/conf/kubewatch.conf.events.yaml diff --git a/README.md b/README.md index 7b5453e25..82318deee 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,8 @@ handler: url: "" webhook: url: "" + +// Resource Config Section for generic alerting resource: deployment: false replicationcontroller: false @@ -333,6 +335,19 @@ resource: secret: false configmap: false ingress: false + +// Events Config Section for granular alerting +event: + global: + - pod + - deployment + create: + - service + update: + + delete: + - job + - service namespace: "" ``` @@ -418,6 +433,31 @@ $ kubewatch resource add --rc --po --svc $ kubewatch resource remove --rc --po --svc ``` +## Events + +Event config section in `.kubewatch.yaml` file can be used for granular alerting. + +``` +handler: + slack: + token: xoxb-xxxxx-yyyyyyy + channel: kube-watch-test + +event: + global: // global alerts for all events + - pod + - deployment + create: // create alerts for resource object creation + - service + update: // update alerts for resource object updation + - + delete: // delete alerts for resource object deletion + - job + - service + +namespace: "" +``` + # Build ### Using go diff --git a/cmd/resource.go b/cmd/resource.go index f36a454e1..7f9ed747e 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -83,7 +83,7 @@ func configureResource(operation string, cmd *cobra.Command, conf *config.Config }{ { "svc", - &conf.Resource.Services, + &conf.Resource.Service, }, { "deploy", @@ -173,7 +173,7 @@ func init() { resourceConfigRemoveCmd, ) // Add resource object flags as PersistentFlags to resourceConfigCmd - resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for services") + resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for Service") resourceConfigCmd.PersistentFlags().Bool("deploy", false, "watch for deployments") resourceConfigCmd.PersistentFlags().Bool("po", false, "watch for pods") resourceConfigCmd.PersistentFlags().Bool("rc", false, "watch for replication controllers") diff --git a/cmd/root.go b/cmd/root.go index 7b29c372a..743f256db 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,6 +54,7 @@ supported webhooks: logrus.Fatal(err) } config.CheckMissingResourceEnvvars() + config.UnmarshallConfig() c.Run(config) }, } diff --git a/config/config.go b/config/config.go index f0ab26e1a..a01305197 100644 --- a/config/config.go +++ b/config/config.go @@ -24,6 +24,7 @@ import ( "path/filepath" "runtime" + "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -52,7 +53,7 @@ type Resource struct { ReplicationController bool `json:"rc"` ReplicaSet bool `json:"rs"` DaemonSet bool `json:"ds"` - Services bool `json:"svc"` + Service bool `json:"svc"` Pod bool `json:"po"` Job bool `json:"job"` Node bool `json:"node"` @@ -65,18 +66,24 @@ type Resource struct { Ingress bool `json:"ing"` } +// Event struct for granular config +type Event struct { + Global []string `json:"string,omitempty"` + Create []string `json:"create,omitempty"` + Update []string `json:"update,omitempty"` + Delete []string `json:"delete,omitempty"` +} + // Config struct contains kubewatch configuration type Config struct { // Handlers know how to send notifications to specific services. Handler Handler `json:"handler"` //Reason []string `json:"reason"` - - // Resources to watch. - Resource Resource `json:"resource"` - - // For watching specific namespace, leave it empty for watching all. + Resource Resource `json:"resource,omitempty"` + // for watching specific namespace, leave it empty for watching all. // this config is ignored when watching namespaces + Event Event `json:"event,omitempty"` Namespace string `json:"namespace,omitempty"` } @@ -210,6 +217,7 @@ func (c *Config) Load() error { // CheckMissingResourceEnvvars will read the environment for equivalent config variables to set func (c *Config) CheckMissingResourceEnvvars() { + if !c.Resource.DaemonSet && os.Getenv("KW_DAEMONSET") == "true" { c.Resource.DaemonSet = true } @@ -228,8 +236,8 @@ func (c *Config) CheckMissingResourceEnvvars() { if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" { c.Resource.ReplicationController = true } - if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" { - c.Resource.Services = true + if !c.Resource.Service && os.Getenv("KW_SERVICE") == "true" { + c.Resource.Service = true } if !c.Resource.Job && os.Getenv("KW_JOB") == "true" { c.Resource.Job = true @@ -263,6 +271,112 @@ func (c *Config) CheckMissingResourceEnvvars() { } } +func (c *Config) UnmarshallConfig() { + + // Resource Object Config add events under global scope + if c.Resource != (Resource{}) { + logrus.Info("Configuring Resources For Global Events") + if c.Resource.DaemonSet { + c.Event.Global = append(c.Event.Global, "demonset") + } + if c.Resource.ReplicaSet { + c.Event.Global = append(c.Event.Global, "replicaset") + } + if c.Resource.Namespace { + c.Event.Global = append(c.Event.Global, "namespace") + } + if c.Resource.Deployment { + c.Event.Global = append(c.Event.Global, "deployment") + } + if c.Resource.Pod { + c.Event.Global = append(c.Event.Global, "pod") + } + if c.Resource.ReplicationController { + c.Event.Global = append(c.Event.Global, "replicationcontroller") + } + if c.Resource.Service { + c.Event.Global = append(c.Event.Global, "service") + } + if c.Resource.Job { + c.Event.Global = append(c.Event.Global, "job") + } + if c.Resource.PersistentVolume { + c.Event.Global = append(c.Event.Global, "persistentvolume") + } + if c.Resource.Secret { + c.Event.Global = append(c.Event.Global, "secret") + } + if c.Resource.ConfigMap { + c.Event.Global = append(c.Event.Global, "configmap") + } + if c.Resource.Ingress { + c.Event.Global = append(c.Event.Global, "ingress") + } + } else { + // Configured using Events Config + logrus.Info("Configuring Resources Based on Events Config") + c.configureEvents(c.Event.Global) + c.configureEvents(c.Event.Create) + c.configureEvents(c.Event.Update) + c.configureEvents(c.Event.Delete) + } +} + +func (c *Config) configureEvents(s []string) { + for i := 0; i < len(s); i++ { + switch s[i] { + case "deployment": + { + c.Resource.Deployment = true + } + case "replicationcontroller": + { + c.Resource.ReplicationController = true + } + case "replicaset": + { + c.Resource.ReplicaSet = true + } + case "daemonset": + { + c.Resource.DaemonSet = true + } + case "service": + { + c.Resource.Service = true + } + case "pod": + { + c.Resource.Pod = true + } + case "job": + { + c.Resource.Job = true + } + case "persistentvolume": + { + c.Resource.PersistentVolume = true + } + case "namespace": + { + c.Resource.Namespace = true + } + case "secret": + { + c.Resource.Secret = true + } + case "configmap": + { + c.Resource.ConfigMap = true + } + case "ingress": + { + c.Resource.Ingress = true + } + } + } +} + func (c *Config) Write() error { f, err := os.OpenFile(getConfigFile(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { diff --git a/config/config_test.go b/config/config_test.go index f18bae3c6..6b26ae6f2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,7 +39,7 @@ var configStr = ` "replicationcontroller": "false", "replicaset": "false", "daemonset": "false", - "services": "false", + "Service": "false", "pod": "false", "secret": "true", "configmap": "true", diff --git a/examples/conf/kubewatch.conf.events.yaml b/examples/conf/kubewatch.conf.events.yaml new file mode 100644 index 000000000..9f7f66ee6 --- /dev/null +++ b/examples/conf/kubewatch.conf.events.yaml @@ -0,0 +1,15 @@ +handler: + flock: + url: "https://api.flock.com/hooks/sendMessage/XXXXXXXX" # XXXXXXXX to be replaced with incomming webhooks of the flock channl +event: + global: + - pod + - deployment + create: + - service + update: + delete: + - job + - service +namespace: "" + diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7f70eaa0e..a8b703b94 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -50,6 +50,12 @@ const maxRetries = 5 var serverStartTime time.Time +// Maps for holding events config +var global map[string]uint8 +var create map[string]uint8 +var delete map[string]uint8 +var update map[string]uint8 + // Event indicate the informerEvent type Event struct { key string @@ -69,6 +75,10 @@ type Controller struct { // Start prepares watchers and run their controllers, then waits for process termination signals func Start(conf *config.Config, eventHandler handlers.Handler) { + + // loads events config into memory for granular alerting + loadEventConfig(conf) + var kubeClient kubernetes.Interface if _, err := rest.InClusterConfig(); err != nil { @@ -238,7 +248,7 @@ func Start(conf *config.Config, eventHandler handlers.Handler) { go c.Run(stopCh) } - if conf.Resource.Services { + if conf.Resource.Service { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { @@ -696,3 +706,39 @@ func (c *Controller) processItem(newEvent Event) error { } return nil } + +// loadEventConfig loads event list from Event config for granular alerting +func loadEventConfig(c *config.Config) { + + // Load Global events + if len(c.Event.Global) > 0 { + global = make(map[string]uint8) + for _, r := range c.Event.Global { + global[r] = 0 + } + } + + // Load Create events + if len(c.Event.Create) > 0 { + create = make(map[string]uint8) + for _, r := range c.Event.Create { + create[r] = 0 + } + } + + // Load Update events + if len(c.Event.Update) > 0 { + update = make(map[string]uint8) + for _, r := range c.Event.Update { + update[r] = 0 + } + } + + // Load Delete events + if len(c.Event.Delete) > 0 { + delete = make(map[string]uint8) + for _, r := range c.Event.Delete { + delete[r] = 0 + } + } +} From 88d2cf95d9d9b683bbcbb4e9a8d0dafa595277a9 Mon Sep 17 00:00:00 2001 From: aananthraj Date: Sun, 12 Jul 2020 19:51:20 +0530 Subject: [PATCH 2/2] Resolving review This commit, - attempts to resolve the reviews - replaces Service with Services to follow convention --- README.md | 18 +++++++++--------- cmd/resource.go | 6 +++--- config/config.go | 14 +++++++------- config/config_test.go | 2 +- pkg/controller/controller.go | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 82318deee..96be40ea6 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Once the Pod is running, you will start seeing Kubernetes events in your configu To modify what notifications you get, update the `kubewatch` ConfigMap and turn on and off (true/false) resources: -``` +```yaml resource: deployment: false replicationcontroller: false @@ -297,7 +297,7 @@ Testing Handler configs from .kubewatch.yaml ## Viewing config To view the entire config file `$HOME/.kubewatch.yaml` use the following command. -``` +```yaml $ kubewatch config view Contents of .kubewatch.yaml @@ -318,7 +318,7 @@ handler: webhook: url: "" -// Resource Config Section for generic alerting +# Resource Config Section for generic alerting resource: deployment: false replicationcontroller: false @@ -336,7 +336,7 @@ resource: configmap: false ingress: false -// Events Config Section for granular alerting +# Events Config Section for granular alerting event: global: - pod @@ -437,21 +437,21 @@ $ kubewatch resource remove --rc --po --svc Event config section in `.kubewatch.yaml` file can be used for granular alerting. -``` +```yaml handler: slack: token: xoxb-xxxxx-yyyyyyy channel: kube-watch-test event: - global: // global alerts for all events + global: # global alerts for all events - pod - deployment - create: // create alerts for resource object creation + create: # create alerts for resource object creation - service - update: // update alerts for resource object updation + update: # update alerts for resource object updation - - delete: // delete alerts for resource object deletion + delete: # delete alerts for resource object deletion - job - service diff --git a/cmd/resource.go b/cmd/resource.go index 7f9ed747e..49299d52c 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -17,8 +17,8 @@ limitations under the License. package cmd import ( - "github.com/sirupsen/logrus" "github.com/bitnami-labs/kubewatch/config" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -83,7 +83,7 @@ func configureResource(operation string, cmd *cobra.Command, conf *config.Config }{ { "svc", - &conf.Resource.Service, + &conf.Resource.Services, }, { "deploy", @@ -173,7 +173,7 @@ func init() { resourceConfigRemoveCmd, ) // Add resource object flags as PersistentFlags to resourceConfigCmd - resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for Service") + resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for services") resourceConfigCmd.PersistentFlags().Bool("deploy", false, "watch for deployments") resourceConfigCmd.PersistentFlags().Bool("po", false, "watch for pods") resourceConfigCmd.PersistentFlags().Bool("rc", false, "watch for replication controllers") diff --git a/config/config.go b/config/config.go index a01305197..18076595a 100644 --- a/config/config.go +++ b/config/config.go @@ -53,7 +53,7 @@ type Resource struct { ReplicationController bool `json:"rc"` ReplicaSet bool `json:"rs"` DaemonSet bool `json:"ds"` - Service bool `json:"svc"` + Services bool `json:"svc"` Pod bool `json:"po"` Job bool `json:"job"` Node bool `json:"node"` @@ -80,7 +80,7 @@ type Config struct { Handler Handler `json:"handler"` //Reason []string `json:"reason"` - Resource Resource `json:"resource,omitempty"` + Resource Resource `json:"resource"` // for watching specific namespace, leave it empty for watching all. // this config is ignored when watching namespaces Event Event `json:"event,omitempty"` @@ -236,8 +236,8 @@ func (c *Config) CheckMissingResourceEnvvars() { if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" { c.Resource.ReplicationController = true } - if !c.Resource.Service && os.Getenv("KW_SERVICE") == "true" { - c.Resource.Service = true + if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" { + c.Resource.Services = true } if !c.Resource.Job && os.Getenv("KW_JOB") == "true" { c.Resource.Job = true @@ -294,7 +294,7 @@ func (c *Config) UnmarshallConfig() { if c.Resource.ReplicationController { c.Event.Global = append(c.Event.Global, "replicationcontroller") } - if c.Resource.Service { + if c.Resource.Services { c.Event.Global = append(c.Event.Global, "service") } if c.Resource.Job { @@ -341,9 +341,9 @@ func (c *Config) configureEvents(s []string) { { c.Resource.DaemonSet = true } - case "service": + case "services": { - c.Resource.Service = true + c.Resource.Services = true } case "pod": { diff --git a/config/config_test.go b/config/config_test.go index 6b26ae6f2..f18bae3c6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,7 +39,7 @@ var configStr = ` "replicationcontroller": "false", "replicaset": "false", "daemonset": "false", - "Service": "false", + "services": "false", "pod": "false", "secret": "true", "configmap": "true", diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index a8b703b94..cca9b75d0 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -248,7 +248,7 @@ func Start(conf *config.Config, eventHandler handlers.Handler) { go c.Run(stopCh) } - if conf.Resource.Service { + if conf.Resource.Services { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {