Skip to content
This repository has been archived by the owner on Apr 8, 2022. It is now read-only.

Commit

Permalink
Enhance Config for granular alerting
Browse files Browse the repository at this point in the history
This commit

- Adds "Event" config section in the .kubewatch.yaml file for granular alerting
- Makes Event config optional
- Makes Resource config optional for backward compatibility
- Enables configuration of alerts either using the "Resource" config or "Event" Config
- Renames "Services" option to "Service" option in .kubewatch.yaml file
- Populates Namespace details from events key, if the Namespace filed is empty

Upon Merging

- Enables the user to customize alerts based on the Event configuration provided.
  eg: pod - alerts on creation and deletion events can be configured.
      svc - alerts on deletion can be configured individually.
  • Loading branch information
codenio committed Jul 7, 2020
1 parent a882f75 commit 706daf8
Show file tree
Hide file tree
Showing 7 changed files with 227 additions and 8 deletions.
40 changes: 40 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,8 @@ handler:
url: ""
webhook:
url: ""

// Resource Config Section for generic alerting
resource:
deployment: false
replicationcontroller: false
Expand All @@ -333,6 +335,19 @@ resource:
secret: false
configmap: false
ingress: false

// Events Config Section for granular alerting
event:
global:
- pod
- deployment
create:
- service
update:

delete:
- job
- service
namespace: ""

```
Expand Down Expand Up @@ -418,6 +433,31 @@ $ kubewatch resource add --rc --po --svc
$ kubewatch resource remove --rc --po --svc
```

## Events

Event config section in `.kubewatch.yaml` file can be used for granular alerting.

```
handler:
slack:
token: xoxb-xxxxx-yyyyyyy
channel: kube-watch-test
event:
global: // global alerts for all events
- pod
- deployment
create: // create alerts for resource object creation
- service
update: // update alerts for resource object updation
-
delete: // delete alerts for resource object deletion
- job
- service
namespace: ""
```

# Build

### Using go
Expand Down
4 changes: 2 additions & 2 deletions cmd/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func configureResource(operation string, cmd *cobra.Command, conf *config.Config
}{
{
"svc",
&conf.Resource.Services,
&conf.Resource.Service,
},
{
"deploy",
Expand Down Expand Up @@ -173,7 +173,7 @@ func init() {
resourceConfigRemoveCmd,
)
// Add resource object flags as PersistentFlags to resourceConfigCmd
resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for services")
resourceConfigCmd.PersistentFlags().Bool("svc", false, "watch for Service")
resourceConfigCmd.PersistentFlags().Bool("deploy", false, "watch for deployments")
resourceConfigCmd.PersistentFlags().Bool("po", false, "watch for pods")
resourceConfigCmd.PersistentFlags().Bool("rc", false, "watch for replication controllers")
Expand Down
1 change: 1 addition & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ supported webhooks:
logrus.Fatal(err)
}
config.CheckMissingResourceEnvvars()
config.UnmarshallConfig()
c.Run(config)
},
}
Expand Down
125 changes: 121 additions & 4 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"path/filepath"
"runtime"

"github.com/Sirupsen/logrus"
"gopkg.in/yaml.v3"
)

Expand All @@ -45,7 +46,7 @@ type Resource struct {
ReplicationController bool `json:"rc"`
ReplicaSet bool `json:"rs"`
DaemonSet bool `json:"ds"`
Services bool `json:"svc"`
Service bool `json:"svc"`
Pod bool `json:"po"`
Job bool `json:"job"`
Node bool `json:"node"`
Expand All @@ -58,13 +59,22 @@ type Resource struct {
Ingress bool `json:"ing"`
}

// Event struct for granular config
type Event struct {
Global []string `json:"string,omitempty"`
Create []string `json:"create,omitempty"`
Update []string `json:"update,omitempty"`
Delete []string `json:"delete,omitempty"`
}

// Config struct contains kubewatch configuration
type Config struct {
Handler Handler `json:"handler"`
//Reason []string `json:"reason"`
Resource Resource `json:"resource"`
Resource Resource `json:"resource,omitempty"`
// for watching specific namespace, leave it empty for watching all.
// this config is ignored when watching namespaces
Event Event `json:"event,omitempty"`
Namespace string `json:"namespace,omitempty"`
}

Expand Down Expand Up @@ -177,6 +187,7 @@ func (c *Config) Load() error {

// CheckMissingResourceEnvvars will read the environment for equivalent config variables to set
func (c *Config) CheckMissingResourceEnvvars() {

if !c.Resource.DaemonSet && os.Getenv("KW_DAEMONSET") == "true" {
c.Resource.DaemonSet = true
}
Expand All @@ -195,8 +206,8 @@ func (c *Config) CheckMissingResourceEnvvars() {
if !c.Resource.ReplicationController && os.Getenv("KW_REPLICATION_CONTROLLER") == "true" {
c.Resource.ReplicationController = true
}
if !c.Resource.Services && os.Getenv("KW_SERVICE") == "true" {
c.Resource.Services = true
if !c.Resource.Service && os.Getenv("KW_SERVICE") == "true" {
c.Resource.Service = true
}
if !c.Resource.Job && os.Getenv("KW_JOB") == "true" {
c.Resource.Job = true
Expand Down Expand Up @@ -230,6 +241,112 @@ func (c *Config) CheckMissingResourceEnvvars() {
}
}

func (c *Config) UnmarshallConfig() {

// Resource Object Config add events under global scope
if c.Resource != (Resource{}) {
logrus.Info("Configuring Resources For Global Events")
if c.Resource.DaemonSet {
c.Event.Global = append(c.Event.Global, "demonset")
}
if c.Resource.ReplicaSet {
c.Event.Global = append(c.Event.Global, "replicaset")
}
if c.Resource.Namespace {
c.Event.Global = append(c.Event.Global, "namespace")
}
if c.Resource.Deployment {
c.Event.Global = append(c.Event.Global, "deployment")
}
if c.Resource.Pod {
c.Event.Global = append(c.Event.Global, "pod")
}
if c.Resource.ReplicationController {
c.Event.Global = append(c.Event.Global, "replicationcontroller")
}
if c.Resource.Service {
c.Event.Global = append(c.Event.Global, "service")
}
if c.Resource.Job {
c.Event.Global = append(c.Event.Global, "job")
}
if c.Resource.PersistentVolume {
c.Event.Global = append(c.Event.Global, "persistentvolume")
}
if c.Resource.Secret {
c.Event.Global = append(c.Event.Global, "secret")
}
if c.Resource.ConfigMap {
c.Event.Global = append(c.Event.Global, "configmap")
}
if c.Resource.Ingress {
c.Event.Global = append(c.Event.Global, "ingress")
}
} else {
// Configured using Events Config
logrus.Info("Configuring Resources Based on Events Config")
c.configureEvents(c.Event.Global)
c.configureEvents(c.Event.Create)
c.configureEvents(c.Event.Update)
c.configureEvents(c.Event.Delete)
}
}

func (c *Config) configureEvents(s []string) {
for i := 0; i < len(s); i++ {
switch s[i] {
case "deployment":
{
c.Resource.Deployment = true
}
case "replicationcontroller":
{
c.Resource.ReplicationController = true
}
case "replicaset":
{
c.Resource.ReplicaSet = true
}
case "daemonset":
{
c.Resource.DaemonSet = true
}
case "service":
{
c.Resource.Service = true
}
case "pod":
{
c.Resource.Pod = true
}
case "job":
{
c.Resource.Job = true
}
case "persistentvolume":
{
c.Resource.PersistentVolume = true
}
case "namespace":
{
c.Resource.Namespace = true
}
case "secret":
{
c.Resource.Secret = true
}
case "configmap":
{
c.Resource.ConfigMap = true
}
case "ingress":
{
c.Resource.Ingress = true
}
}
}
}

func (c *Config) Write() error {
f, err := os.OpenFile(getConfigFile(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ var configStr = `
"replicationcontroller": "false",
"replicaset": "false",
"daemonset": "false",
"services": "false",
"Service": "false",
"pod": "false",
"secret": "true",
"configmap": "true",
Expand Down
15 changes: 15 additions & 0 deletions examples/conf/kubewatch.conf.events.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
handler:
flock:
url: "https://api.flock.com/hooks/sendMessage/XXXXXXXX" # XXXXXXXX to be replaced with incomming webhooks of the flock channl
event:
global:
- pod
- deployment
create:
- service
update:
delete:
- job
- service
namespace: ""

48 changes: 47 additions & 1 deletion pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ const maxRetries = 5

var serverStartTime time.Time

// Maps for holding events config
var global map[string]uint8
var create map[string]uint8
var delete map[string]uint8
var update map[string]uint8

// Event indicate the informerEvent
type Event struct {
key string
Expand All @@ -69,6 +75,10 @@ type Controller struct {

// Start prepares watchers and run their controllers, then waits for process termination signals
func Start(conf *config.Config, eventHandler handlers.Handler) {

// loads events config into memory for granular alerting
loadEventConfig(conf)

var kubeClient kubernetes.Interface

if _, err := rest.InClusterConfig(); err != nil {
Expand Down Expand Up @@ -238,7 +248,7 @@ func Start(conf *config.Config, eventHandler handlers.Handler) {
go c.Run(stopCh)
}

if conf.Resource.Services {
if conf.Resource.Service {
informer := cache.NewSharedIndexInformer(
&cache.ListWatch{
ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {
Expand Down Expand Up @@ -696,3 +706,39 @@ func (c *Controller) processItem(newEvent Event) error {
}
return nil
}

// loadEventConfig loads event list from Event config for granular alerting
func loadEventConfig(c *config.Config) {

// Load Global events
if len(c.Event.Global) > 0 {
global = make(map[string]uint8)
for _, r := range c.Event.Global {
global[r] = 0
}
}

// Load Create events
if len(c.Event.Create) > 0 {
create = make(map[string]uint8)
for _, r := range c.Event.Create {
create[r] = 0
}
}

// Load Update events
if len(c.Event.Update) > 0 {
update = make(map[string]uint8)
for _, r := range c.Event.Update {
update[r] = 0
}
}

// Load Delete events
if len(c.Event.Delete) > 0 {
delete = make(map[string]uint8)
for _, r := range c.Event.Delete {
delete[r] = 0
}
}
}

0 comments on commit 706daf8

Please sign in to comment.