From 73fd5f54b6c0b0f73aaf1aa75fd6cf4f851d0930 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Tue, 15 Oct 2024 11:44:53 +0200 Subject: [PATCH 01/10] DEV-46838-configure-logging-for-grafana-alert-notifications --- pkg/services/ngalert/notifier/channels/slack.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index c0db46365ccab..8d377c6f40dc1 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" + "github.com/google/uuid" ) var SlackAPIEndpoint = "https://slack.com/api/chat.postMessage" @@ -175,6 +176,10 @@ type attachment struct { // Notify sends an alert notification to Slack. func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { + + id := uuid.New() + logger := sn.log.New("notificationId", id.String()) + msg, err := sn.buildSlackMessage(ctx, as) if err != nil { return false, fmt.Errorf("build slack message: %w", err) @@ -185,7 +190,7 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, return false, fmt.Errorf("marshal json: %w", err) } - sn.log.Debug("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) + logger.Info("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b)) if err != nil { return false, fmt.Errorf("failed to create HTTP request: %w", err) @@ -198,11 +203,11 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, panic("Token should be set when using the Slack chat API") } } else { - sn.log.Debug("Adding authorization header to HTTP request") + logger.Debug("Adding authorization header to HTTP request") request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", sn.Token)) } - if err := sendSlackRequest(request, sn.log); err != nil { + if err := sendSlackRequest(request, logger); err != nil { return false, err } return true, nil @@ -237,6 +242,7 @@ var sendSlackRequest = func(request *http.Request, logger log.Logger) error { body, err := io.ReadAll(resp.Body) if err != nil { + logger.Error("failed to read response body", "url", request.URL.String()) return fmt.Errorf("failed to read response body: %w", err) } @@ -264,7 +270,7 @@ var sendSlackRequest = func(request *http.Request, logger log.Logger) error { return fmt.Errorf("failed to make Slack API request: %s", rslt.Err) } - logger.Debug("Sending Slack API request succeeded", "url", request.URL.String(), "statusCode", resp.Status) + logger.Info("Sending Slack API request succeeded", "url", request.URL.String(), "statusCode", resp.Status) return nil } From 7a02f13eab0d79244b7044fc8423826033e7cdeb Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Wed, 16 Oct 2024 11:48:29 +0200 Subject: [PATCH 02/10] fix --- pkg/services/ngalert/notifier/channels/slack.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index 8d377c6f40dc1..f72f7ee027cb8 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -190,7 +190,9 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, return false, fmt.Errorf("marshal json: %w", err) } - logger.Info("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) + //logger.Info("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) + logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", "testbody") + request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b)) if err != nil { return false, fmt.Errorf("failed to create HTTP request: %w", err) From 2a5444ef0463428b9ed55f8b74f8975cfe3e13cf Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Wed, 16 Oct 2024 12:51:11 +0200 Subject: [PATCH 03/10] minify fix --- pkg/services/ngalert/notifier/channels/slack.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index f72f7ee027cb8..669fbf751f20f 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -191,7 +191,7 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, } //logger.Info("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) - logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", "testbody") + logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", sn.minify(b)) request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b)) if err != nil { @@ -351,3 +351,11 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, as []*types.Aler func (sn *SlackNotifier) SendResolved() bool { return !sn.GetDisableResolveMessage() } + +func (sn *SlackNotifier) minify(src []byte) string { + dst := &bytes.Buffer{} + if err := json.Compact(dst, []byte(src)); err != nil { + return strings.ReplaceAll(string(src), "\n", "") + } + return dst.String() +} From 8d508184821a68dd8fa3ab89863abdbf683a92b9 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Wed, 16 Oct 2024 13:39:33 +0200 Subject: [PATCH 04/10] fix --- pkg/services/ngalert/notifier/channels/slack.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index 669fbf751f20f..36d37aa00d749 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -355,7 +355,9 @@ func (sn *SlackNotifier) SendResolved() bool { func (sn *SlackNotifier) minify(src []byte) string { dst := &bytes.Buffer{} if err := json.Compact(dst, []byte(src)); err != nil { - return strings.ReplaceAll(string(src), "\n", "") + r := strings.NewReplacer("\n", "","\"", " ") + return r.Replace(string(src)) } - return dst.String() + + return strings.ReplaceAll(dst.String(), "\"", " ") } From 5276ceda7bf6ec9e320090d2193302f994be3e8d Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Wed, 16 Oct 2024 16:25:18 +0200 Subject: [PATCH 05/10] opsgenie logs added --- .../ngalert/notifier/channels/opsgenie.go | 27 +++++++++++++++---- .../ngalert/notifier/channels/slack.go | 3 +-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/opsgenie.go b/pkg/services/ngalert/notifier/channels/opsgenie.go index 49d35638a8454..0c06b53eb4834 100644 --- a/pkg/services/ngalert/notifier/channels/opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/opsgenie.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" + "github.com/google/uuid" ) const ( @@ -106,30 +107,43 @@ func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender, // Notify sends an alert notification to Opsgenie func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { - on.log.Debug("Executing Opsgenie notification", "notification", on.Name) + + id := uuid.New() + logger := on.log.New("notificationId", id.String()) + + logger.Info("Executing Opsgenie notification", "notification", on.Name) alerts := types.Alerts(as...) if alerts.Status() == model.AlertResolved && !on.SendResolved() { - on.log.Debug("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved()) + logger.Info("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved()) return true, nil } bodyJSON, url, err := on.buildOpsgenieMessage(ctx, alerts, as) if err != nil { - return false, fmt.Errorf("build Opsgenie message: %w", err) + error := fmt.Errorf("build Opsgenie message: %w", err) + logger.Error(error.Error()) + return false, error } if url == "" { // Resolved alert with no auto close. // Hence skip sending anything. + logger.Info("Resolved alert with no auto close, not sending anything") return true, nil } + + body, err := json.Marshal(bodyJSON) if err != nil { - return false, fmt.Errorf("marshal json: %w", err) + error := fmt.Errorf("marshal json: %w", err) + logger.Error(error.Error()) + return false, error } + logger.Info("Sending Opsgenie API request", "url", url, "body", minify(body)) + cmd := &models.SendWebhookSync{ Url: url, Body: string(body), @@ -141,9 +155,12 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo } if err := on.ns.SendWebhookSync(ctx, cmd); err != nil { - return false, fmt.Errorf("send notification to Opsgenie: %w", err) + error := fmt.Errorf("send notification to Opsgenie: %w", err) + logger.Error(error.Error()) + return false, error } + logger.Info("Sending Opsgenie API request succeeded", "url", url) return true, nil } diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index 36d37aa00d749..45db1569901c2 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -190,7 +190,6 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, return false, fmt.Errorf("marshal json: %w", err) } - //logger.Info("Sending Slack API request", "url", sn.URL.String(), "data", string(b)) logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", sn.minify(b)) request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b)) @@ -352,7 +351,7 @@ func (sn *SlackNotifier) SendResolved() bool { return !sn.GetDisableResolveMessage() } -func (sn *SlackNotifier) minify(src []byte) string { +func minify(src []byte) string { dst := &bytes.Buffer{} if err := json.Compact(dst, []byte(src)); err != nil { r := strings.NewReplacer("\n", "","\"", " ") From 3bdfe1cf1e0d717b73b1c9d81ff4b43149221ce7 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Wed, 16 Oct 2024 16:54:44 +0200 Subject: [PATCH 06/10] fix --- pkg/services/ngalert/notifier/channels/slack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index 45db1569901c2..c1b7f12b9d5ea 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -190,7 +190,7 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, return false, fmt.Errorf("marshal json: %w", err) } - logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", sn.minify(b)) + logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", minify(b)) request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b)) if err != nil { From b922230842629e437a2f8c6b9d91e27c93e33c15 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Thu, 17 Oct 2024 10:40:07 +0200 Subject: [PATCH 07/10] fix --- .../ngalert/notifier/channels/logzio_opsgenie.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go b/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go index 870896f191dc2..f92820b8c0dff 100644 --- a/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "net/http" + "github.com/google/uuid" ) // LOGZ.IO GRAFANA CHANGE :: DEV-35483 - Add type for logzio Opsgenie integration @@ -81,11 +82,15 @@ func NewLogzioOpsgenieNotifier(config *LogzioOpsgenieConfig, ns notifications.We } func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { - on.log.Debug("Executing Opsgenie (Logzio Integration) notification", "notification", on.Name) + + id := uuid.New() + logger := on.log.New("notificationId", id.String()) + + logger.Info("Executing Opsgenie (Logzio Integration) notification", "notification", on.Name) alerts := types.Alerts(as...) if alerts.Status() == model.AlertResolved && !on.SendResolved() { - on.log.Debug("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved()) + logger.Info("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved()) return true, nil } From c2e849410b297db945e931cff34728913ff19dfd Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Thu, 17 Oct 2024 12:06:09 +0200 Subject: [PATCH 08/10] logs added --- pkg/services/ngalert/notifier/channels/opsgenie.go | 4 ++-- pkg/services/ngalert/notifier/channels/slack.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/opsgenie.go b/pkg/services/ngalert/notifier/channels/opsgenie.go index 0c06b53eb4834..3079b6a3dc22b 100644 --- a/pkg/services/ngalert/notifier/channels/opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/opsgenie.go @@ -109,7 +109,7 @@ func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender, func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { id := uuid.New() - logger := on.log.New("notificationId", id.String()) + logger := on.log.New("notificationId", id.String(), "contactpointId", on.UID) logger.Info("Executing Opsgenie notification", "notification", on.Name) @@ -155,7 +155,7 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo } if err := on.ns.SendWebhookSync(ctx, cmd); err != nil { - error := fmt.Errorf("send notification to Opsgenie: %w", err) + error := fmt.Errorf("Sending Opsgenie API request failed: %w", err) logger.Error(error.Error()) return false, error } diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index c1b7f12b9d5ea..22487212a7e0a 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -178,7 +178,7 @@ type attachment struct { func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { id := uuid.New() - logger := sn.log.New("notificationId", id.String()) + logger := sn.log.New("notificationId", id.String(), "contactpointId", sn.UID) msg, err := sn.buildSlackMessage(ctx, as) if err != nil { From ef35617aa767c1d2ceb31a1bb7290a961b83b026 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Thu, 17 Oct 2024 12:53:52 +0200 Subject: [PATCH 09/10] fixes --- pkg/services/ngalert/notifier/channels/opsgenie.go | 4 ++-- pkg/services/ngalert/notifier/channels/slack.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/opsgenie.go b/pkg/services/ngalert/notifier/channels/opsgenie.go index 3079b6a3dc22b..714f1fe514d29 100644 --- a/pkg/services/ngalert/notifier/channels/opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/opsgenie.go @@ -100,7 +100,7 @@ func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender, OverridePriority: config.OverridePriority, SendTagsAs: config.SendTagsAs, tmpl: t, - log: log.New("alerting.notifier." + config.Name), + log: log.New("alerting.notifier.opsgenie"), ns: ns, } } @@ -109,7 +109,7 @@ func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender, func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { id := uuid.New() - logger := on.log.New("notificationId", id.String(), "contactpointId", on.UID) + logger := on.log.New("requestId", id.String(), "notificationId", on.UID) logger.Info("Executing Opsgenie notification", "notification", on.Name) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index 22487212a7e0a..acb1a4da162b7 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -178,7 +178,7 @@ type attachment struct { func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { id := uuid.New() - logger := sn.log.New("notificationId", id.String(), "contactpointId", sn.UID) + logger := sn.log.New("requestId", id.String(), "notificationId", sn.UID) msg, err := sn.buildSlackMessage(ctx, as) if err != nil { From c8cfdafc77b2494eba50ecd8be5dd15f723ae0f2 Mon Sep 17 00:00:00 2001 From: kotkovkirill Date: Sat, 19 Oct 2024 10:43:31 +0200 Subject: [PATCH 10/10] logzio integration logs added --- .../notifier/channels/logzio_opsgenie.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go b/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go index f92820b8c0dff..90dddbbf6cd88 100644 --- a/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/logzio_opsgenie.go @@ -76,7 +76,7 @@ func NewLogzioOpsgenieNotifier(config *LogzioOpsgenieConfig, ns notifications.We APIKey: config.APIKey, APIUrl: config.APIUrl, tmpl: t, - log: log.New("alerting.notifier." + config.Name), + log: log.New("alerting.notifier.logzio_opsgenie"), ns: ns, } } @@ -84,7 +84,7 @@ func NewLogzioOpsgenieNotifier(config *LogzioOpsgenieConfig, ns notifications.We func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { id := uuid.New() - logger := on.log.New("notificationId", id.String()) + logger := on.log.New("requestId", id.String(), "notificationId", on.UID) logger.Info("Executing Opsgenie (Logzio Integration) notification", "notification", on.Name) @@ -96,16 +96,23 @@ func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert bodyJSON, err := on.buildOpsgenieMessage(ctx, alerts, as) if err != nil { - return false, fmt.Errorf("build Opsgenie message: %w", err) + error := fmt.Errorf("build Opsgenie message: %w", err) + logger.Error(error.Error()) + return false, error } body, err := json.Marshal(bodyJSON) if err != nil { - return false, fmt.Errorf("marshal json: %w", err) + error := fmt.Errorf("marshal json: %w", err) + logger.Error(error.Error()) + return false, error } url := fmt.Sprintf("%s?apiKey=%s", on.APIUrl, on.APIKey) + + logger.Info("Sending Opsgenie (Logzio Integration) API request", "url", on.APIUrl, "body", minify(body)) + cmd := &models.SendWebhookSync{ Url: url, Body: string(body), @@ -116,9 +123,12 @@ func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert } if err := on.ns.SendWebhookSync(ctx, cmd); err != nil { - return false, fmt.Errorf("send notification to Opsgenie (Logzio Integration): %w", err) + error := fmt.Errorf("Sending Opsgenie (Logzio Integration) API request failed: %w", err) + logger.Error(error.Error()) + return false, error } + logger.Info("Sending Opsgenie API request succeeded", "url", on.APIUrl) return true, nil }