Skip to content

Commit

Permalink
Merge pull request #49 from logzio/DEV-46838-configure-logging-for-gr…
Browse files Browse the repository at this point in the history 
…afana-alert-notifications

DEV-46838-configure-logging-for-grafana-alert-notifications
  • Loading branch information
@kotkovkirill
kotkovkirill authored Oct 19, 2024
2 parents d1cf24c + c8cfdaf commit cd77e37
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 16 deletions.
27 changes: 21 additions & 6 deletions pkg/services/ngalert/notifier/channels/logzio_opsgenie.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
"net/http"
"github.com/google/uuid"
)

// LOGZ.IO GRAFANA CHANGE :: DEV-35483 - Add type for logzio Opsgenie integration
Expand Down Expand Up @@ -75,32 +76,43 @@ func NewLogzioOpsgenieNotifier(config *LogzioOpsgenieConfig, ns notifications.We
APIKey: config.APIKey,
APIUrl: config.APIUrl,
tmpl: t,
log: log.New("alerting.notifier." + config.Name),
log: log.New("alerting.notifier.logzio_opsgenie"),
ns: ns,
}
}

func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
on.log.Debug("Executing Opsgenie (Logzio Integration) notification", "notification", on.Name)

id := uuid.New()
logger := on.log.New("requestId", id.String(), "notificationId", on.UID)

logger.Info("Executing Opsgenie (Logzio Integration) notification", "notification", on.Name)

alerts := types.Alerts(as...)
if alerts.Status() == model.AlertResolved && !on.SendResolved() {
on.log.Debug("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved())
logger.Info("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved())
return true, nil
}

bodyJSON, err := on.buildOpsgenieMessage(ctx, alerts, as)
if err != nil {
return false, fmt.Errorf("build Opsgenie message: %w", err)
error := fmt.Errorf("build Opsgenie message: %w", err)
logger.Error(error.Error())
return false, error
}

body, err := json.Marshal(bodyJSON)
if err != nil {
return false, fmt.Errorf("marshal json: %w", err)
error := fmt.Errorf("marshal json: %w", err)
logger.Error(error.Error())
return false, error
}

url := fmt.Sprintf("%s?apiKey=%s", on.APIUrl, on.APIKey)


logger.Info("Sending Opsgenie (Logzio Integration) API request", "url", on.APIUrl, "body", minify(body))

cmd := &models.SendWebhookSync{
Url: url,
Body: string(body),
Expand All @@ -111,9 +123,12 @@ func (on *LogzioOpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert
}

if err := on.ns.SendWebhookSync(ctx, cmd); err != nil {
return false, fmt.Errorf("send notification to Opsgenie (Logzio Integration): %w", err)
error := fmt.Errorf("Sending Opsgenie (Logzio Integration) API request failed: %w", err)
logger.Error(error.Error())
return false, error
}

logger.Info("Sending Opsgenie API request succeeded", "url", on.APIUrl)
return true, nil
}

Expand Down
29 changes: 23 additions & 6 deletions pkg/services/ngalert/notifier/channels/opsgenie.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
"github.com/google/uuid"
)

const (
Expand Down Expand Up @@ -99,37 +100,50 @@ func NewOpsgenieNotifier(config *OpsgenieConfig, ns notifications.WebhookSender,
OverridePriority: config.OverridePriority,
SendTagsAs: config.SendTagsAs,
tmpl: t,
log: log.New("alerting.notifier." + config.Name),
log: log.New("alerting.notifier.opsgenie"),
ns: ns,
}
}

// Notify sends an alert notification to Opsgenie
func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
on.log.Debug("Executing Opsgenie notification", "notification", on.Name)

id := uuid.New()
logger := on.log.New("requestId", id.String(), "notificationId", on.UID)

logger.Info("Executing Opsgenie notification", "notification", on.Name)

alerts := types.Alerts(as...)
if alerts.Status() == model.AlertResolved && !on.SendResolved() {
on.log.Debug("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved())
logger.Info("Not sending a trigger to Opsgenie", "status", alerts.Status(), "auto resolve", on.SendResolved())
return true, nil
}

bodyJSON, url, err := on.buildOpsgenieMessage(ctx, alerts, as)
if err != nil {
return false, fmt.Errorf("build Opsgenie message: %w", err)
error := fmt.Errorf("build Opsgenie message: %w", err)
logger.Error(error.Error())
return false, error
}

if url == "" {
// Resolved alert with no auto close.
// Hence skip sending anything.
logger.Info("Resolved alert with no auto close, not sending anything")
return true, nil
}



body, err := json.Marshal(bodyJSON)
if err != nil {
return false, fmt.Errorf("marshal json: %w", err)
error := fmt.Errorf("marshal json: %w", err)
logger.Error(error.Error())
return false, error
}

logger.Info("Sending Opsgenie API request", "url", url, "body", minify(body))

cmd := &models.SendWebhookSync{
Url: url,
Body: string(body),
Expand All @@ -141,9 +155,12 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
}

if err := on.ns.SendWebhookSync(ctx, cmd); err != nil {
return false, fmt.Errorf("send notification to Opsgenie: %w", err)
error := fmt.Errorf("Sending Opsgenie API request failed: %w", err)
logger.Error(error.Error())
return false, error
}

logger.Info("Sending Opsgenie API request succeeded", "url", url)
return true, nil
}

Expand Down
25 changes: 21 additions & 4 deletions pkg/services/ngalert/notifier/channels/slack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
"github.com/google/uuid"
)

var SlackAPIEndpoint = "https://slack.com/api/chat.postMessage"
Expand Down Expand Up @@ -175,6 +176,10 @@ type attachment struct {

// Notify sends an alert notification to Slack.
func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {

id := uuid.New()
logger := sn.log.New("requestId", id.String(), "notificationId", sn.UID)

msg, err := sn.buildSlackMessage(ctx, as)
if err != nil {
return false, fmt.Errorf("build slack message: %w", err)
Expand All @@ -185,7 +190,8 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
return false, fmt.Errorf("marshal json: %w", err)
}

sn.log.Debug("Sending Slack API request", "url", sn.URL.String(), "data", string(b))
logger.Info("Sending Slack API request", "url", sn.URL.String(), "body", minify(b))

request, err := http.NewRequestWithContext(ctx, http.MethodPost, sn.URL.String(), bytes.NewReader(b))
if err != nil {
return false, fmt.Errorf("failed to create HTTP request: %w", err)
Expand All @@ -198,11 +204,11 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
panic("Token should be set when using the Slack chat API")
}
} else {
sn.log.Debug("Adding authorization header to HTTP request")
logger.Debug("Adding authorization header to HTTP request")
request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", sn.Token))
}

if err := sendSlackRequest(request, sn.log); err != nil {
if err := sendSlackRequest(request, logger); err != nil {
return false, err
}
return true, nil
Expand Down Expand Up @@ -237,6 +243,7 @@ var sendSlackRequest = func(request *http.Request, logger log.Logger) error {

body, err := io.ReadAll(resp.Body)
if err != nil {
logger.Error("failed to read response body", "url", request.URL.String())
return fmt.Errorf("failed to read response body: %w", err)
}

Expand Down Expand Up @@ -264,7 +271,7 @@ var sendSlackRequest = func(request *http.Request, logger log.Logger) error {
return fmt.Errorf("failed to make Slack API request: %s", rslt.Err)
}

logger.Debug("Sending Slack API request succeeded", "url", request.URL.String(), "statusCode", resp.Status)
logger.Info("Sending Slack API request succeeded", "url", request.URL.String(), "statusCode", resp.Status)
return nil
}

Expand Down Expand Up @@ -343,3 +350,13 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, as []*types.Aler
func (sn *SlackNotifier) SendResolved() bool {
return !sn.GetDisableResolveMessage()
}

func minify(src []byte) string {
dst := &bytes.Buffer{}
if err := json.Compact(dst, []byte(src)); err != nil {
r := strings.NewReplacer("\n", "","\"", " ")
return r.Replace(string(src))
}

return strings.ReplaceAll(dst.String(), "\"", " ")
}

0 comments on commit cd77e37

Please sign in to comment.