-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Vyom Yadav <[email protected]>
- Loading branch information
1 parent
6d4a5df
commit eb5c7d0
Showing
5 changed files
with
235 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2024 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Package metrics provides metrics for the reminder service | ||
| package metrics | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "go.opentelemetry.io/otel/metric" | ||
| ) | ||
|
|
||
| // Default bucket boundaries in seconds for the delay histograms | ||
| var delayBuckets = []float64{ | ||
| 60, // 1 minute | ||
| 300, // 5 minutes | ||
| 600, // 10 minutes | ||
| 1800, // 30 minutes | ||
| 3600, // 1 hour | ||
| 7200, // 2 hours | ||
| 10800, // 3 hours | ||
| 18000, // 5 hours | ||
| 25200, // 7 hours | ||
| 36000, // 10 hours | ||
| } | ||
|
|
||
| // Metrics contains all the metrics for the reminder service | ||
| type Metrics struct { | ||
| // Time between when a reminder became eligible and when it was sent | ||
| SendDelay metric.Float64Histogram | ||
|
|
||
| // Time between when a reminder became eligible and when it was sent for the first time | ||
| NewSendDelay metric.Float64Histogram | ||
|
|
||
| // Current number of reminders in the batch | ||
| BatchSize metric.Int64Histogram | ||
| } | ||
|
|
||
| // NewMetrics creates a new metrics instance | ||
| func NewMetrics(meter metric.Meter) (*Metrics, error) { | ||
| sendDelay, err := meter.Float64Histogram( | ||
| "send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds)"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| newSendDelay, err := meter.Float64Histogram( | ||
| "new_send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds) for first time reminders"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| batchSize, err := meter.Int64Histogram( | ||
| "batch_size", | ||
| metric.WithDescription("Current number of reminders in the batch"), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &Metrics{ | ||
| SendDelay: sendDelay, | ||
| NewSendDelay: newSendDelay, | ||
| BatchSize: batchSize, | ||
| }, nil | ||
| } | ||
|
|
||
| // RecordBatch records the metrics for a batch of reminders | ||
| func (m *Metrics) RecordBatch(ctx context.Context, size int64) { | ||
| m.BatchSize.Record(ctx, size) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2024 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package reminder | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus/promhttp" | ||
| "github.com/rs/zerolog" | ||
| "go.opentelemetry.io/otel" | ||
| "go.opentelemetry.io/otel/exporters/prometheus" | ||
| sdkmetric "go.opentelemetry.io/otel/sdk/metric" | ||
| "go.opentelemetry.io/otel/sdk/resource" | ||
| semconv "go.opentelemetry.io/otel/semconv/v1.17.0" | ||
| ) | ||
|
|
||
| const ( | ||
| metricsPath = "/metrics" | ||
| readHeaderTimeout = 2 * time.Second | ||
| ) | ||
|
|
||
| func (r *reminder) startMetricServer(ctx context.Context, mpReady chan<- struct{}) error { | ||
| logger := zerolog.Ctx(ctx) | ||
|
|
||
| prometheusExporter, err := prometheus.New( | ||
| prometheus.WithNamespace("reminder"), | ||
| ) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create Prometheus exporter: %w", err) | ||
| } | ||
|
|
||
| res := resource.NewWithAttributes( | ||
| semconv.SchemaURL, | ||
| semconv.ServiceName("reminder"), | ||
| // TODO: Make this auto-generated | ||
| semconv.ServiceVersion("v0.1.0"), | ||
| ) | ||
|
|
||
| mp := sdkmetric.NewMeterProvider( | ||
| sdkmetric.WithReader(prometheusExporter), | ||
| sdkmetric.WithResource(res), | ||
| ) | ||
|
|
||
| otel.SetMeterProvider(mp) | ||
|
|
||
| // Indicates that a global MeterProvider is available | ||
| close(mpReady) | ||
|
|
||
| mux := http.NewServeMux() | ||
| mux.Handle(metricsPath, promhttp.Handler()) | ||
|
|
||
| server := &http.Server{ | ||
| Addr: r.cfg.MetricServer.GetAddress(), | ||
| Handler: mux, | ||
| ReadHeaderTimeout: readHeaderTimeout, | ||
| } | ||
|
|
||
| logger.Info().Msgf("starting metrics server on %s", server.Addr) | ||
|
|
||
| errCh := make(chan error) | ||
| go func() { | ||
| errCh <- server.ListenAndServe() | ||
| }() | ||
|
|
||
| select { | ||
| case err := <-errCh: | ||
| return err | ||
| case <-ctx.Done(): | ||
| case <-r.stop: | ||
| } | ||
|
|
||
| // shutdown the metrics server when either the context is done or when reminder is stopped | ||
| shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 5*time.Second) | ||
| defer shutdownRelease() | ||
|
|
||
| logger.Info().Msg("shutting down metrics server") | ||
|
|
||
| if err := mp.Shutdown(shutdownCtx); err != nil { | ||
| logger.Err(err).Msg("error shutting down metrics provider") | ||
| } | ||
|
|
||
| return server.Shutdown(shutdownCtx) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters