Merge pull request #54 from intergral/alert_rules

Alert rules
intergral · Aug 13, 2024 · 9f20ea6 · 9f20ea6
2 parents a8ef953 + 5656b0e
commit 9f20ea6
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 14 deletions.
diff --git a/charts/deep-distributed/Chart.yaml b/charts/deep-distributed/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.9
+version: 1.0.10
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/deep-distributed/values.yaml b/charts/deep-distributed/values.yaml
@@ -1021,6 +1021,8 @@ monitoring:
       app_name: '{{ include "deep.fullname" . }}'
     # Additional annotations to add to the alerts
     alertAnnotations: {}
+    # Additional rules to add to all alert rules
+    additionalRuleExpr: ""
     # The rules to add
     rules:
       - alert: Unhealthy Ring Member
@@ -1032,47 +1034,47 @@ monitoring:
           runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/
           summary: One or more ring members are unhealthy
       - alert: Required Ring Member Missing
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name=~"ingester|tracepoint"} == 0
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name=~"ingester|tracepoint"} == 0
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Tracepoint ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="tracepoint"} != {{ .Values.tracepoint.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="tracepoint"} != {{ .Values.tracepoint.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Ingester ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="ingester"} != {{ .Values.ingester.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="ingester"} != {{ .Values.ingester.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Compactor ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="compactor"} != {{ .Values.compactor.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="compactor"} != {{ .Values.compactor.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Metrics Generator ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.distributor.replicas }} {{else}}0{{end}}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.distributor.replicas }} {{else}}0{{end}}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Rapid increase in blocks
-        expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" .}}"}[24h]) > 30
+        expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}}[24h]) > 30
         for: 48h
         labels:
           severity: warning

diff --git a/charts/deep/Chart.yaml b/charts/deep/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.23
+version: 1.0.24
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/deep/values.yaml b/charts/deep/values.yaml
@@ -466,6 +466,8 @@ monitoring:
       app_name: '{{ include "deep.fullname" . }}'
     # Additional annotations to add to the alerts
     alertAnnotations: {}
+    # Additional rules to add to all alert rules
+    additionalRuleExpr: ""
     # The rules to add
     rules:
       - alert: Unhealthy Ring Member
@@ -477,47 +479,47 @@ monitoring:
           runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/
           summary: One or more ring members are unhealthy
       - alert: Required Ring Member Missing
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name=~"ingester|tracepoint"} == 0
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name=~"ingester|tracepoint"} == 0
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Tracepoint ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="tracepoint"} != {{ .Values.deep.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="tracepoint"} != {{ .Values.deep.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Ingester ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="ingester"} != {{ .Values.deep.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="ingester"} != {{ .Values.deep.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Compactor ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="compactor"} != {{ .Values.deep.replicas }}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="compactor"} != {{ .Values.deep.replicas }}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Metrics Generator ring missing member
-        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}", name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.deep.replicas }} {{else}}0{{end}}
+        expr: deep_ring_members{state="ACTIVE", cluster="{{ include "deep.clusterName" .}}"{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}, name="metrics-generator"} != {{ if .Values.metricsGenerator.enabled }} {{.Values.deep.replicas }} {{else}}0{{end}}
         labels:
           severity: critical
         annotations:
           description: "{{ $labels.service }} has {{ $value }} active nodes in ring {{ $labels.name }}."
           runbook_url: https://intergral.github.io/deep/runbooks/missing_ring_node/
           summary: One or more ring has missing required nodes
       - alert: Rapid increase in blocks
-        expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" .}}"}[24h]) > 30
+        expr: delta(deep_db_blocklist_length{cluster="{{ include "deep.clusterName" .}}{{ tpl .Values.monitoring.rules.additionalRuleExpr $ }}"}[24h]) > 30
         for: 48h
         labels:
           severity: warning