Merge pull request #614 from redpanda-data/ae/568b

feat: add prometheus for testing

.github/workflows/nightly.yaml

@@ -120,6 +120,16 @@ jobs:
             --create-namespace --version v1.11.0 jetstack/cert-manager \
             --set installCRDs=true --wait --wait-for-jobs
 
+      - name: install prometheus-operator
+        run: |
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &&
+          helm install prometheus prometheus-community/kube-prometheus-stack \
+            --namespace prometheus --create-namespace \
+            --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
+            --set nodeExporter.enabled=false \
+            --set grafana.enabled=false --set kubeStateMetrics.enabled=false --set alertmanager.enabled=false \
+            --wait --wait-for-jobs
+
       - name: install metallb
         run: |
           helm repo add metallb https://metallb.github.io/metallb &&

.github/workflows/nightly_redpanda_tip.yaml

@@ -105,6 +105,16 @@ jobs:
             --create-namespace --version v1.11.0 jetstack/cert-manager \
             --set installCRDs=true --wait --wait-for-jobs
 
+      - name: install prometheus-operator
+        run: |
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &&
+          helm install prometheus prometheus-community/kube-prometheus-stack \
+            --namespace prometheus --create-namespace \
+            --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
+            --set nodeExporter.enabled=false \
+            --set grafana.enabled=false --set kubeStateMetrics.enabled=false --set alertmanager.enabled=false \
+            --wait --wait-for-jobs
+
       - name: install metallb
         run: |
           helm repo add metallb https://metallb.github.io/metallb &&

.github/workflows/pull_requests.yaml

@@ -170,6 +170,17 @@ jobs:
             --create-namespace --version v1.11.0 jetstack/cert-manager \
             --set installCRDs=true --wait --wait-for-jobs
 
+      - name: install prometheus-operator
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &&
+          helm install prometheus prometheus-community/kube-prometheus-stack \
+            --namespace prometheus --create-namespace \
+            --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
+            --set nodeExporter.enabled=false \
+            --set grafana.enabled=false --set kubeStateMetrics.enabled=false --set alertmanager.enabled=false \
+            --wait --wait-for-jobs
+
       - name: install metallb
         if: steps.list-changed.outputs.changed == 'true'
         run: |

.github/workflows/pull_requests_from_origin.yaml

@@ -117,6 +117,17 @@ jobs:
             --create-namespace --version v1.11.0 jetstack/cert-manager \
             --set installCRDs=true --wait --wait-for-jobs
 
+      - name: install prometheus-operator
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &&
+          helm install prometheus prometheus-community/kube-prometheus-stack \
+            --namespace prometheus --create-namespace \
+            --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
+            --set nodeExporter.enabled=false \
+            --set grafana.enabled=false --set kubeStateMetrics.enabled=false --set alertmanager.enabled=false \
+            --wait --wait-for-jobs
+
       - name: install metallb
         if: steps.list-changed.outputs.changed == 'true'
         run: |

.gitignore

@@ -24,6 +24,7 @@ Chart.lock
 charts/redpanda/templates/external-service.yaml
 charts/redpanda/templates/external-tls-secret.yaml
 charts/redpanda/templates/some-users-updated.yaml
+charts/redpanda/templates/prometheus.yaml
 charts/kminion/templates/redpanda-tls.yaml
 charts/connectors/templates/hidden-only-for-ci/
 

charts/redpanda/Chart.yaml

@@ -23,7 +23,7 @@ type: application
 # The chart version and the app version are not the same and will not track
 # together. The chart version is a semver representation of changes to this
 # chart.
-version: 5.0.6
+version: 5.0.7
 
 # The app version is the default version of Redpanda to install.
 appVersion: v23.2.3

charts/redpanda/ci/14-prometheus-no-tls-values.yaml

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+tls:
+  enabled: false
+
+monitoring:
+  enabled: true

charts/redpanda/ci/15-prometheus-tls-values.yaml

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+tls:
+  enabled: true
+
+monitoring:
+  enabled: true

charts/redpanda/templates/servicemonitor.yaml

@@ -1,4 +1,3 @@
-{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.monitoring.enabled }}
 {{/*
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
@@ -15,6 +14,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.monitoring.enabled }}
 ---
 # This servicemonitor is used by Prometheus Operator to scrape the metrics
 apiVersion: monitoring.coreos.com/v1

charts/redpanda/templates/tests/test-prometheus-targets.yaml

@@ -0,0 +1,72 @@
+{{/*
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  */}}
+{{- if .Values.monitoring.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "redpanda.fullname" . }}-test-prometheus-targets"
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+  {{- with include "full.labels" . }}
+  {{- . | nindent 4 }}
+  {{- end }}
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  restartPolicy: Never
+  securityContext:
+    runAsUser: 65535
+    runAsGroup: 65535
+  {{- with .Values.imagePullSecrets }}
+  imagePullSecrets: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  containers:
+    - name: {{ template "redpanda.name" . }}
+      image: mintel/docker-alpine-bash-curl-jq:latest
+      command: [ "/bin/bash", "-c" ]
+      args:
+        - |
+          set -xe
+          # added since it takes 20s to start
+          sleep 20
+          curl_prometheus_retry() {
+            local retries="$1"
+
+            # Run the command, and save the exit code
+            # from: https://prometheus.io/docs/prometheus/latest/querying/api/
+            local RESULT=$( curl --fail http://prometheus-operated.prometheus.svc.cluster.local:9090/api/v1/targets?scrapePool=serviceMonitor/{{ .Release.Namespace }}/{{ .Release.Namespace }}/0 | jq . | grep '"health": "up"' | wc -l  )
+            local exit_code=$?
+
+            # If the exit code is non-zero (i.e. command failed), and we have not
+            # reached the maximum number of retries, run the command again
+            if [[ ( $exit_code -ne 0 || $RESULT -ne {{ .Values.statefulset.replicas }}  ) && $retries -gt 0 ]]; then
+              sleep 2
+              echo $( curl_prometheus_retry $(($retries - 1)) )
+              return
+            fi
+          
+            echo $RESULT
+          }
+          RESULT=$(curl_prometheus_retry 20)
+          set +x
+          if [ $RESULT != {{ .Values.statefulset.replicas }} ]; then
+              curl --fail http://prometheus-operated.prometheus.svc.cluster.local:9090/api/v1/targets?scrapePool=serviceMonitor/{{ .Release.Namespace }}/{{ .Release.Namespace }}/0 | jq .
+              echo "the number of targets unexpected; got ${RESULT} targets 'up', but was expecting {{ .Values.statefulset.replicas }} "
+              exit 1
+          fi
+{{- end }}