feat: OB-36454 update values to new conventions and add and update tests

.gitignore

@@ -17,3 +17,5 @@ tfplan
 venv
 __pycache__
 .pytest_*
+
+local*

charts/agent/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: agent
 description: Chart to install K8s collection stack based on Observe Agent
 type: application
-version: 0.18.0
+version: 0.19.0
 appVersion: "1.1.0"
 dependencies:
   - name: opentelemetry-collector

charts/agent/README.md

@@ -1,6 +1,6 @@
 # agent
 
-![Version: 0.18.0](https://img.shields.io/badge/Version-0.18.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square)
+![Version: 0.19.0](https://img.shields.io/badge/Version-0.19.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square)
 
 > [!CAUTION]
 > This chart is under active development and is not meant to be installed yet.
@@ -33,6 +33,13 @@ Chart to install K8s collection stack based on Observe Agent
 | agent.config.global.service.telemetry.logging_level | string | `"WARN"` |  |
 | agent.config.global.service.telemetry.metrics_level | string | `"normal"` |  |
 | agent.selfMonitor.enabled | bool | `true` |  |
+| application.prometheusScrape.enabled | bool | `false` |  |
+| application.prometheusScrape.interval | string | `"10s"` |  |
+| application.prometheusScrape.metricDropRegex | string | `".*bucket"` |  |
+| application.prometheusScrape.metricKeepRegex | string | `"(.*)"` |  |
+| application.prometheusScrape.namespaceDropRegex | string | `"(.*istio.*|.*ingress.*|kube-system)"` |  |
+| application.prometheusScrape.namespaceKeepRegex | string | `"(.*)"` |  |
+| application.prometheusScrape.portKeepRegex | string | `".*metrics"` |  |
 | cluster-events.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"observeinc.com/unschedulable"` |  |
 | cluster-events.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"DoesNotExist"` |  |
 | cluster-events.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[1].key | string | `"kubernetes.io/os"` |  |
@@ -89,6 +96,7 @@ Chart to install K8s collection stack based on Observe Agent
 | cluster-events.podAnnotations.observe_monitor_port | string | `"8888"` |  |
 | cluster-events.podAnnotations.observe_monitor_purpose | string | `"observecollection"` |  |
 | cluster-events.podAnnotations.observe_monitor_scrape | string | `"true"` |  |
+| cluster-events.podAnnotations.observeinc_com_scrape | string | `"false"` |  |
 | cluster-events.ports.metrics.containerPort | int | `8888` |  |
 | cluster-events.ports.metrics.enabled | bool | `true` |  |
 | cluster-events.ports.metrics.protocol | string | `"TCP"` |  |
@@ -152,6 +160,7 @@ Chart to install K8s collection stack based on Observe Agent
 | cluster-metrics.podAnnotations.observe_monitor_port | string | `"8888"` |  |
 | cluster-metrics.podAnnotations.observe_monitor_purpose | string | `"observecollection"` |  |
 | cluster-metrics.podAnnotations.observe_monitor_scrape | string | `"true"` |  |
+| cluster-metrics.podAnnotations.observeinc_com_scrape | string | `"false"` |  |
 | cluster-metrics.ports.metrics.containerPort | int | `8888` |  |
 | cluster-metrics.ports.metrics.enabled | bool | `true` |  |
 | cluster-metrics.ports.metrics.protocol | string | `"TCP"` |  |
@@ -166,14 +175,6 @@ Chart to install K8s collection stack based on Observe Agent
 | cluster.events.enabled | bool | `true` |  |
 | cluster.events.pullInterval | string | `"20m"` |  |
 | cluster.metrics.enabled | bool | `true` |  |
-| cluster.metrics.pod.action | string | `"keep"` |  |
-| cluster.metrics.pod.enabled | bool | `false` |  |
-| cluster.metrics.pod.interval | string | `"10s"` |  |
-| cluster.metrics.pod.metric_drop_regex | string | `".*bucket"` |  |
-| cluster.metrics.pod.metric_keep_regex | string | `"(.*)"` |  |
-| cluster.metrics.pod.namespace_drop_regex | string | `"(.*istio.*|.*ingress.*|kube-system|observe)"` |  |
-| cluster.metrics.pod.namespace_keep_regex | string | `"(.*)"` |  |
-| cluster.metrics.pod.port_keep_regex | string | `".*metrics"` |  |
 | cluster.name | string | `"observe-agent-monitored-cluster"` |  |
 | cluster.namespaceOverride.value | string | `"observe"` |  |
 | cluster.uidOverride.value | string | `""` |  |
@@ -229,6 +230,7 @@ Chart to install K8s collection stack based on Observe Agent
 | monitor.podAnnotations.observe_monitor_port | string | `"8888"` |  |
 | monitor.podAnnotations.observe_monitor_purpose | string | `"observecollection"` |  |
 | monitor.podAnnotations.observe_monitor_scrape | string | `"false"` |  |
+| monitor.podAnnotations.observeinc_com_scrape | string | `"false"` |  |
 | monitor.ports.metrics.containerPort | int | `8888` |  |
 | monitor.ports.metrics.enabled | bool | `true` |  |
 | monitor.ports.metrics.protocol | string | `"TCP"` |  |
@@ -315,6 +317,7 @@ Chart to install K8s collection stack based on Observe Agent
 | node-logs-metrics.podAnnotations.observe_monitor_port | string | `"8888"` |  |
 | node-logs-metrics.podAnnotations.observe_monitor_purpose | string | `"observecollection"` |  |
 | node-logs-metrics.podAnnotations.observe_monitor_scrape | string | `"true"` |  |
+| node-logs-metrics.podAnnotations.observeinc_com_scrape | string | `"false"` |  |
 | node-logs-metrics.ports.metrics.containerPort | int | `8888` |  |
 | node-logs-metrics.ports.metrics.enabled | bool | `true` |  |
 | node-logs-metrics.ports.metrics.protocol | string | `"TCP"` |  |

charts/agent/templates/_cluster-metrics-config.tpl

@@ -23,33 +23,34 @@ receivers:
     metrics:
       k8s.node.condition:
         enabled: true
-  {{- if .Values.cluster.metrics.pod.enabled }}
+
+  {{- if .Values.application.prometheusScrape.enabled }}
   prometheus/pod_metrics:
       config:
         scrape_configs:
         - job_name: pod-metrics
-          scrape_interval: {{.Values.cluster.metrics.pod.interval}}
+          scrape_interval: {{.Values.application.prometheusScrape.interval}}
           honor_labels: true
           kubernetes_sd_configs:
           - role: pod
           relabel_configs:
-          # this is defaulted to keep but if set to drop then pod metrics will not be collected
-          - action: {{.Values.cluster.metrics.pod.action}}
+          # this is defaulted to keep so we start with everything
+          - action: keep
 
           # Drop anything matching the configured namespace.
           - action: 'drop'
             source_labels: ['__meta_kubernetes_namespace']
-            regex: {{.Values.cluster.metrics.pod.namespace_drop_regex}}
+            regex: {{.Values.application.prometheusScrape.namespaceDropRegex}}
 
           # Drop anything not matching the configured namespace.
           - action: 'keep'
             source_labels: ['__meta_kubernetes_namespace']
-            regex: {{.Values.cluster.metrics.pod.namespace_keep_regex}}
+            regex: {{.Values.application.prometheusScrape.namespaceKeepRegex}}
 
           # Drop endpoints without one of: a port name suffixed with the configured regex, or an explicit prometheus port annotation.
           - action: 'keep'
             source_labels: ['__meta_kubernetes_pod_container_port_name', '__meta_kubernetes_pod_annotation_prometheus_io_port']
-            regex: '({{.Values.cluster.metrics.pod.port_keep_regex}};|.*;\d+)'
+            regex: '({{.Values.application.prometheusScrape.portKeepRegex}};|.*;\d+)'
 
           # Drop pods with phase Succeeded or Failed.
           - action: 'drop'
@@ -109,8 +110,9 @@ receivers:
           - source_labels: [__address__, __meta_kubernetes_pod_annotation_observeinc_com_port]
             action: replace
             regex: ([^:]+)(?::\d+)?;(\d+)
-            replacement: $$1:$$2
+            replacement: '$1:$2'
             target_label: __address__
+          ################################################################
 
           # Maps all Kubernetes pod labels to Prometheus labels with the prefix removed (e.g., __meta_kubernetes_pod_label_app becomes app).
           - action: labelmap
@@ -125,6 +127,18 @@ receivers:
           - source_labels: [__meta_kubernetes_pod_name]
             action: replace
             target_label: kubernetes_pod_name
+
+          metric_relabel_configs:
+            - action: drop
+              regex: {{.Values.application.prometheusScrape.metricDropRegex}}
+              source_labels:
+                - __name__
+            - action: keep
+              regex: {{.Values.application.prometheusScrape.metricKeepRegex}}
+              source_labels:
+                - __name__
+
+
   {{ end }}
 
 processors:
@@ -155,7 +169,7 @@ service:
         receivers: [k8s_cluster]
         processors: [memory_limiter, batch, k8sattributes, attributes/observe_common, attributes/debug_source_cluster_metrics]
         exporters: [prometheusremotewrite, debug/override]
-      {{- if .Values.cluster.metrics.pod.enabled }}
+      {{- if .Values.application.prometheusScrape.enabled }}
       metrics/pod_metrics:
         receivers: [prometheus/pod_metrics]
         processors: [memory_limiter, batch, k8sattributes, attributes/observe_common, attributes/debug_source_pod_metrics]

charts/agent/values.yaml

@@ -21,23 +21,32 @@ cluster:
     enabled: true
   metrics:
     enabled: true
-    pod:
-      enabled: false
-      action: keep
-      interval: 10s
-      # add whatever namespace the collection is deployed to
-      namespace_drop_regex: (.*istio.*|.*ingress.*|kube-system|observe)
-      namespace_keep_regex: (.*)
-      port_keep_regex: .*metrics
-      metric_drop_regex: .*bucket
-      metric_keep_regex: (.*)
 
   namespaceOverride:
     # ! This needs to have same value as namespaceOverride in deployments and daemonsets below
     value: observe
   uidOverride:
     value: ""
 
+application:
+  # use this option to scrape prometheus metrics from pods
+  # cluster.metrics.enabled must be set to true
+  prometheusScrape:
+    enabled: false
+    interval: 10s
+    # namespaces to exclude from scraping
+    # drop is processed first so all namespaces that match regex will be dropped
+    namespaceDropRegex: (.*istio.*|.*ingress.*|kube-system)
+    # namespaces to explicity include for scraping - can use or (ns1|ns2)
+    # keep is processed after drop so only remaining namespaces that match regex will be kept
+    namespaceKeepRegex: (.*)
+    # port names to scrape from - can use or .*metrics|otherportname
+    portKeepRegex: .*metrics
+    # metrics to drop
+    metricDropRegex: .*bucket
+    # metrics to keep
+    metricKeepRegex: (.*)
+
 node:
   enabled: true
   metrics:
@@ -126,6 +135,8 @@ cluster-events:
 
 
   podAnnotations: {
+    # This stops optional prometheus scrape config from picking up these pods
+    observeinc_com_scrape: 'false',
     observe_monitor_purpose: observecollection,
     observe_monitor_scrape: 'true',
     observe_monitor_path: '/metrics',
@@ -259,6 +270,8 @@ cluster-metrics:
     egressRules: [{}]
 
   podAnnotations: {
+    # This stops optional prometheus scrape config from picking up these pods
+    observeinc_com_scrape: 'false',
     observe_monitor_purpose: observecollection,
     observe_monitor_scrape: 'true',
     observe_monitor_path: '/metrics',
@@ -385,6 +398,8 @@ node-logs-metrics:
     egressRules: [{}]
 
   podAnnotations: {
+    # This stops optional prometheus scrape config from picking up these pods
+    observeinc_com_scrape: 'false',
     observe_monitor_purpose: observecollection,
     observe_monitor_scrape: 'true',
     observe_monitor_path: '/metrics',
@@ -544,6 +559,8 @@ monitor:
     egressRules: [{}]
 
   podAnnotations: {
+    # This stops optional prometheus scrape config from picking up these pods
+    observeinc_com_scrape: 'false',
     observe_monitor_purpose: observecollection,
     observe_monitor_scrape: 'false',
     observe_monitor_path: '/metrics',

examples/agent/pod_metrics/README.md

@@ -13,16 +13,28 @@ kubectl apply -f sample-pod.yaml
 
 kubectl apply -f sample-pod-no.yaml
 
+kubectl apply -f sample-pod-prometheus-labels.yaml
+
+kubectl apply -f sample-pod-prometheus-labels-no.yaml
+
 ```
 
 ## Port forward
+You can curl a given service to increment counters from local machine by port forwarding.
+
+These sevices are defined in sample-pod files
 ```
 kubectl port-forward service/prometheus-example-app-service 8080:8080
 
 kubectl port-forward service/prometheus-example-app-no-service 8080:8080
+
+kubectl port-forward service/prometheus-example-app-promlabel-service 8080:8080
+
+kubectl port-forward service/prometheus-example-app-promlabel-no-service 8080:8080
 ```
 
 ## Call service locally
+If port forwarded you can use these curl commands
 ```
 # Get metrics
 curl localhost:8080/metrics
@@ -40,14 +52,35 @@ curl localhost:8080/err
 kubectl delete -f sample-pod.yaml
 
 kubectl delete -f sample-pod-no.yaml
+
+kubectl delete -f sample-pod-prometheus-labels.yaml
+
+kubectl delete -f sample-pod-prometheus-labels-no.yaml
 ```
 
 ## Deploy k8s monitoring with pod metrics collection enabled
 The sample-pod.yaml is deployed in the default namespace unless you change the provided command.
 
-It has annotations to change the metrics port to 8080 (observeinc_com_port: '8080') which will tell the scrape config to not use the default port of 8888.
+It has annotations to change the metrics port, metrics path and that this pod should be scraped:
+```
+      annotations:
+        observeinc_com_scrape: 'true'
+        observeinc_com_path: '/metrics'
+        observeinc_com_port: '8080'
+```
+
+The sample-pod-no.yaml sets observeinc_com_scrape: 'false' so no metrics should be collected from this pod.
+
+Pods created by sample-pod-prometheus-labels and sample-pod-prometheus-labels-no yaml files use prometheus annotations to accomplish same functions as pods above:
+```
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/path: '/metrics'
+        prometheus.io/port: '8080'
+```
+
+The pod-metrics-values.yaml file sets the namespace to scrape metrics from to default with namespaceKeepRegex and adds the web port name as valid with portKeepRegex.
 
-The pod-metrics-values.yaml file sets the namespace to scrape metrics from to default with namespace_keep_regex and adds the web port name as valid with port_keep_regex.
 
 ```
 helm install pod-metrics-example -n k8smonitoring \

examples/agent/pod_metrics/pod-metrics-values.yaml

@@ -5,15 +5,28 @@ cluster:
     enabled: true
   metrics:
     enabled: true
-    pod:
-      enabled: true
-      namespace_keep_regex: (default)
-      metric_drop_regex: ""
-      port_keep_regex: .*metrics|web
 
   namespaceOverride:
     value: k8smonitoring
 
+application:
+  # use this option to scrape prometheus metrics from pods
+  prometheusScrape:
+    enabled: true
+    interval: 10s
+    # namespaces to exclude from scraping
+    namespaceDropRegex: (.*istio.*|.*ingress.*|kube-system)
+    # namespaces to explicity include for scraping - can use or (ns1|ns2)
+    namespaceKeepRegex: (default)
+    # port names to scrape from - can use or .*metrics|otherportname
+    portKeepRegex: .*metrics|web
+    # metrics to drop
+    metricDropRegex: .*bucket
+    # metrics to keep
+    metricKeepRegex: (.*)
+
+
+
 containers:
   logs:
     enabled: true