Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support datadog Autodiscovery v2 api annotation #1199

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

feat: support datadog Autodiscovery v2 api annotation #1199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f926708
add new datadog autorecovery version field
ericsyh Dec 30, 2024
5da7dc5
support on bookie
ericsyh Dec 30, 2024
10b4dd4
support on broker
ericsyh Dec 30, 2024
f6e289b
support on prometheus
ericsyh Dec 30, 2024
5f004b4
support on proxy
ericsyh Dec 30, 2024
de81988
support on zk
ericsyh Dec 30, 2024
8e518c5
update
ericsyh Dec 30, 2024
c21d3c9
apply on sn-platform
ericsyh Dec 30, 2024
ad1114f
fix lint
ericsyh Dec 30, 2024
fb4890d
fix lint
ericsyh Dec 30, 2024
76359e3
fix metrics type
ericsyh Dec 31, 2024
8bcdb2c
update the dd v1 template
ericsyh Jan 2, 2025
731cbbe
support on sn-platform chart
ericsyh Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 29 additions & 24 deletions charts/sn-platform-slim/templates/bookkeeper/_autorecovery.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,49 +20,54 @@ ${HOSTNAME}.{{ template "pulsar.autorecovery.service" . }}.{{ template "pulsar.n
{{/*Define autorecovery datadog annotation*/}}
{{- define "pulsar.autorecovery.datadog.annotation" -}}
{{- if .Values.datadog.components.autorecovery.enabled }}
{{- if eq .Values.datadog.adVersion "v1" }}
ad.datadoghq.com/{{ template "pulsar.autorecovery.podName" . }}.check_names: |
["openmetrics"]
ad.datadoghq.com/{{ template "pulsar.autorecovery.podName" . }}.init_configs: |
[{}]
ad.datadoghq.com/{{ template "pulsar.autorecovery.podName" . }}.instances: |
[
{
"prometheus_url": "http://%%host%%:{{ .Values.autorecovery.ports.http }}/metrics",
"openmetrics_endpoint": "http://%%host%%:{{ .Values.autorecovery.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"enable_health_service_check": true,
"timeout": 1000,
"metrics": {{ .Values.datadog.components.autorecovery.metrics }},
"health_service_check": true,
"prometheus_timeout": 1000,
"max_returned_metrics": 1000000,
"type_overrides": {
"jvm_memory_bytes_used": "gauge",
"jvm_memory_bytes_committed": "gauge",
"jvm_memory_bytes_max": "gauge",
"jvm_memory_bytes_init": "gauge",
"jvm_memory_pool_bytes_used": "gauge",
"jvm_memory_pool_bytes_committed": "gauge",
"jvm_memory_pool_bytes_max": "gauge",
"jvm_memory_pool_bytes_init": "gauge",
"jvm_memory_direct_bytes_used": "gauge",
"jvm_threads_current": "gauge",
"jvm_threads_daemon": "gauge",
"jvm_threads_peak": "gauge",
"jvm_threads_started_total": "gauge",
"jvm_threads_deadlocked": "gauge",
"jvm_threads_deadlocked_monitor": "gauge",
"jvm_gc_collection_seconds_count": "gauge",
"jvm_gc_collection_seconds_sum": "gauge",
"jvm_memory_direct_bytes_max": "gauge"
},
"tags": [
"pulsar-autorecovery: {{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
]
}
]
{{- end }}
{{- if eq .Values.datadog.adVersion "v2" }}
ad.datadoghq.com/{{ template "pulsar.autorecovery.podName" . }}.checks: |
{
"openmetrics": {
"init_config": [{}],
"instances": [
{
"openmetrics_endpoint": "http://%%host%%:{{ .Values.autorecovery.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"metrics": {{ .Values.datadog.components.autorecovery.metrics }},
"enable_health_service_check": true,
"timeout": 1000,
"tags": [
"pulsar-autorecovery: {{ template "pulsar.fullname" . }}-{{ .Values.autorecovery.component }}"
]
}
]
}
}
{{- end }}
{{- end }}
{{- end }}

{{/*
Expand Down
53 changes: 29 additions & 24 deletions charts/sn-platform-slim/templates/bookkeeper/_bookkeeper.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,49 +170,54 @@ Define bookkeeper log volumes
{{/*Define bookkeeper datadog annotation*/}}
{{- define "pulsar.bookkeeper.datadog.annotation" -}}
{{- if .Values.datadog.components.bookkeeper.enabled }}
{{- if eq .Values.datadog.adVersion "v1" }}
ad.datadoghq.com/{{ template "pulsar.bookkeeper.podName" . }}.check_names: |
["openmetrics"]
ad.datadoghq.com/{{ template "pulsar.bookkeeper.podName" . }}.init_configs: |
[{}]
ad.datadoghq.com/{{ template "pulsar.bookkeeper.podName" . }}.instances: |
[
{
"prometheus_url": "http://%%host%%:{{ .Values.bookkeeper.ports.http }}/metrics",
"openmetrics_endpoint": "http://%%host%%:{{ .Values.bookkeeper.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"metrics": {{ .Values.datadog.components.bookkeeper.metrics }},
"health_service_check": true,
"prometheus_timeout": 1000,
"max_returned_metrics": 1000000,
"type_overrides": {
"jvm_memory_bytes_used": "gauge",
"jvm_memory_bytes_committed": "gauge",
"jvm_memory_bytes_max": "gauge",
"jvm_memory_bytes_init": "gauge",
"jvm_memory_pool_bytes_used": "gauge",
"jvm_memory_pool_bytes_committed": "gauge",
"jvm_memory_pool_bytes_max": "gauge",
"jvm_memory_pool_bytes_init": "gauge",
"jvm_memory_direct_bytes_used": "gauge",
"jvm_threads_current": "gauge",
"jvm_threads_daemon": "gauge",
"jvm_threads_peak": "gauge",
"jvm_threads_started_total": "gauge",
"jvm_threads_deadlocked": "gauge",
"jvm_threads_deadlocked_monitor": "gauge",
"jvm_gc_collection_seconds_count": "gauge",
"jvm_gc_collection_seconds_sum": "gauge",
"jvm_memory_direct_bytes_max": "gauge"
},
"enable_health_service_check": true,
"timeout": 1000,
"tags": [
"pulsar-bookie: {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
]
}
]
{{- end }}
{{- if eq .Values.datadog.adVersion "v2" }}
ad.datadoghq.com/{{ template "pulsar.bookkeeper.podName" . }}.checks: |
{
"openmetrics": {
"init_config": [{}],
"instances": [
{
"openmetrics_endpoint": "http://%%host%%:{{ .Values.bookkeeper.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"metrics": {{ .Values.datadog.components.bookkeeper.metrics }},
"enable_health_service_check": true,
"timeout": 1000,
"tags": [
"pulsar-bookie: {{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}"
]
}
]
}
}
{{- end }}
{{- end }}
{{- end }}

{{/*Define bookkeeper service account*/}}
Expand Down
186 changes: 14 additions & 172 deletions charts/sn-platform-slim/templates/broker/_broker.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,210 +172,52 @@ Define function worker config volume
{{/*Define broker datadog annotation*/}}
{{- define "pulsar.broker.datadog.annotation" -}}
{{- if .Values.datadog.components.broker.enabled }}
{{- if eq (.Values.datadog.components.broker.checkType | default "openmetrics") "openmetrics" }}
{{- if eq .Values.datadog.adVersion "v1" }}
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.check_names: |
["openmetrics"]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.init_configs: |
[{}]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.instances: |
[
{
"prometheus_url": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
"openmetrics_endpoint": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"metrics": {{ .Values.datadog.components.broker.metrics }},
"health_service_check": true,
"prometheus_timeout": 1000,
"max_returned_metrics": 1000000,
"type_overrides": {
"pulsar_topics_count": "gauge",
"pulsar_rate_in": "gauge",
"pulsar_rate_out": "gauge",
"pulsar_subscriptions_count": "gauge",
"pulsar_producers_count": "gauge",
"pulsar_consumers_count": "gauge",
"pulsar_throughput_in": "gauge",
"pulsar_throughput_out": "gauge",
"pulsar_storage_size": "gauge",
"pulsar_msg_backlog": "gauge",
"pulsar_storage_backlog_size": "gauge",
"pulsar_storage_offloaded_size": "gauge",
"pulsar_storage_write_latency_le_0_5": "gauge",
"pulsar_storage_write_latency_le_1": "gauge",
"pulsar_storage_write_latency_le_5": "gauge",
"pulsar_storage_write_latency_le_10": "gauge",
"pulsar_storage_write_latency_le_20": "gauge",
"pulsar_storage_write_latency_le_50": "gauge",
"pulsar_storage_write_latency_le_100": "gauge",
"pulsar_storage_write_latency_le_200": "gauge",
"pulsar_storage_write_latency_le_1000": "gauge",
"pulsar_storage_write_latency_overflow": "gauge",
"pulsar_entry_size_le_128": "gauge",
"pulsar_entry_size_le_512": "gauge",
"pulsar_entry_size_le_1_kb": "gauge",
"pulsar_entry_size_le_2_kb": "gauge",
"pulsar_entry_size_le_4_kb": "gauge",
"pulsar_entry_size_le_16_kb": "gauge",
"pulsar_entry_size_le_100_kb": "gauge",
"pulsar_entry_size_le_1_mb": "gauge",
"pulsar_entry_size_le_overflow": "gauge",
"pulsar_subscription_back_log": "gauge",
"pulsar_subscription_back_log_no_delayed": "gauge",
"pulsar_subscription_delayed": "gauge",
"pulsar_subscription_msg_rate_redeliver": "gauge",
"pulsar_subscription_unacked_messages": "gauge",
"pulsar_subscription_blocked_on_unacked_messages": "gauge",
"pulsar_subscription_msg_rate_out": "gauge",
"pulsar_subscription_msg_throughput_out": "gauge",
"pulsar_in_bytes_total": "counter",
"pulsar_in_messages_total": "counter",
"topic_load_times": "counter",
"jvm_memory_bytes_used": "gauge",
"jvm_memory_bytes_committed": "gauge",
"jvm_memory_bytes_max": "gauge",
"jvm_memory_bytes_init": "gauge",
"jvm_memory_pool_bytes_used": "gauge",
"jvm_memory_pool_bytes_committed": "gauge",
"jvm_memory_pool_bytes_max": "gauge",
"jvm_memory_pool_bytes_init": "gauge",
"jvm_classes_loaded": "gauge",
"jvm_classes_loaded_total": "counter",
"jvm_classes_unloaded_total": "counter",
"jvm_buffer_pool_used_bytes": "gauge",
"jvm_buffer_pool_capacity_bytes": "gauge",
"jvm_buffer_pool_used_buffers": "gauge",
"jvm_threads_current": "gauge",
"jvm_threads_daemon": "gauge",
"jvm_threads_peak": "gauge",
"jvm_threads_started_total": "counter",
"jvm_threads_deadlocked": "gauge",
"jvm_threads_deadlocked_monitor": "gauge",
"jvm_gc_collection_seconds_count": "gauge",
"jvm_gc_collection_seconds_sum": "gauge",
"jvm_memory_direct_bytes_max": "gauge"
},
"tags": [
"pulsar-broker: {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
]
}
]
{{- else if eq (.Values.datadog.components.broker.checkType | default "openmetrics") "native" }}
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.check_names: |
["pulsar"]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.init_configs: |
[{}]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.instances: |
[
{
"openmetrics_endpoint": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
"enable_health_service_check": true,
"timeout": 300,
"timeout": 1000,
"tags": [
"pulsar-broker: {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
]
}
]
{{- else if eq (.Values.datadog.components.broker.checkType | default "openmetrics") "both" }}
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.check_names: |
["openmetrics", "pulsar"]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.init_configs: |
[{}, {}]
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.instances: |
[
{{- end }}
{{- if eq .Values.datadog.adVersion "v2" }}
ad.datadoghq.com/{{ template "pulsar.broker.podName" . }}.checks: |
{
"openmetrics": {
"init_config": [{}],
"instances": [
{
"prometheus_url": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
"openmetrics_endpoint": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
{{ if .Values.datadog.namespace -}}
"namespace": "{{ .Values.datadog.namespace }}",
{{ else -}}
"namespace": "{{ template "pulsar.namespace" . }}",
{{ end -}}
"metrics": {{ .Values.datadog.components.broker.metrics }},
"health_service_check": true,
"prometheus_timeout": 1000,
"max_returned_metrics": 1000000,
"type_overrides": {
"pulsar_topics_count": "gauge",
"pulsar_rate_in": "gauge",
"pulsar_rate_out": "gauge",
"pulsar_subscriptions_count": "gauge",
"pulsar_producers_count": "gauge",
"pulsar_consumers_count": "gauge",
"pulsar_throughput_in": "gauge",
"pulsar_throughput_out": "gauge",
"pulsar_storage_size": "gauge",
"pulsar_msg_backlog": "gauge",
"pulsar_storage_backlog_size": "gauge",
"pulsar_storage_offloaded_size": "gauge",
"pulsar_storage_write_latency_le_0_5": "gauge",
"pulsar_storage_write_latency_le_1": "gauge",
"pulsar_storage_write_latency_le_5": "gauge",
"pulsar_storage_write_latency_le_10": "gauge",
"pulsar_storage_write_latency_le_20": "gauge",
"pulsar_storage_write_latency_le_50": "gauge",
"pulsar_storage_write_latency_le_100": "gauge",
"pulsar_storage_write_latency_le_200": "gauge",
"pulsar_storage_write_latency_le_1000": "gauge",
"pulsar_storage_write_latency_overflow": "gauge",
"pulsar_entry_size_le_128": "gauge",
"pulsar_entry_size_le_512": "gauge",
"pulsar_entry_size_le_1_kb": "gauge",
"pulsar_entry_size_le_2_kb": "gauge",
"pulsar_entry_size_le_4_kb": "gauge",
"pulsar_entry_size_le_16_kb": "gauge",
"pulsar_entry_size_le_100_kb": "gauge",
"pulsar_entry_size_le_1_mb": "gauge",
"pulsar_entry_size_le_overflow": "gauge",
"pulsar_subscription_back_log": "gauge",
"pulsar_subscription_back_log_no_delayed": "gauge",
"pulsar_subscription_delayed": "gauge",
"pulsar_subscription_msg_rate_redeliver": "gauge",
"pulsar_subscription_unacked_messages": "gauge",
"pulsar_subscription_blocked_on_unacked_messages": "gauge",
"pulsar_subscription_msg_rate_out": "gauge",
"pulsar_subscription_msg_throughput_out": "gauge",
"pulsar_in_bytes_total": "counter",
"pulsar_in_messages_total": "counter",
"topic_load_times": "counter",
"jvm_memory_bytes_used": "gauge",
"jvm_memory_bytes_committed": "gauge",
"jvm_memory_bytes_max": "gauge",
"jvm_memory_bytes_init": "gauge",
"jvm_memory_pool_bytes_used": "gauge",
"jvm_memory_pool_bytes_committed": "gauge",
"jvm_memory_pool_bytes_max": "gauge",
"jvm_memory_pool_bytes_init": "gauge",
"jvm_classes_loaded": "gauge",
"jvm_classes_loaded_total": "counter",
"jvm_classes_unloaded_total": "counter",
"jvm_buffer_pool_used_bytes": "gauge",
"jvm_buffer_pool_capacity_bytes": "gauge",
"jvm_buffer_pool_used_buffers": "gauge",
"jvm_threads_current": "gauge",
"jvm_threads_daemon": "gauge",
"jvm_threads_peak": "gauge",
"jvm_threads_started_total": "counter",
"jvm_threads_deadlocked": "gauge",
"jvm_threads_deadlocked_monitor": "gauge",
"jvm_gc_collection_seconds_count": "gauge",
"jvm_gc_collection_seconds_sum": "gauge",
"jvm_memory_direct_bytes_max": "gauge"
},
"tags": [
"pulsar-broker: {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
]
},
{
"openmetrics_endpoint": "http://%%host%%:{{ .Values.broker.ports.http }}/metrics",
"enable_health_service_check": true,
"timeout": 300,
"timeout": 1000,
"tags": [
"pulsar-broker: {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
]
}
]
}
}
{{- end }}
{{- end }}
{{- end }}
Expand Down
Loading
Loading