[BREAKING CHANGE] Switch protocol for internal traces from Jaeger to …

…OTLP/HTTP Exporting internal traces (traces generated by Tempo and the gateway) via the Jaeger/Thrift protocol and the `.spec.observability.tracing.jaeger_agent_endpoint` setting is deprecated. Please migrate to the OTLP/HTTP protocol and the new `.spec.observability.tracing.otlp_http_endpoint` setting. Signed-off-by: Andreas Gerstmayr <[email protected]>
grafana · Jan 24, 2025 · 190fbd0 · 190fbd0
1 parent 988f89a
commit 190fbd0
Show file tree

Hide file tree

Showing 16 changed files with 127 additions and 96 deletions.
diff --git a/.chloggen/internal_tracing_protocol.yaml b/.chloggen/internal_tracing_protocol.yaml
@@ -0,0 +1,18 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Switch protocol for internal traces from Jaeger to OTLP/HTTP
+
+# One or more tracking issues related to the change
+issues: []
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Exporting internal traces (traces generated by Tempo and the gateway) via the Jaeger/Thrift protocol and the `.spec.observability.tracing.jaeger_agent_endpoint` setting is deprecated.
+  Please migrate to the OTLP/HTTP protocol and the new `.spec.observability.tracing.otlp_http_endpoint` setting.
diff --git a/api/tempo/v1alpha1/tempostack_types.go b/api/tempo/v1alpha1/tempostack_types.go
@@ -183,7 +183,15 @@ type TracingConfigSpec struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Sampling Fraction"
 	SamplingFraction string `json:"sampling_fraction,omitempty"`
 
-	// JaegerAgentEndpoint defines the jaeger endpoint data gets send to.
+	// OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send to.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="localhost:4318"
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OTLP/HTTP Endpoint"
+	OTLPHTTPEndpoint string `json:"otlp_http_endpoint,omitempty"`
+
+	// DEPRECATED. JaegerAgentEndpoint defines the jaeger endpoint data gets send to.
 	//
 	// +optional
 	// +kubebuilder:validation:Optional

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.2
-    createdAt: "2025-01-16T09:58:41Z"
+    createdAt: "2025-01-24T15:21:14Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -757,10 +757,10 @@ spec:
       - description: Tracing defines a config for operands.
         displayName: Tracing Config
         path: observability.tracing
-      - description: JaegerAgentEndpoint defines the jaeger endpoint data gets send
+      - description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send
           to.
-        displayName: Jaeger-Agent-Endpoint
-        path: observability.tracing.jaeger_agent_endpoint
+        displayName: OTLP/HTTP Endpoint
+        path: observability.tracing.otlp_http_endpoint
       - description: SamplingFraction defines the sampling ratio. Valid values are
           0 to 1.
         displayName: Sampling Fraction

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -291,9 +291,9 @@ spec:
                   tracing:
                     description: Tracing defines a config for operands.
                     properties:
-                      jaeger_agent_endpoint:
-                        default: localhost:6831
-                        description: JaegerAgentEndpoint defines the jaeger endpoint
+                      otlp_http_endpoint:
+                        default: localhost:4318
+                        description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint
                           data gets send to.
                         type: string
                       sampling_fraction:

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.2
-    createdAt: "2025-01-16T09:58:39Z"
+    createdAt: "2025-01-24T15:21:13Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -757,10 +757,10 @@ spec:
       - description: Tracing defines a config for operands.
         displayName: Tracing Config
         path: observability.tracing
-      - description: JaegerAgentEndpoint defines the jaeger endpoint data gets send
+      - description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send
           to.
-        displayName: Jaeger-Agent-Endpoint
-        path: observability.tracing.jaeger_agent_endpoint
+        displayName: OTLP/HTTP Endpoint
+        path: observability.tracing.otlp_http_endpoint
       - description: SamplingFraction defines the sampling ratio. Valid values are
           0 to 1.
         displayName: Sampling Fraction

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -291,9 +291,9 @@ spec:
                   tracing:
                     description: Tracing defines a config for operands.
                     properties:
-                      jaeger_agent_endpoint:
-                        default: localhost:6831
-                        description: JaegerAgentEndpoint defines the jaeger endpoint
+                      otlp_http_endpoint:
+                        default: localhost:4318
+                        description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint
                           data gets send to.
                         type: string
                       sampling_fraction:

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -289,7 +289,12 @@ spec:
                     properties:
                       jaeger_agent_endpoint:
                         default: localhost:6831
-                        description: JaegerAgentEndpoint defines the jaeger endpoint
+                        description: DEPRECATED. JaegerAgentEndpoint defines the jaeger
+                          endpoint data gets send to.
+                        type: string
+                      otlp_http_endpoint:
+                        default: localhost:4318
+                        description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint
                           data gets send to.
                         type: string
                       sampling_fraction:

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -686,10 +686,10 @@ spec:
       - description: Tracing defines a config for operands.
         displayName: Tracing Config
         path: observability.tracing
-      - description: JaegerAgentEndpoint defines the jaeger endpoint data gets send
+      - description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send
           to.
-        displayName: Jaeger-Agent-Endpoint
-        path: observability.tracing.jaeger_agent_endpoint
+        displayName: OTLP/HTTP Endpoint
+        path: observability.tracing.otlp_http_endpoint
       - description: SamplingFraction defines the sampling ratio. Valid values are
           0 to 1.
         displayName: Sampling Fraction

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -686,10 +686,10 @@ spec:
       - description: Tracing defines a config for operands.
         displayName: Tracing Config
         path: observability.tracing
-      - description: JaegerAgentEndpoint defines the jaeger endpoint data gets send
+      - description: OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send
           to.
-        displayName: Jaeger-Agent-Endpoint
-        path: observability.tracing.jaeger_agent_endpoint
+        displayName: OTLP/HTTP Endpoint
+        path: observability.tracing.otlp_http_endpoint
       - description: SamplingFraction defines the sampling ratio. Valid values are
           0 to 1.
         displayName: Sampling Fraction

diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml
@@ -53,7 +53,7 @@ spec:                                    # TempoStackSpec defines the desired st
       createPrometheusRules: false       # CreatePrometheusRules specifies if Prometheus rules for alerts should be created for Tempo components.
       createServiceMonitors: false       # CreateServiceMonitors specifies if ServiceMonitors should be created for Tempo components.
     tracing:                             # Tracing defines a config for operands.
-      jaeger_agent_endpoint: "localhost:6831" # JaegerAgentEndpoint defines the jaeger endpoint data gets send to.
+      otlp_http_endpoint: "localhost:4318" # OTLPHTTPEndpoint defines the OTLP/HTTP endpoint data gets send to.
       sampling_fraction: ""              # SamplingFraction defines the sampling ratio. Valid values are 0 to 1.
   replicationFactor: 0                   # The replication factor is a configuration setting that determines how many ingesters need to acknowledge the data from the distributors before accepting a span.
   retention:                             # Retention period defined by dataset. User can specify how long data should be stored.

diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go
@@ -4,7 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"net"
+	"net/url"
 	"path"
 
 	"github.com/imdario/mergo"
@@ -362,15 +362,14 @@ func patchTracing(tempo v1alpha1.TempoStack, pod corev1.PodTemplateSpec) (corev1
 		return pod, nil
 	}
 
-	host, port, err := net.SplitHostPort(tempo.Spec.Observability.Tracing.JaegerAgentEndpoint)
+	_, err := url.ParseRequestURI(tempo.Spec.Observability.Tracing.OTLPHTTPEndpoint)
 	if err != nil {
 		return corev1.PodTemplateSpec{}, err
 	}
 
 	container := corev1.Container{
 		Args: []string{
-			fmt.Sprintf("--internal.tracing.endpoint=%s:%s", host, port),
-			"--internal.tracing.endpoint-type=agent",
+			fmt.Sprintf("--internal.tracing.otlp-http-endpoint=%s", tempo.Spec.Observability.Tracing.OTLPHTTPEndpoint),
 			fmt.Sprintf("--internal.tracing.sampling-fraction=%s", tempo.Spec.Observability.Tracing.SamplingFraction),
 		},
 	}

diff --git a/internal/manifests/gateway/gateway_test.go b/internal/manifests/gateway/gateway_test.go
@@ -1,8 +1,9 @@
 package gateway
 
 import (
+	"errors"
 	"fmt"
-	"net"
+	"net/url"
 	"reflect"
 	"testing"
 
@@ -283,8 +284,8 @@ func TestPatchTracing(t *testing.T) {
 				Spec: v1alpha1.TempoStackSpec{
 					Observability: v1alpha1.ObservabilitySpec{
 						Tracing: v1alpha1.TracingConfigSpec{
-							SamplingFraction:    "1.0",
-							JaegerAgentEndpoint: "agent:1234",
+							SamplingFraction: "1.0",
+							OTLPHTTPEndpoint: "http://collector:4318",
 						},
 					},
 				},
@@ -319,8 +320,7 @@ func TestPatchTracing(t *testing.T) {
 							Name: containerNameTempoGateway,
 							Args: []string{
 								"--abc",
-								"--internal.tracing.endpoint=agent:1234",
-								"--internal.tracing.endpoint-type=agent",
+								"--internal.tracing.otlp-http-endpoint=http://collector:4318",
 								"--internal.tracing.sampling-fraction=1.0",
 							},
 						},
@@ -374,17 +374,18 @@ func TestPatchTracing(t *testing.T) {
 				Spec: v1alpha1.TempoStackSpec{
 					Observability: v1alpha1.ObservabilitySpec{
 						Tracing: v1alpha1.TracingConfigSpec{
-							SamplingFraction:    "0.5",
-							JaegerAgentEndpoint: "---invalid----",
+							SamplingFraction: "0.5",
+							OTLPHTTPEndpoint: "---invalid----",
 						},
 					},
 				},
 			},
 			inputPod:  corev1.PodTemplateSpec{},
 			expectPod: corev1.PodTemplateSpec{},
-			expectErr: &net.AddrError{
-				Addr: "---invalid----",
-				Err:  "missing port in address",
+			expectErr: &url.Error{
+				Op:  "parse",
+				URL: "---invalid----",
+				Err: errors.New("invalid URI for request"),
 			},
 		},
 	}

diff --git a/internal/manifests/manifestutils/tracing.go b/internal/manifests/manifestutils/tracing.go
@@ -1,41 +1,38 @@
 package manifestutils
 
 import (
-	"net"
+	"net/url"
 
 	"github.com/imdario/mergo"
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/grafana/tempo-operator/api/tempo/v1alpha1"
 )
 
-// PatchTracingJaegerEnv adds configures jaeger-sdk via environment variables if
+// PatchTracingJaegerEnv adds configures OTEL SDK via environment variables if
 // operand observability settings exist.
 func PatchTracingJaegerEnv(tempo v1alpha1.TempoStack, pod corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) {
 	if tempo.Spec.Observability.Tracing.SamplingFraction == "" {
 		return pod, nil
 	}
-	host, port, err := net.SplitHostPort(tempo.Spec.Observability.Tracing.JaegerAgentEndpoint)
+
+	_, err := url.ParseRequestURI(tempo.Spec.Observability.Tracing.OTLPHTTPEndpoint)
 	if err != nil {
 		return corev1.PodTemplateSpec{}, err
 	}
 
 	container := corev1.Container{
 		Env: []corev1.EnvVar{
 			{
-				Name:  "JAEGER_AGENT_HOST",
-				Value: host,
-			},
-			{
-				Name:  "JAEGER_AGENT_PORT",
-				Value: port,
+				Name:  "OTEL_EXPORTER_OTLP_ENDPOINT",
+				Value: tempo.Spec.Observability.Tracing.OTLPHTTPEndpoint,
 			},
 			{
-				Name:  "JAEGER_SAMPLER_TYPE",
-				Value: "const",
+				Name:  "OTEL_TRACES_SAMPLER",
+				Value: "parentbased_traceidratio",
 			},
 			{
-				Name:  "JAEGER_SAMPLER_PARAM",
+				Name:  "OTEL_TRACES_SAMPLER_ARG",
 				Value: tempo.Spec.Observability.Tracing.SamplingFraction,
 			},
 		},

diff --git a/internal/manifests/manifestutils/tracing_test.go b/internal/manifests/manifestutils/tracing_test.go
@@ -1,7 +1,8 @@
 package manifestutils
 
 import (
-	"net"
+	"errors"
+	"net/url"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -26,8 +27,8 @@ func Test_PatchTracingJaegerEnv(t *testing.T) {
 				Spec: v1alpha1.TempoStackSpec{
 					Observability: v1alpha1.ObservabilitySpec{
 						Tracing: v1alpha1.TracingConfigSpec{
-							SamplingFraction:    "1.0",
-							JaegerAgentEndpoint: "agent:1234",
+							SamplingFraction: "1.0",
+							OTLPHTTPEndpoint: "http://collector:4318",
 						},
 					},
 				},
@@ -69,19 +70,15 @@ func Test_PatchTracingJaegerEnv(t *testing.T) {
 									Value: "1234",
 								},
 								{
-									Name:  "JAEGER_AGENT_HOST",
-									Value: "agent",
+									Name:  "OTEL_EXPORTER_OTLP_ENDPOINT",
+									Value: "http://collector:4318",
 								},
 								{
-									Name:  "JAEGER_AGENT_PORT",
-									Value: "1234",
-								},
-								{
-									Name:  "JAEGER_SAMPLER_TYPE",
-									Value: "const",
+									Name:  "OTEL_TRACES_SAMPLER",
+									Value: "parentbased_traceidratio",
 								},
 								{
-									Name:  "JAEGER_SAMPLER_PARAM",
+									Name:  "OTEL_TRACES_SAMPLER_ARG",
 									Value: "1.0",
 								},
 							},
@@ -148,17 +145,18 @@ func Test_PatchTracingJaegerEnv(t *testing.T) {
 				Spec: v1alpha1.TempoStackSpec{
 					Observability: v1alpha1.ObservabilitySpec{
 						Tracing: v1alpha1.TracingConfigSpec{
-							SamplingFraction:    "0.5",
-							JaegerAgentEndpoint: "---invalid----",
+							SamplingFraction: "0.5",
+							OTLPHTTPEndpoint: "---invalid----",
 						},
 					},
 				},
 			},
 			inputPod:  corev1.PodTemplateSpec{},
 			expectPod: corev1.PodTemplateSpec{},
-			expectErr: &net.AddrError{
-				Addr: "---invalid----",
-				Err:  "missing port in address",
+			expectErr: &url.Error{
+				Op:  "parse",
+				URL: "---invalid----",
+				Err: errors.New("invalid URI for request"),
 			},
 		},
 	}