Add Nexus failure_reason metric tag (#1671)

* Add failure_reason tag * Remove redunant WorkflowTaskFailureReason tag
temporalio · Oct 18, 2024 · 1a13bf3 · 1a13bf3
1 parent 959f581
commit 1a13bf3
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 26 deletions.
diff --git a/internal/common/metrics/constants.go b/internal/common/metrics/constants.go
@@ -92,19 +92,19 @@ const (
 
 // Metric tag keys
 const (
-	NamespaceTagName          = "namespace"
-	ClientTagName             = "client_name"
-	PollerTypeTagName         = "poller_type"
-	WorkerTypeTagName         = "worker_type"
-	WorkflowTypeNameTagName   = "workflow_type"
-	ActivityTypeNameTagName   = "activity_type"
-	NexusServiceTagName       = "nexus_service"
-	NexusOperationTagName     = "nexus_operation"
-	TaskQueueTagName          = "task_queue"
-	OperationTagName          = "operation"
-	CauseTagName              = "cause"
-	WorkflowTaskFailureReason = "failure_reason"
-	RequestFailureCode        = "status_code"
+	NamespaceTagName        = "namespace"
+	ClientTagName           = "client_name"
+	PollerTypeTagName       = "poller_type"
+	WorkerTypeTagName       = "worker_type"
+	WorkflowTypeNameTagName = "workflow_type"
+	ActivityTypeNameTagName = "activity_type"
+	NexusServiceTagName     = "nexus_service"
+	NexusOperationTagName   = "nexus_operation"
+	FailureReasonTagName    = "failure_reason"
+	TaskQueueTagName        = "task_queue"
+	OperationTagName        = "operation"
+	CauseTagName            = "cause"
+	RequestFailureCode      = "status_code"
 )
 
 // Metric tag values

diff --git a/internal/common/metrics/tags.go b/internal/common/metrics/tags.go
@@ -82,6 +82,13 @@ func NexusTags(service, operation, taskQueueName string) map[string]string {
 	}
 }
 
+// NexusTaskFailureTags returns a set of tags for Nexus Operation failures.
+func NexusTaskFailureTags(reason string) map[string]string {
+	return map[string]string{
+		FailureReasonTagName: reason,
+	}
+}
+
 // TaskQueueTags returns a set of tags for a task queue.
 func TaskQueueTags(taskQueue string) map[string]string {
 	return map[string]string{
@@ -106,7 +113,7 @@ func PollerTags(pollerType string) map[string]string {
 // WorkflowTaskFailedTags returns a set of tags for a workflow task failure.
 func WorkflowTaskFailedTags(reason string) map[string]string {
 	return map[string]string{
-		WorkflowTaskFailureReason: reason,
+		FailureReasonTagName: reason,
 	}
 }
 

diff --git a/internal/internal_nexus_task_poller.go b/internal/internal_nexus_task_poller.go
@@ -157,8 +157,21 @@ func (ntp *nexusTaskPoller) ProcessTask(task interface{}) error {
 	// Internal error processing the task.
 	// Failure from user handler.
 	// Special case for the start response with operation error.
-	if err != nil || failure != nil || res.Response.GetStartOperation().GetOperationError() != nil {
-		metricsHandler.Counter(metrics.NexusTaskExecutionFailedCounter).Inc(1)
+	if err != nil {
+		metricsHandler.
+			WithTags(metrics.NexusTaskFailureTags("internal_sdk_error")).
+			Counter(metrics.NexusTaskExecutionFailedCounter).
+			Inc(1)
+	} else if failure != nil {
+		metricsHandler.
+			WithTags(metrics.NexusTaskFailureTags("handler_error_" + failure.GetError().GetErrorType())).
+			Counter(metrics.NexusTaskExecutionFailedCounter).
+			Inc(1)
+	} else if e := res.Response.GetStartOperation().GetOperationError(); e != nil {
+		metricsHandler.
+			WithTags(metrics.NexusTaskFailureTags("operation_" + e.GetOperationState())).
+			Counter(metrics.NexusTaskExecutionFailedCounter).
+			Inc(1)
 	}
 
 	// Let the poller machinery drop the task, nothing to report back.

diff --git a/test/nexus_test.go b/test/nexus_test.go
@@ -146,11 +146,12 @@ func (tc *testContext) requireTimer(t *assert.CollectT, metric, service, operati
 	}))
 }
 
-func (tc *testContext) requireCounter(t *assert.CollectT, metric, service, operation string) {
+func (tc *testContext) requireFailureCounter(t *assert.CollectT, service, operation, failureType string) {
 	assert.True(t, slices.ContainsFunc(tc.metricsHandler.Counters(), func(ct *metrics.CapturedCounter) bool {
-		return ct.Name == metric &&
+		return ct.Name == metrics.NexusTaskExecutionFailedCounter &&
 			ct.Tags[metrics.NexusServiceTagName] == service &&
-			ct.Tags[metrics.NexusOperationTagName] == operation
+			ct.Tags[metrics.NexusOperationTagName] == operation &&
+			ct.Tags[metrics.FailureReasonTagName] == failureType
 	}))
 }
 
@@ -256,7 +257,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "operation_failed")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -271,7 +272,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_INTERNAL")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -286,7 +287,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_BAD_REQUEST")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -301,7 +302,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_BAD_REQUEST")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -316,7 +317,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_INTERNAL")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -331,7 +332,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_BAD_REQUEST")
 		}, time.Second*3, time.Millisecond*100)
 	})
 
@@ -346,7 +347,7 @@ func TestNexusSyncOperation(t *testing.T) {
 			tc.requireTimer(t, metrics.NexusTaskEndToEndLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskScheduleToStartLatency, service.Name, syncOp.Name())
 			tc.requireTimer(t, metrics.NexusTaskExecutionLatency, service.Name, syncOp.Name())
-			tc.requireCounter(t, metrics.NexusTaskExecutionFailedCounter, service.Name, syncOp.Name())
+			tc.requireFailureCounter(t, service.Name, syncOp.Name(), "handler_error_INTERNAL")
 		}, time.Second*3, time.Millisecond*100)
 	})
 }