Add Local Prompty tracing support to 'evaluate.py` script (#159)

* Tracing in evaluate

* `init_tracing` now allows to optionally use Prompty tracer locally without enforcing OpenTelemetry when local tracing is not used

* Use Prompty tracer for evaluations if `LOCAL_TRACING=true`

OpenTelemetry is never used for evaluations

src/api/evaluate/evaluate.py

@@ -9,6 +9,8 @@
 from promptflow.evals.evaluate import evaluate
 from evaluate.evaluators import ArticleEvaluator
 from orchestrator import create
+from prompty.tracer import trace
+from tracing import init_tracing
 
 from dotenv import load_dotenv
 
@@ -78,6 +80,7 @@ def run_orchestrator(research_context, product_context, assignment_context):
         "response": json.dumps(response),
     }
 
+@trace
 def evaluate_orchestrator(model_config, data_path):
     writer_evaluator = ArticleEvaluator(model_config)
 
@@ -149,6 +152,12 @@ def evaluate_row(research_context, product_context, assignment_context):
     print(f"Starting evaluate...")
     print(os.environ["BING_SEARCH_ENDPOINT"])
     print("value: ", os.environ["BING_SEARCH_KEY"], len(os.environ["BING_SEARCH_KEY"]))
+
+    # Use LOCAL_TRACING to determine if we should use the PromptyTracer locally
+    # Never use remote_tracing for evaluation, it's not really relevant
+    LOCAL_TRACING = True if os.getenv("LOCAL_TRACING", "false").lower() == "true" else False
+    tracer = init_tracing(local_tracing=LOCAL_TRACING, remote_tracing=False)
+
     eval_result = evaluate_orchestrator(model_config, data_path=folder +"/eval_inputs.jsonl")
 
     end=time.time()

src/api/tracing.py

@@ -27,13 +27,22 @@ def verbose_trace(key, value):
         yield verbose_trace
 
 
-def init_tracing(local_tracing: bool = False):
+def init_tracing(remote_tracing: bool, local_tracing: bool = False):
+    """
+    Initialize tracing for the application
+    If local_tracing is True, use the PromptyTracer
+    If remote_tracing is True, use the OpenTelemetry tracer
+    If remote_tracing is not specified, defaults to using the OpenTelemetry tracer only if local_tracing is False
+    """
+
+    if remote_tracing is None:
+        remote_tracing = not local_tracing
 
     if local_tracing:
         local_trace = PromptyTracer()
         Tracer.add("PromptyTracer", local_trace.tracer)
         # Tracer.add("ConsoleTracer", console_tracer)
-    else:
+    elif remote_tracing:
         Tracer.add("OpenTelemetry", trace_span)
 
     azmon_logger = logging.getLogger("azure")