diff --git a/src/promptflow-core/promptflow/core/_prompty_utils.py b/src/promptflow-core/promptflow/core/_prompty_utils.py
index aa1a265d954..1e7cb1c05a3 100644
--- a/src/promptflow-core/promptflow/core/_prompty_utils.py
+++ b/src/promptflow-core/promptflow/core/_prompty_utils.py
@@ -146,7 +146,7 @@ def convert_prompt_template(template, inputs, api):
 def prepare_open_ai_request_params(model_config, template, connection):
     params = copy.copy(model_config.parameters)
     if isinstance(connection, AzureOpenAIConnection):
-        params["extra_headers"] = {"ms-azure-ai-promptflow-called-from": "promptflow-core"}
+        params.setdefault("extra_headers", {}).update({"ms-azure-ai-promptflow-called-from": "promptflow-core"})
     params["model"] = model_config._model
 
     if model_config.api == "completion":
diff --git a/src/promptflow-evals/promptflow/evals/evaluators/_relevance/_relevance.py b/src/promptflow-evals/promptflow/evals/evaluators/_relevance/_relevance.py
index f688f27818e..8e67f0e34ab 100644
--- a/src/promptflow-evals/promptflow/evals/evaluators/_relevance/_relevance.py
+++ b/src/promptflow-evals/promptflow/evals/evaluators/_relevance/_relevance.py
@@ -9,6 +9,10 @@
 
 from promptflow.client import load_flow
 from promptflow.core import AzureOpenAIModelConfiguration
+try:
+    from ..._user_agent import USER_AGENT
+except ImportError:
+    USER_AGENT = None
 
 
 class RelevanceEvaluator:
@@ -44,7 +48,13 @@ def __init__(self, model_config: AzureOpenAIModelConfiguration):
         if model_config.api_version is None:
             model_config.api_version = "2024-02-15-preview"
 
-        prompty_model_config = {"configuration": model_config}
+        prompty_model_config = {
+            "configuration": model_config,
+        }
+
+        prompty_model_config.update({"parameters": {"extra_headers": {"x-ms-user-agent": USER_AGENT}}})\
+            if USER_AGENT and isinstance(model_config, AzureOpenAIModelConfiguration) else None
+
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, "relevance.prompty")
         self._flow = load_flow(source=prompty_path, model=prompty_model_config)