fix: chat ui need to infer signature of csharp (#2898)

# Description

1. fix for csharp flex flow chat ui.
2. refactor current csharp tests - give them a specific marker and skip
the marker in regular tests

Hard to add corresponding tests for now given it's wired to do a copy
for all api test utils/conftest

# All Promptflow Contribution checklist:
- [x] **The pull request does not introduce [breaking changes].**
- [x] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml

@@ -0,0 +1,55 @@
+parameters:
+- name: promptflowCsPat
+  displayName: "PAT to clone csharp repository"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+  - task: UseDotNet@2
+    inputs:
+      version: '6.x'
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.9.x'
+      architecture: 'x64'
+
+  - task: PowerShell@2
+    displayName: 'Install promptflow cli'
+    inputs:
+      targetType: 'inline'
+      script: |
+        Set-PSDebug -Trace 1
+        pip install -r src/promptflow/dev_requirements.txt
+        pip install src/promptflow-tracing
+        pip install src/promptflow-core[executor-service]
+        pip install src/promptflow-devkit
+        pip install src/promptflow-azure
+        pip install src/promptflow-recording
+        pip freeze
+
+  - task: PowerShell@2
+    displayName: 'Clone csharp repository'
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+
+  - task: NuGetAuthenticate@1
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'restore'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet restore'
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'build'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet build'

.github/pipelines/promptflow-csharp-e2e-test-tests.yml

@@ -0,0 +1,47 @@
+parameters:
+- name: azureOpenAiApiKey
+  displayName: "Azure OpenAI API Key"
+  type: string
+- name: azureOpenAiApiBase
+  displayName: "Azure OpenAI API Base"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+- task: PowerShell@2
+  displayName: 'Copy local connections for ci pipeline'
+  inputs:
+    targetType: 'inline'
+    script: |
+      Copy-Item dev-connections.json.example connections.json
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow
+
+- task: PowerShell@2
+  displayName: 'Run sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      pytest tests/ -m "csharp"
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-devkit
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    IS_IN_CI_PIPELINE: true
+
+# enable this after we fix recording issue
+#- task: PowerShell@2
+#  displayName: 'Run azure sdk cli tests'
+#  inputs:
+#    targetType: 'inline'
+#    script: |
+#      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
+#    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
+#  env:
+#    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+#    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+#    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+#    PROMPT_FLOW_TEST_MODE: "replay"
+#    IS_IN_CI_PIPELINE: true

.github/pipelines/promptflow-csharp-e2e-test.yml

@@ -44,25 +44,6 @@ jobs:
   pool:
     name: promptflow-1ES-ubuntu20
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9.x'
-        architecture: 'x64'
-    - task: Bash@3
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
     - task: Bash@3
       displayName: 'Set environment variables'
       inputs:
@@ -74,43 +55,16 @@ jobs:
           export ACS_CONNECTION=$(ACS_CONNECTION)
           export IS_IN_CI_PIPELINE=true
 
-    - task: Bash@3
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
-
-    - task: NuGetAuthenticate@1
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: Bash@3
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()
@@ -119,28 +73,6 @@ jobs:
   pool:
     name: promptflow-1ES-win
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.8.x'
-        architecture: 'x64'
-
-    - task: PowerShell@2
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
-
     - task: PowerShell@2
       displayName: 'Set environment variables'
       inputs:
@@ -152,43 +84,16 @@ jobs:
           setx ACS_CONNECTION $(ACS_CONNECTION)
           setx IS_IN_CI_PIPELINE true
 
-    - task: PowerShell@2
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: NuGetAuthenticate@1
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: PowerShell@2
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()

.github/workflows/promptflow-sdk-cli-test.yml

@@ -72,7 +72,7 @@ jobs:
     - name: run devkit tests
       run: |
         poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
-        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
         --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results

.github/workflows/sdk-cli-azure-test-pull-request.yml

@@ -70,7 +70,7 @@ jobs:
         working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
           poetry run pytest ./tests/sdk_cli_azure_test --cov=promptflow --cov-config=pyproject.toml `
-          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
+          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp"
 
       - name: Upload Test Results
         if: always()

src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py

@@ -7,7 +7,7 @@
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import Callable
+from typing import Callable, TypedDict
 from unittest.mock import patch
 
 import jwt
@@ -626,3 +626,26 @@ def mock_trace_provider_to_cloud(
     )
     with patch("promptflow._sdk._configuration.Configuration.get_trace_provider", return_value=trace_provider):
         yield
+
+
+class CSharpProject(TypedDict):
+    flow_dir: str
+    data: str
+    init: str
+
+
+def construct_csharp_test_project(flow_name: str) -> CSharpProject:
+    root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None)
+    if not root_of_test_cases:
+        pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.")
+    root_of_test_cases = Path(root_of_test_cases)
+    return {
+        "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(),
+        "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(),
+        "init": (root_of_test_cases / flow_name / "init.json").as_posix(),
+    }
+
+
+@pytest.fixture
+def csharp_test_project_basic() -> CSharpProject:
+    return construct_csharp_test_project("Basic")

src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py

@@ -0,0 +1,33 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable
+
+import pytest
+
+from promptflow._sdk.entities import Run
+from promptflow.azure import PFClient
+
+from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
+
+
+@pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD)
+@pytest.mark.e2etest
+@pytest.mark.csharp
+@pytest.mark.usefixtures(
+    "mock_set_headers_with_user_aml_token",
+    "single_worker_thread_pool",
+    "vcr_recording",
+)
+class TestCSharpSdk:
+    def test_basic_run_bulk(self, pf: PFClient, randstr: Callable[[str], str], csharp_test_project_basic):
+        name = randstr("name")
+
+        run = pf.run(
+            flow=csharp_test_project_basic["flow_dir"],
+            data=csharp_test_project_basic["data"],
+            name=name,
+        )
+        assert isinstance(run, Run)
+        assert run.name == name

src/promptflow-core/promptflow/_utils/flow_utils.py

@@ -194,7 +194,7 @@ def is_prompty_flow(file_path: Union[str, Path], raise_error: bool = False):
     return Path(file_path).suffix.lower() == PROMPTY_EXTENSION
 
 
-def resolve_entry_file(entry: str, working_dir: Path) -> Optional[str]:
+def resolve_python_entry_file(entry: str, working_dir: Path) -> Optional[str]:
     """Resolve entry file from entry.
     If entry is a local file, e.g. my.local.file:entry_function, return the local file: my/local/file.py
         and executor will import it from local file.