ci: split pipeline

.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml

@@ -0,0 +1,55 @@
+parameters:
+- name: promptflowCsPat
+  displayName: "PAT to clone csharp repository"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+  - task: UseDotNet@2
+    inputs:
+      version: '6.x'
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.9.x'
+      architecture: 'x64'
+
+  - task: PowerShell@2
+    displayName: 'Install promptflow cli'
+    inputs:
+      targetType: 'inline'
+      script: |
+        Set-PSDebug -Trace 1
+        pip install -r src/promptflow/dev_requirements.txt
+        pip install src/promptflow-tracing
+        pip install src/promptflow-core[executor-service]
+        pip install src/promptflow-devkit
+        pip install src/promptflow-azure
+        pip install src/promptflow-recording
+        pip freeze
+
+  - task: PowerShell@2
+    displayName: 'Clone csharp repository'
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+
+  - task: NuGetAuthenticate@1
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'restore'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet restore'
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'build'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet build'

.github/pipelines/promptflow-csharp-e2e-test-tests.yml

@@ -0,0 +1,38 @@
+parameters:
+- name: azureOpenAiApiKey
+  displayName: "Azure OpenAI API Key"
+  type: string
+- name: azureOpenAiApiBase
+  displayName: "Azure OpenAI API Base"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+- task: PowerShell@2
+  displayName: 'Run sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      cd ./src/promptflow-devkit
+      pytest tests/ -m "csharp"
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    IS_IN_CI_PIPELINE: true
+
+- task: PowerShell@2
+  displayName: 'Run azure sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      cd ./src/promptflow-azure
+      poetry run pytest tests/ -m "csharp"
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    PROMPT_FLOW_TEST_MODE: "replay"
+    IS_IN_CI_PIPELINE: true

.github/pipelines/promptflow-csharp-e2e-test.yml

@@ -44,25 +44,6 @@ jobs:
   pool:
     name: promptflow-1ES-ubuntu20
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9.x'
-        architecture: 'x64'
-    - task: Bash@3
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
     - task: Bash@3
       displayName: 'Set environment variables'
       inputs:
@@ -74,43 +55,16 @@ jobs:
           export ACS_CONNECTION=$(ACS_CONNECTION)
           export IS_IN_CI_PIPELINE=true
 
-    - task: Bash@3
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
-
-    - task: NuGetAuthenticate@1
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: Bash@3
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()
@@ -119,28 +73,6 @@ jobs:
   pool:
     name: promptflow-1ES-win
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.8.x'
-        architecture: 'x64'
-
-    - task: PowerShell@2
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
-
     - task: PowerShell@2
       displayName: 'Set environment variables'
       inputs:
@@ -152,43 +84,16 @@ jobs:
           setx ACS_CONNECTION $(ACS_CONNECTION)
           setx IS_IN_CI_PIPELINE true
 
-    - task: PowerShell@2
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: NuGetAuthenticate@1
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: PowerShell@2
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()

.github/workflows/promptflow-sdk-cli-test.yml

@@ -72,7 +72,7 @@ jobs:
     - name: run devkit tests
       run: |
         poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
-        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
         --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results

.github/workflows/sdk-cli-azure-test-pull-request.yml

@@ -70,7 +70,7 @@ jobs:
         working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
           poetry run pytest ./tests/sdk_cli_azure_test --cov=promptflow --cov-config=pyproject.toml `
-          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
+          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp"
 
       - name: Upload Test Results
         if: always()

src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py

@@ -7,7 +7,7 @@
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import Callable
+from typing import Callable, TypedDict
 from unittest.mock import patch
 
 import jwt
@@ -626,3 +626,26 @@ def mock_trace_provider_to_cloud(
     )
     with patch("promptflow._sdk._configuration.Configuration.get_trace_provider", return_value=trace_provider):
         yield
+
+
+class CSharpProject(TypedDict):
+    flow_dir: str
+    data: str
+    init: str
+
+
+def construct_csharp_test_project(flow_name: str) -> CSharpProject:
+    root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None)
+    if not root_of_test_cases:
+        pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.")
+    root_of_test_cases = Path(root_of_test_cases)
+    return {
+        "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(),
+        "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(),
+        "init": (root_of_test_cases / flow_name / "init.json").as_posix(),
+    }
+
+
+@pytest.fixture
+def csharp_test_project_basic() -> CSharpProject:
+    return construct_csharp_test_project("Basic")

src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py

@@ -0,0 +1,33 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Callable
+
+import pytest
+
+from promptflow._sdk.entities import Run
+from promptflow.azure import PFClient
+
+from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD
+
+
+@pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD)
+@pytest.mark.e2etest
+@pytest.mark.csharp
+@pytest.mark.usefixtures(
+    "mock_set_headers_with_user_aml_token",
+    "single_worker_thread_pool",
+    "vcr_recording",
+)
+class TestCSharpSdk:
+    def test_basic_run_bulk(self, pf: PFClient, randstr: Callable[[str], str], csharp_test_project_basic):
+        name = randstr("name")
+
+        run = pf.run(
+            flow=csharp_test_project_basic["flow_dir"],
+            data=csharp_test_project_basic["data"],
+            name=name,
+        )
+        assert isinstance(run, Run)
+        assert run.name == name

src/promptflow-devkit/pyproject.toml

@@ -121,7 +121,8 @@ pf = "promptflow._cli.pf:main"
 [tool.pytest.ini_options]
 markers = [
   "unittest",
-  "e2etest"
+  "e2etest",
+  "csharp"
 ]
 # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action)
 # durations - list the slowest test durations