Merge branch 'main' into chw/pf/revert_skip_assistant_tests

microsoft · Apr 23, 2024 · bd36f11 · bd36f11
2 parents dca0b22 + 63fafe4
commit bd36f11
Show file tree

Hide file tree

Showing 106 changed files with 4,950 additions and 792 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -9,6 +9,7 @@ RUN apt-get update \
     && apt-get -y install build-essential \
     && apt-get -y install docker.io
 
+# Install notebook depenency
 RUN pip install ipython ipykernel
 RUN ipython kernel install --user --name promptflow
 

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,22 +1,29 @@
 {
-	"name": "Promptflow-Python39",
-	// "context" is the path that the Codespaces docker build command should be run from, relative to devcontainer.json
-	"context": ".",
-	"dockerFile": "Dockerfile",
-
-	// Set *default* container specific settings.json values on container create.
-	"settings": {
-		"terminal.integrated.shell.linux": "/bin/bash"
-	},
-
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"ms-python.python",
-		"ms-toolsai.vscode-ai",
-		"ms-toolsai.jupyter",
-		"redhat.vscode-yaml",
-		"prompt-flow.prompt-flow"
-	],
-
-	"runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"]
+  "name": "Promptflow-Python39",
+  // "context" is the path that the Codespaces docker build command should be run from, relative to devcontainer.json
+  "context": ".",
+  "dockerFile": "Dockerfile",
+  "runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"],
+  "customizations": {
+    "codespaces": {
+      "openFiles": ["README.md", "examples/README.md"]
+    },
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash"
+      },
+      // Add the IDs of extensions you want installed when the container is created.
+      "extensions": [
+        "ms-python.python",
+        "ms-toolsai.vscode-ai",
+        "ms-toolsai.jupyter",
+        "redhat.vscode-yaml",
+        "prompt-flow.prompt-flow"
+      ]
+    }
+  },
+  "features": { 
+	"ghcr.io/devcontainers/features/azure-cli:1": {}
+  }
 }
diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt
@@ -1,3 +1,2 @@
-azure-cli
-promptflow[azure]
+promptflow[azure]>=1.9.0
 promptflow-tools
diff --git a/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml b/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml
@@ -0,0 +1,55 @@
+parameters:
+- name: promptflowCsPat
+  displayName: "PAT to clone csharp repository"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+  - task: UseDotNet@2
+    inputs:
+      version: '6.x'
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.9.x'
+      architecture: 'x64'
+
+  - task: PowerShell@2
+    displayName: 'Install promptflow cli'
+    inputs:
+      targetType: 'inline'
+      script: |
+        Set-PSDebug -Trace 1
+        pip install -r src/promptflow/dev_requirements.txt
+        pip install src/promptflow-tracing
+        pip install src/promptflow-core[executor-service]
+        pip install src/promptflow-devkit
+        pip install src/promptflow-azure
+        pip install src/promptflow-recording
+        pip freeze
+
+  - task: PowerShell@2
+    displayName: 'Clone csharp repository'
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+
+  - task: NuGetAuthenticate@1
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'restore'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet restore'
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: 'build'
+      projects: '$(flowProjectRelativePath)/**/*.csproj'
+      feedsToUse: 'config'
+      nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
+    displayName: 'dotnet build'
diff --git a/.github/pipelines/promptflow-csharp-e2e-test-tests.yml b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml
@@ -0,0 +1,47 @@
+parameters:
+- name: azureOpenAiApiKey
+  displayName: "Azure OpenAI API Key"
+  type: string
+- name: azureOpenAiApiBase
+  displayName: "Azure OpenAI API Base"
+  type: string
+- name: flowProjectRelativePath
+  displayName: "Flow Project Relative Path"
+  type: string
+
+steps:
+- task: PowerShell@2
+  displayName: 'Copy local connections for ci pipeline'
+  inputs:
+    targetType: 'inline'
+    script: |
+      Copy-Item dev-connections.json.example connections.json
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow
+
+- task: PowerShell@2
+  displayName: 'Run sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      pytest tests/ -m "csharp"
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-devkit
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    IS_IN_CI_PIPELINE: true
+
+# enable this after we fix recording issue
+#- task: PowerShell@2
+#  displayName: 'Run azure sdk cli tests'
+#  inputs:
+#    targetType: 'inline'
+#    script: |
+#      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
+#    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
+#  env:
+#    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+#    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+#    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+#    PROMPT_FLOW_TEST_MODE: "replay"
+#    IS_IN_CI_PIPELINE: true
diff --git a/.github/pipelines/promptflow-csharp-e2e-test.yml b/.github/pipelines/promptflow-csharp-e2e-test.yml
@@ -44,25 +44,6 @@ jobs:
   pool:
     name: promptflow-1ES-ubuntu20
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9.x'
-        architecture: 'x64'
-    - task: Bash@3
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
     - task: Bash@3
       displayName: 'Set environment variables'
       inputs:
@@ -74,43 +55,16 @@ jobs:
           export ACS_CONNECTION=$(ACS_CONNECTION)
           export IS_IN_CI_PIPELINE=true
 
-    - task: Bash@3
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
-
-    - task: NuGetAuthenticate@1
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: Bash@3
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()
@@ -119,28 +73,6 @@ jobs:
   pool:
     name: promptflow-1ES-win
   steps:
-    - task: UseDotNet@2
-      inputs:
-        version: '6.x'
-
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.8.x'
-        architecture: 'x64'
-
-    - task: PowerShell@2
-      displayName: 'Install promptflow cli'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Set-PSDebug -Trace 1
-          pip install -r ./src/promptflow/dev_requirements.txt
-          pip install ./src/promptflow-tracing
-          pip install ./src/promptflow-core[executor-service]
-          pip install ./src/promptflow-devkit
-          pip install ./src/promptflow-azure
-          pip freeze
-
     - task: PowerShell@2
       displayName: 'Set environment variables'
       inputs:
@@ -152,43 +84,16 @@ jobs:
           setx ACS_CONNECTION $(ACS_CONNECTION)
           setx IS_IN_CI_PIPELINE true
 
-    - task: PowerShell@2
-      displayName: 'Clone csharp repository'
-      inputs:
-        targetType: 'inline'
-        script: |
-          git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
+    - template: promptflow-csharp-e2e-test-env-setup.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
 
-    - task: NuGetAuthenticate@1
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'restore'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet restore'
-
-    - task: DotNetCoreCLI@2
-      inputs:
-        command: 'build'
-        projects: '$(flowProjectRelativePath)/**/*.csproj'
-        feedsToUse: 'config'
-        nugetConfigPath: 'csharp/src/TestProjects/nuget.config'
-      displayName: 'dotnet build'
-
-    - task: PowerShell@2
-      displayName: 'Run tests'
-      inputs:
-        targetType: 'inline'
-        script: |
-          cd ./src/promptflow-devkit
-          pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py
-      env:
-        CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-        AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY)
-        AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)
-        IS_IN_CI_PIPELINE: true
+    - template: promptflow-csharp-e2e-test-tests.yml
+      parameters:
+        flowProjectRelativePath: '$(flowProjectRelativePath)'
+        azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
+        azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
 
     - publish: $(flowProjectRelativePath)
       condition: always()

diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml
@@ -8,6 +8,7 @@ on:
     paths:
       - 'README.md'
       - 'docs/**'
+      - 'examples/**.ipynb'
       - 'scripts/docs/**'
       - '.github/workflows/build_doc_ci.yml'
       - 'src/promptflow-tracing/promptflow/**'

diff --git a/.github/workflows/promptflow-evals-e2e-test.yml b/.github/workflows/promptflow-evals-e2e-test.yml
@@ -12,7 +12,6 @@ on:
 env:
   IS_IN_CI_PIPELINE: "true"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
-  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 jobs:
   build:
@@ -70,8 +69,8 @@ jobs:
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: generate end-to-end test config from secret
         run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
         working-directory: ${{ env.WORKING_DIRECTORY }}

diff --git a/.github/workflows/promptflow-evals-unit-test.yml b/.github/workflows/promptflow-evals-unit-test.yml
@@ -12,7 +12,6 @@ on:
 env:
   IS_IN_CI_PIPELINE: "true"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
-  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 jobs:
   build:
@@ -66,8 +65,8 @@ jobs:
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: run unit tests
         run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}

diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml
@@ -72,7 +72,7 @@ jobs:
     - name: run devkit tests
       run: |
         poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
-        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
         --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: Upload Test Results

diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml
@@ -9,6 +9,7 @@ on:
     paths:
       - 'README.md'
       - 'docs/**'
+      - 'examples/**.ipynb'
       - 'scripts/docs/**'
       - '.github/workflows/publish_doc.yml'
       - 'src/promptflow-tracing/promptflow/**'