Merge branch 'main' of https://github.com/microsoft/promptflow into n…

…irovins/add_chat_evaluation
microsoft · Apr 23, 2024 · c393dd6 · c393dd6
2 parents 9eecaff + d614feb
commit c393dd6
Show file tree

Hide file tree

Showing 102 changed files with 1,365 additions and 781 deletions.
diff --git a/.github/pipelines/promptflow-csharp-e2e-test-tests.yml b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml
@@ -31,17 +31,16 @@ steps:
     AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
     IS_IN_CI_PIPELINE: true
 
-# enable this after we fix recording issue
-#- task: PowerShell@2
-#  displayName: 'Run azure sdk cli tests'
-#  inputs:
-#    targetType: 'inline'
-#    script: |
-#      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
-#    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
-#  env:
-#    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-#    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
-#    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
-#    PROMPT_FLOW_TEST_MODE: "replay"
-#    IS_IN_CI_PIPELINE: true
+- task: PowerShell@2
+  displayName: 'Run azure sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    PROMPT_FLOW_TEST_MODE: "replay"
+    IS_IN_CI_PIPELINE: true
diff --git a/docs/how-to-guides/chat-with-a-flow/index.md b/docs/how-to-guides/chat-with-a-flow/index.md
@@ -0,0 +1,40 @@
+# Chat with a flow
+
+:::{admonition} Experimental feature
+This is an experimental feature, and may change at any time. Learn [more](../faq.md#stable-vs-experimental).
+:::
+
+Prompt flow provides the chat window feature to facilitate an interactive chat experience within a local environment. 
+You can engage in conversation with the flow and view its responses directly within the chat window.
+
+## Initiating a chat window
+There are two methods to open a chat window: executing the prompt flow CLI command or clicking the 
+`Open test chat page` button when viewing a flow YAML file in the Prompt flow VS Code extension.
+
+::::{tab-set}
+:::{tab-item} CLI
+:sync: CLI
+
+The following CLI command allows you to trigger a chat window.
+```shell
+pf flow test --flow . --ui
+```
+Running the above command will yield the following example output:
+```
+Starting prompt flow service...
+...
+You can begin chat flow on http://127.0.0.1:**/v1.0/ui/chat?flow=***
+```
+The browser page corresponding to the chat URL will automatically open and direct the user to a chat page 
+corresponding to the passed flow:
+![chat-basic-dag-flow](../../media/how-to-guides/chat-with-a-flow/chat-basic-dag-flow.png)
+:::
+
+:::{tab-item} VS Code Extension
+:sync: VSC
+
+Click the `Open test chat page` button while viewing a flow YAML file in the Prompt flow VS Code extension, and you 
+will be directed to the chat page.
+![start-chat-window-in-vsc](../../media/how-to-guides/chat-with-a-flow/start-chat-window-in-vsc.png)
+:::
+::::
diff --git a/docs/how-to-guides/develop-a-tool/create-dynamic-list-tool-input.md b/docs/how-to-guides/develop-a-tool/create-dynamic-list-tool-input.md
@@ -187,7 +187,7 @@ pip install my-tools-package[azure]>=0.0.8
 
 If you are unable to see any options in a dynamic list tool input, you may see an error message below the input field stating:
 
-"Unable to display list of items due to XXX. Please contact the tool author/support team for troubleshooting assistance."
+"Unable to retrieve result due to XXX. Please contact the tool author/support team for troubleshooting assistance."
 
 If this occurs, follow these troubleshooting steps:
 

diff --git a/docs/how-to-guides/index.md b/docs/how-to-guides/index.md
@@ -12,8 +12,9 @@ tracing/index
 :caption: Flow
 :maxdepth: 1
 develop-a-flow/index
-run-and-evaluate-a-flow/index
 execute-flow-as-a-function
+chat-with-a-flow/index
+run-and-evaluate-a-flow/index
 ```
 
 ```{toctree}

diff --git a/docs/how-to-guides/tracing/index.md b/docs/how-to-guides/tracing/index.md
@@ -117,4 +117,47 @@ pf run create --flow . --data "./data/bert-paper-qna.jsonl" --column-mapping cha
 ```
 Then you will get a run related trace URL, e.g. http://localhost:<port>/v1.0/ui/traces?run=chat_with_pdf_20240226_181222_219335
 
-![batch_run_record](../../media/trace/batch_run_record.png)
+![batch_run_record](../../media/trace/batch_run_record.png)
+
+### Search
+
+Trace UI supports simple Python expression for search experience, which is demonstrated in below GIF:
+
+![advanced_search](../../media/trace/advanced-search.gif)
+
+Currently it supports bool operator `and` and `or`, compare operator `==`, `!=`, `>`, `>=`, `<`, `<=`; and the fields that are searchable: `name`, `kind`, `status`, `start_time`, `cumulative_token_count.total`, `cumulative_token_count.prompt` and `cumulative_token_count.completion`. You can find the hints by clicking the button right to the search edit box.
+
+![search_hint](../../media/trace/trace-ui-search-hint.png)
+
+## Local trace management
+
+### Delete
+
+Prompt flow provides capability to delete traces in local storage, user can delete traces by collection, time range or prompt flow run with both CLI and SDK:
+
+::::{tab-set}
+:::{tab-item} CLI
+:sync: CLI
+
+```bash
+pf trace delete --collection <collection-name>  # delete specific collection
+pf trace delete --collection <collection-name> --started-before '2024-03-01T16:00:00.123456'  # delete traces started before the time in specific collection
+pf trace delete --run <run-name>  # delete traces originated from specific prompt flow run
+```
+:::
+
+:::{tab-item} SDK
+:sync: SDK
+
+```python
+from promptflow.client import PFClient
+
+pf = PFClient()
+pf.traces.delete(collection="<collection-name>")  # delete specific collection
+pf.traces.delete(collection="<collection-name>", started_before="2024-03-01T16:00:00.123456")  # delete traces started before the time in specific collection
+pf.traces.delete(run="<run-name>")  # delete traces originated from specific prompt flow run
+```
+
+:::
+
+::::
diff --git a/docs/media/how-to-guides/chat-with-a-flow/chat-basic-dag-flow.png b/docs/media/how-to-guides/chat-with-a-flow/chat-basic-dag-flow.png
diff --git a/docs/media/how-to-guides/chat-with-a-flow/start-chat-window-in-vsc.png b/docs/media/how-to-guides/chat-with-a-flow/start-chat-window-in-vsc.png
diff --git a/docs/media/trace/advanced-search.gif b/docs/media/trace/advanced-search.gif
diff --git a/docs/media/trace/trace-ui-search-hint.png b/docs/media/trace/trace-ui-search-hint.png
diff --git a/docs/reference/pf-command-reference.md b/docs/reference/pf-command-reference.md
@@ -100,6 +100,7 @@ pf flow test --flow
              [--debug]
              [--interactive]
              [--verbose]
+             [--ui]
 ```
 
 #### Examples
@@ -152,6 +153,12 @@ Chat in the flow.
 pf flow test --flow <path-to-flow-directory> --node <node_name> --interactive
 ```
 
+Chat in the chat window.
+
+```bash
+pf flow test --flow <path-to-flow-directory> --ui
+```
+
 #### Required Parameter
 
 `--flow`
@@ -184,6 +191,10 @@ Start a interactive chat session for chat flow.
 
 Displays the output for each step in the chat flow.
 
+`--ui`
+
+The flag to start an interactive chat experience in local chat window.
+
 ### pf flow validate
 
 Validate the prompt flow and generate a `flow.tools.json` under `.promptflow`. This file is required when using flow as a component in a Azure ML pipeline.
@@ -908,7 +919,7 @@ Manage prompt flow service.
 
 ### pf service start
 
-Start prompt flow service.
+Start the prompt flow service.
 
 ```bash
 pf service start [--port]
@@ -917,17 +928,25 @@ pf service start [--port]
 ```
 
 #### Examples
-Start prompt flow service.
+Prompt flow will try to start the service on the default port 23333. If the port is already taken, prompt flow will 
+sequentially probe new ports, incrementing by one each time. Prompt flow retains the port number for future reference 
+and will utilize it for subsequent service startups.
+
 ```bash
 pf service start
 ```
 
-Force restart promptflow service.
+Forcefully start the prompt flow service. If the port is already in use, the existing service will be terminated and 
+restart a new service
+
 ```bash
 pf service start --force
 ```
 
-Start promptflow service with specific port.
+Start the prompt flow service with a specified port. If the port is already taken, prompt flow will raise an error 
+unless forcefully start the service with the `--force` flag. Upon availability, prompt flow retains the port number for 
+future reference and will utilize it for subsequent service startups.
+
 ```bash
 pf service start --port 65553
 ```
@@ -941,7 +960,7 @@ pf service start --debug
 
 `--port -p`
 
-Port of the promptflow service.
+The designated port of the prompt flow service and port number will be remembered if port is available.
 
 `--force`
 

diff --git a/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb b/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb
@@ -4,8 +4,13 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Getting started with flex flow in Azure\n",
-    "\n",
+    "# Getting started with flex flow in Azure"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "**Learning Objectives** - Upon completing this tutorial, you should be able to:\n",
     "\n",
     "- Write LLM application using notebook and visualize the trace of your application.\n",
@@ -239,8 +244,13 @@
  ],
  "metadata": {
   "build_doc": {
+   "author": [
+    "[email protected]",
+    "[email protected]"
+   ],
    "category": "azure",
-   "section": "Flow"
+   "section": "Flow",
+   "weight": 10
   },
   "description": "A quickstart tutorial to run a flex flow and evaluate it in azure.",
   "kernelspec": {

diff --git a/examples/flex-flows/basic/flex-flow-quickstart.ipynb b/examples/flex-flows/basic/flex-flow-quickstart.ipynb
@@ -4,7 +4,13 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Getting started with flex flow\n",
+    "# Getting started with flex flow"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "\n",
     "**Learning Objectives** - Upon completing this tutorial, you should be able to:\n",
     "\n",
@@ -310,8 +316,13 @@
  ],
  "metadata": {
   "build_doc": {
+   "author": [
+    "[email protected]",
+    "[email protected]"
+   ],
    "category": "local",
-   "section": "Flow"
+   "section": "Flow",
+   "weight": 10
   },
   "description": "A quickstart tutorial to run a flex flow and evaluate it.",
   "kernelspec": {

diff --git a/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb b/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb
@@ -4,7 +4,13 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Chat with PDF in Azure\n",
+    "# Chat with PDF in Azure"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "\n",
     "This is a simple flow that allow you to ask questions about the content of a PDF file and get answers.\n",
     "You can run the flow with a URL to a PDF file and question as argument.\n",
@@ -290,8 +296,13 @@
  ],
  "metadata": {
   "build_doc": {
+   "author": [
+    "[email protected]",
+    "[email protected]"
+   ],
    "category": "azure",
-   "section": "Flow"
+   "section": "Flow",
+   "weight": 30
   },
   "description": "A tutorial of chat-with-pdf flow that executes in Azure AI",
   "kernelspec": {

diff --git a/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb b/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb
@@ -1,11 +1,17 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Chat with PDF - test, evaluation and experimentation"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Chat with PDF - test, evaluation and experimentation\n",
     "\n",
     "We will walk you through how to use prompt flow Python SDK to test, evaluate and experiment with the \"Chat with PDF\" flow.\n",
     "\n",
@@ -303,8 +309,13 @@
  ],
  "metadata": {
   "build_doc": {
+   "author": [
+    "[email protected]",
+    "[email protected]"
+   ],
    "category": "local",
-   "section": "Flow"
+   "section": "Flow",
+   "weight": 30
   },
   "description": "A tutorial of chat-with-pdf flow that allows user ask questions about the content of a PDF file and get answers",
   "kernelspec": {
@@ -322,7 +333,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.9.19"
   }
  },
  "nbformat": 4,

diff --git a/examples/prompty/basic/prompty-quickstart.ipynb b/examples/prompty/basic/prompty-quickstart.ipynb
@@ -4,7 +4,13 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Getting started with prompty\n",
+    "# Getting started with prompty"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "\n",
     "**Learning Objectives** - Upon completing this tutorial, you should be able to:\n",
     "\n",
@@ -275,8 +281,13 @@
  ],
  "metadata": {
   "build_doc": {
+   "author": [
+    "[email protected]",
+    "[email protected]"
+   ],
    "category": "local",
-   "section": "Prompty"
+   "section": "Prompty",
+   "weight": 10
   },
   "description": "A quickstart tutorial to run a prompty and evaluate it.",
   "kernelspec": {