diff --git a/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml b/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml new file mode 100644 index 00000000000..d84bcaa71dd --- /dev/null +++ b/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml @@ -0,0 +1,55 @@ +parameters: +- name: promptflowCsPat + displayName: "PAT to clone csharp repository" + type: string +- name: flowProjectRelativePath + displayName: "Flow Project Relative Path" + type: string + +steps: + - task: UseDotNet@2 + inputs: + version: '6.x' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9.x' + architecture: 'x64' + + - task: PowerShell@2 + displayName: 'Install promptflow cli' + inputs: + targetType: 'inline' + script: | + Set-PSDebug -Trace 1 + pip install -r src/promptflow/dev_requirements.txt + pip install src/promptflow-tracing + pip install src/promptflow-core[executor-service] + pip install src/promptflow-devkit + pip install src/promptflow-azure + pip install src/promptflow-recording + pip freeze + + - task: PowerShell@2 + displayName: 'Clone csharp repository' + inputs: + targetType: 'inline' + script: | + git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp + + - task: NuGetAuthenticate@1 + + - task: DotNetCoreCLI@2 + inputs: + command: 'restore' + projects: '$(flowProjectRelativePath)/**/*.csproj' + feedsToUse: 'config' + nugetConfigPath: '$(flowProjectRelativePath)/nuget.config' + displayName: 'dotnet restore' + + - task: DotNetCoreCLI@2 + inputs: + command: 'build' + projects: '$(flowProjectRelativePath)/**/*.csproj' + feedsToUse: 'config' + nugetConfigPath: '$(flowProjectRelativePath)/nuget.config' + displayName: 'dotnet build' diff --git a/.github/pipelines/promptflow-csharp-e2e-test-tests.yml b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml new file mode 100644 index 00000000000..c88884a1875 --- /dev/null +++ b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml @@ -0,0 +1,47 @@ +parameters: +- name: azureOpenAiApiKey + displayName: "Azure OpenAI API Key" + type: string +- name: azureOpenAiApiBase + displayName: "Azure OpenAI API Base" + type: string +- name: flowProjectRelativePath + displayName: "Flow Project Relative Path" + type: string + +steps: +- task: PowerShell@2 + displayName: 'Copy local connections for ci pipeline' + inputs: + targetType: 'inline' + script: | + Copy-Item dev-connections.json.example connections.json + workingDirectory: $(Build.SourcesDirectory)/src/promptflow + +- task: PowerShell@2 + displayName: 'Run sdk cli tests' + inputs: + targetType: 'inline' + script: | + pytest tests/ -m "csharp" + workingDirectory: $(Build.SourcesDirectory)/src/promptflow-devkit + env: + CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) + AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey) + AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase) + IS_IN_CI_PIPELINE: true + +# enable this after we fix recording issue +#- task: PowerShell@2 +# displayName: 'Run azure sdk cli tests' +# inputs: +# targetType: 'inline' +# script: | +# pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py +# workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure +# env: +# CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) +# AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey) +# AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase) +# PROMPT_FLOW_TEST_MODE: "replay" +# IS_IN_CI_PIPELINE: true diff --git a/.github/pipelines/promptflow-csharp-e2e-test.yml b/.github/pipelines/promptflow-csharp-e2e-test.yml index 585cdc04d62..7af220811ba 100644 --- a/.github/pipelines/promptflow-csharp-e2e-test.yml +++ b/.github/pipelines/promptflow-csharp-e2e-test.yml @@ -44,25 +44,6 @@ jobs: pool: name: promptflow-1ES-ubuntu20 steps: - - task: UseDotNet@2 - inputs: - version: '6.x' - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9.x' - architecture: 'x64' - - task: Bash@3 - displayName: 'Install promptflow cli' - inputs: - targetType: 'inline' - script: | - Set-PSDebug -Trace 1 - pip install -r ./src/promptflow/dev_requirements.txt - pip install ./src/promptflow-tracing - pip install ./src/promptflow-core[executor-service] - pip install ./src/promptflow-devkit - pip install ./src/promptflow-azure - pip freeze - task: Bash@3 displayName: 'Set environment variables' inputs: @@ -74,43 +55,16 @@ jobs: export ACS_CONNECTION=$(ACS_CONNECTION) export IS_IN_CI_PIPELINE=true - - task: Bash@3 - displayName: 'Clone csharp repository' - inputs: - targetType: 'inline' - script: | - git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp - - - task: NuGetAuthenticate@1 + - template: promptflow-csharp-e2e-test-env-setup.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + promptflowCsPat: '$(PROMPTFLOW_CS_PAT)' - - task: DotNetCoreCLI@2 - inputs: - command: 'restore' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet restore' - - - task: DotNetCoreCLI@2 - inputs: - command: 'build' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet build' - - - task: Bash@3 - displayName: 'Run tests' - inputs: - targetType: 'inline' - script: | - cd ./src/promptflow-devkit - pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py - env: - CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) - AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - IS_IN_CI_PIPELINE: true + - template: promptflow-csharp-e2e-test-tests.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)' + azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)' - publish: $(flowProjectRelativePath) condition: always() @@ -119,28 +73,6 @@ jobs: pool: name: promptflow-1ES-win steps: - - task: UseDotNet@2 - inputs: - version: '6.x' - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8.x' - architecture: 'x64' - - - task: PowerShell@2 - displayName: 'Install promptflow cli' - inputs: - targetType: 'inline' - script: | - Set-PSDebug -Trace 1 - pip install -r ./src/promptflow/dev_requirements.txt - pip install ./src/promptflow-tracing - pip install ./src/promptflow-core[executor-service] - pip install ./src/promptflow-devkit - pip install ./src/promptflow-azure - pip freeze - - task: PowerShell@2 displayName: 'Set environment variables' inputs: @@ -152,43 +84,16 @@ jobs: setx ACS_CONNECTION $(ACS_CONNECTION) setx IS_IN_CI_PIPELINE true - - task: PowerShell@2 - displayName: 'Clone csharp repository' - inputs: - targetType: 'inline' - script: | - git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp + - template: promptflow-csharp-e2e-test-env-setup.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + promptflowCsPat: '$(PROMPTFLOW_CS_PAT)' - - task: NuGetAuthenticate@1 - - - task: DotNetCoreCLI@2 - inputs: - command: 'restore' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet restore' - - - task: DotNetCoreCLI@2 - inputs: - command: 'build' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet build' - - - task: PowerShell@2 - displayName: 'Run tests' - inputs: - targetType: 'inline' - script: | - cd ./src/promptflow-devkit - pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py - env: - CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) - AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - IS_IN_CI_PIPELINE: true + - template: promptflow-csharp-e2e-test-tests.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)' + azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)' - publish: $(flowProjectRelativePath) condition: always() diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 6dec7fb6e60..e7955629903 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -72,7 +72,7 @@ jobs: - name: run devkit tests run: | poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \ - --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \ + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \ --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py working-directory: ${{ env.WORKING_DIRECTORY }} - name: Upload Test Results diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml index a989e8c7d19..31d1222b5cd 100644 --- a/.github/workflows/sdk-cli-azure-test-pull-request.yml +++ b/.github/workflows/sdk-cli-azure-test-pull-request.yml @@ -70,7 +70,7 @@ jobs: working-directory: ${{ env.WORKING_DIRECTORY }} run: | poetry run pytest ./tests/sdk_cli_azure_test --cov=promptflow --cov-config=pyproject.toml ` - --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" - name: Upload Test Results if: always() diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py index df0c93adcb2..e6643bd60a8 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py @@ -7,7 +7,7 @@ import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from typing import Callable +from typing import Callable, TypedDict from unittest.mock import patch import jwt @@ -626,3 +626,26 @@ def mock_trace_provider_to_cloud( ) with patch("promptflow._sdk._configuration.Configuration.get_trace_provider", return_value=trace_provider): yield + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +def construct_csharp_test_project(flow_name: str) -> CSharpProject: + root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None) + if not root_of_test_cases: + pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.") + root_of_test_cases = Path(root_of_test_cases) + return { + "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(), + "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(), + "init": (root_of_test_cases / flow_name / "init.json").as_posix(), + } + + +@pytest.fixture +def csharp_test_project_basic() -> CSharpProject: + return construct_csharp_test_project("Basic") diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py deleted file mode 100644 index ed014372187..00000000000 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import os.path -from typing import Callable - -import pytest - -from promptflow.azure import PFClient - - -def get_repo_base_path(): - return os.getenv("CSHARP_REPO_BASE_PATH", None) - - -@pytest.mark.usefixtures( - "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" -) -@pytest.mark.cli_test -@pytest.mark.e2etest -@pytest.mark.skipif(get_repo_base_path() is None, reason="available locally only before csharp support go public") -class TestCSharpCli: - def test_eager_flow_run_without_yaml(self, pf: PFClient, randstr: Callable[[str], str]): - pf.run( - flow=f"{get_repo_base_path()}\\src\\PromptflowCSharp\\Sample\\Basic\\bin\\Debug\\net6.0", - ) diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py new file mode 100644 index 00000000000..f1ffc69754b --- /dev/null +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py @@ -0,0 +1,33 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +from typing import Callable + +import pytest + +from promptflow._sdk.entities import Run +from promptflow.azure import PFClient + +from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD + + +@pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD) +@pytest.mark.e2etest +@pytest.mark.csharp +@pytest.mark.usefixtures( + "mock_set_headers_with_user_aml_token", + "single_worker_thread_pool", + "vcr_recording", +) +class TestCSharpSdk: + def test_basic_run_bulk(self, pf: PFClient, randstr: Callable[[str], str], csharp_test_project_basic): + name = randstr("name") + + run = pf.run( + flow=csharp_test_project_basic["flow_dir"], + data=csharp_test_project_basic["data"], + name=name, + ) + assert isinstance(run, Run) + assert run.name == name diff --git a/src/promptflow-core/promptflow/_utils/flow_utils.py b/src/promptflow-core/promptflow/_utils/flow_utils.py index 1fba097f265..0962a675de8 100644 --- a/src/promptflow-core/promptflow/_utils/flow_utils.py +++ b/src/promptflow-core/promptflow/_utils/flow_utils.py @@ -194,7 +194,7 @@ def is_prompty_flow(file_path: Union[str, Path], raise_error: bool = False): return Path(file_path).suffix.lower() == PROMPTY_EXTENSION -def resolve_entry_file(entry: str, working_dir: Path) -> Optional[str]: +def resolve_python_entry_file(entry: str, working_dir: Path) -> Optional[str]: """Resolve entry file from entry. If entry is a local file, e.g. my.local.file:entry_function, return the local file: my/local/file.py and executor will import it from local file. diff --git a/src/promptflow-core/promptflow/core/_utils.py b/src/promptflow-core/promptflow/core/_utils.py index b4b92e757eb..b97d1ced01b 100644 --- a/src/promptflow-core/promptflow/core/_utils.py +++ b/src/promptflow-core/promptflow/core/_utils.py @@ -9,7 +9,7 @@ from jinja2 import Template from promptflow._constants import AZURE_WORKSPACE_REGEX_FORMAT -from promptflow._utils.flow_utils import is_flex_flow, resolve_entry_file, resolve_flow_path +from promptflow._utils.flow_utils import is_flex_flow, resolve_flow_path, resolve_python_entry_file from promptflow._utils.logger_utils import LoggerFactory from promptflow._utils.utils import _match_reference from promptflow._utils.yaml_utils import load_yaml @@ -44,7 +44,7 @@ def init_executable(*, flow_dag: dict = None, flow_path: Path = None, working_di if is_flex_flow(yaml_dict=flow_dag): entry = flow_dag.get("entry") - entry_file = resolve_entry_file(entry=entry, working_dir=working_dir) + entry_file = resolve_python_entry_file(entry=entry, working_dir=working_dir) from promptflow._core.entry_meta_generator import generate_flow_meta diff --git a/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py b/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py index 50f1e699de3..e607bb39d1f 100644 --- a/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py +++ b/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py @@ -10,7 +10,7 @@ from promptflow._core.run_tracker import RunTracker from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT, FLOW_TOOLS_JSON_GEN_TIMEOUT from promptflow._sdk._utils import can_accept_kwargs -from promptflow._utils.flow_utils import resolve_entry_file +from promptflow._utils.flow_utils import resolve_python_entry_file from promptflow._utils.logger_utils import bulk_logger from promptflow._utils.yaml_utils import load_yaml from promptflow.contracts.run_mode import RunMode @@ -44,7 +44,7 @@ def _generate_flow_json( # generate flow.json only for eager flow for now return generate_flow_meta( flow_directory=working_dir, - source_path=resolve_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), + source_path=resolve_python_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), data=flow_dag, dump=dump, timeout=timeout, diff --git a/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py b/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py index b18d45f9519..ccb5b051e5b 100644 --- a/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py +++ b/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py @@ -5,7 +5,7 @@ from promptflow._constants import FlowEntryRegex from promptflow._core.entry_meta_generator import _generate_flow_meta from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT -from promptflow._utils.flow_utils import is_flex_flow, resolve_entry_file +from promptflow._utils.flow_utils import is_flex_flow, resolve_python_entry_file from ._base_inspector_proxy import AbstractInspectorProxy @@ -41,7 +41,7 @@ def get_entry_meta( # generate flow.json only for eager flow for now return _generate_flow_meta( flow_directory=working_dir, - source_path=resolve_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), + source_path=resolve_python_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), data=flow_dag, timeout=timeout, load_in_subprocess=load_in_subprocess, diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py index 04483203ff5..67f1223ce1f 100644 --- a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py +++ b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py @@ -39,7 +39,6 @@ _get_additional_includes, _merge_local_code_and_additional_includes, copy_tree_respect_template_and_ignore_file, - entry_string_to_callable, format_signature_type, generate_flow_tools_json, generate_random_string, @@ -55,7 +54,6 @@ is_flex_flow, is_prompty_flow, parse_variant, - resolve_entry_file, ) from promptflow._utils.yaml_utils import dump_yaml, load_yaml from promptflow.exceptions import ErrorTarget, UserErrorException @@ -1038,15 +1036,20 @@ def _infer_signature(entry: Union[Callable, FlexFlow, Flow, Prompty], include_pr flow_meta["init"] = init_dict format_signature_type(flow_meta) elif isinstance(entry, FlexFlow): - # TODO: this part will fail for csharp - entry_file = resolve_entry_file(entry=entry.entry, working_dir=entry.code) - entry_func = entry_string_to_callable(entry_file, entry.entry) + # non-python flow depends on dumped flow meta to infer signature + ProxyFactory().create_inspector_proxy(language=entry.language).prepare_metadata( + flow_file=entry.path, + working_dir=entry.code, + ) flow_meta, _, _ = FlowOperations._infer_signature_flex_flow( - entry=entry_func, language=entry.language, include_primitive_output=include_primitive_output + entry=entry.entry, + code=entry.code.as_posix(), + language=entry.language, + include_primitive_output=include_primitive_output, ) elif inspect.isclass(entry) or inspect.isfunction(entry): flow_meta, _, _ = FlowOperations._infer_signature_flex_flow( - entry=entry, include_primitive_output=include_primitive_output + entry=entry, include_primitive_output=include_primitive_output, language=FlowLanguage.Python ) else: # TODO support to get infer signature of dag flow @@ -1057,16 +1060,13 @@ def _infer_signature(entry: Union[Callable, FlexFlow, Flow, Prompty], include_pr def _infer_signature_flex_flow( entry: Union[Callable, str], *, + language: str, code: str = None, keep_entry: bool = False, validate: bool = True, - language: str = FlowLanguage.Python, include_primitive_output: bool = False, ) -> Tuple[dict, Path, List[str]]: - """Infer signature of a flow entry. - - Note that this is a Python only feature. - """ + """Infer signature of a flow entry.""" snapshot_list = None # resolve entry and code if isinstance(entry, str): @@ -1117,7 +1117,6 @@ def _infer_signature_flex_flow( # this path is actually not used flow = FlexFlow(path=code / FLOW_FLEX_YAML, code=code, data=flow_meta, entry=flow_meta["entry"]) flow._validate(raise_error=True) - flow_meta.pop("language", None) if include_primitive_output and "outputs" not in flow_meta: flow_meta["outputs"] = { @@ -1126,9 +1125,11 @@ def _infer_signature_flex_flow( } } - if not keep_entry: - flow_meta.pop("entry", None) - return flow_meta, code, snapshot_list + keys_to_keep = ["inputs", "outputs", "init"] + if keep_entry: + keys_to_keep.append("entry") + filtered_meta = {k: flow_meta[k] for k in keys_to_keep if k in flow_meta} + return filtered_meta, code, snapshot_list @monitor_operation(activity_name="pf.flows.infer_signature", activity_type=ActivityType.PUBLICAPI) def infer_signature(self, entry: Union[Callable, FlexFlow, Flow, Prompty], **kwargs) -> dict: diff --git a/src/promptflow-devkit/pyproject.toml b/src/promptflow-devkit/pyproject.toml index 78c2e135051..b4169508782 100644 --- a/src/promptflow-devkit/pyproject.toml +++ b/src/promptflow-devkit/pyproject.toml @@ -121,7 +121,8 @@ pf = "promptflow._cli.pf:main" [tool.pytest.ini_options] markers = [ "unittest", - "e2etest" + "e2etest", + "csharp" ] # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action) # durations - list the slowest test durations diff --git a/src/promptflow-devkit/tests/conftest.py b/src/promptflow-devkit/tests/conftest.py index a8184a3ab78..d366f7953cf 100644 --- a/src/promptflow-devkit/tests/conftest.py +++ b/src/promptflow-devkit/tests/conftest.py @@ -4,6 +4,7 @@ import tempfile from multiprocessing import Lock from pathlib import Path +from typing import TypedDict from unittest.mock import MagicMock, patch import pytest @@ -256,3 +257,44 @@ def reset_tracer_provider(): "opentelemetry.trace._TRACER_PROVIDER_SET_ONCE._done", False ): yield + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +def construct_csharp_test_project(flow_name: str) -> CSharpProject: + root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None) + if not root_of_test_cases: + pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.") + root_of_test_cases = Path(root_of_test_cases) + return { + "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(), + "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(), + "init": (root_of_test_cases / flow_name / "init.json").as_posix(), + } + + +@pytest.fixture +def csharp_test_project_basic() -> CSharpProject: + return construct_csharp_test_project("Basic") + + +@pytest.fixture +def csharp_test_project_basic_chat() -> CSharpProject: + return construct_csharp_test_project("BasicChat") + + +@pytest.fixture +def csharp_test_project_function_mode_basic() -> CSharpProject: + return construct_csharp_test_project("FunctionModeBasic") + + +@pytest.fixture +def csharp_test_project_class_init_flex_flow() -> CSharpProject: + is_in_ci_pipeline = os.getenv("IS_IN_CI_PIPELINE", "false").lower() == "true" + if is_in_ci_pipeline: + pytest.skip(reason="need to avoid fetching connection from local pfs to enable this in ci") + return construct_csharp_test_project("ClassInitFlexFlow") diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py index d4eab61a457..3493f1f0d36 100644 --- a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py @@ -1,78 +1,15 @@ -import dataclasses import json import os import os.path import sys from pathlib import Path -from typing import Optional +from typing import TypedDict import pytest from promptflow._cli._pf.entry import main -class TestCase: - def __init__(self, root_of_test_cases: Path, flow_name: str, skip_reason: str = None): - self._flow_dir = (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix() - self.data = (root_of_test_cases / flow_name / "data.jsonl").as_posix() - self.init = (root_of_test_cases / flow_name / "init.json").as_posix() - self._skip_reason = skip_reason - - @property - def flow_dir(self): - if self._skip_reason: - pytest.skip(self._skip_reason) - return self._flow_dir - - -@dataclasses.dataclass -class TestCases: - basic: TestCase - function_mode_basic: TestCase - basic_with_builtin_llm: TestCase - class_init_flex_flow: TestCase - basic_chat: TestCase - - def __init__(self, root_of_test_cases: Path): - is_in_ci_pipeline = os.getenv("IS_IN_CI_PIPELINE", "false").lower() == "true" - self.basic = TestCase(root_of_test_cases, "Basic") - self.function_mode_basic = TestCase(root_of_test_cases, "FunctionModeBasic") - self.class_init_flex_flow = TestCase( - root_of_test_cases, - "ClassInitFlexFlow", - "need to avoid fetching connection from local pfs to enable this in ci" if is_in_ci_pipeline else None, - ) - self.basic_chat = TestCase(root_of_test_cases, "BasicChat") - - package_root = Path(__file__).parent.parent.parent.parent.parent / "promptflow" - dev_connections_path = package_root / "connections.json" - - if is_in_ci_pipeline and not dev_connections_path.exists(): - dev_connections_path.write_text( - json.dumps( - { - "azure_open_ai_connection": { - "type": "AzureOpenAIConnection", - "value": { - "api_key": os.getenv("AZURE_OPENAI_API_KEY", "00000000000000000000000000000000"), - "api_base": os.getenv("AZURE_OPENAI_ENDPOINT", "https://openai.azure.com/"), - "api_type": "azure", - "api_version": "2023-07-01-preview", - }, - "module": "promptflow.connections", - } - } - ) - ) - print(f"Using dev connections file: {dev_connections_path}") - - -def get_root_test_cases() -> Optional[TestCases]: - target_path = os.getenv("CSHARP_TEST_CASES_ROOT", None) - target_path = Path(target_path or Path(__file__).parent) - return TestCases(target_path) - - # TODO: move this to a shared utility module def run_pf_command(*args, cwd=None): """Run a pf command with the given arguments and working directory. @@ -91,51 +28,57 @@ def run_pf_command(*args, cwd=None): os.chdir(origin_cwd) -root_test_cases = get_root_test_cases() +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str @pytest.mark.usefixtures( - "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" + "use_secrets_config_file", + "recording_injection", + "setup_local_connection", + "install_custom_tool_pkg", ) @pytest.mark.cli_test @pytest.mark.e2etest -@pytest.mark.skipif( - not os.getenv("CSHARP_TEST_CASES_ROOT", None), reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT." -) +@pytest.mark.csharp class TestCSharpCli: @pytest.mark.parametrize( - "test_case", + "target_fixture_name", [ - pytest.param(root_test_cases.basic, id="basic"), - pytest.param(root_test_cases.basic_chat, id="basic_chat"), - pytest.param(root_test_cases.function_mode_basic, id="function_mode_basic"), - pytest.param(root_test_cases.class_init_flex_flow, id="class_init_flex_flow"), + pytest.param("csharp_test_project_basic", id="basic"), + pytest.param("csharp_test_project_basic_chat", id="basic_chat"), + pytest.param("csharp_test_project_function_mode_basic", id="function_mode_basic"), + pytest.param("csharp_test_project_class_init_flex_flow", id="class_init_flex_flow"), ], ) - def test_pf_run_create(self, test_case: TestCase): + def test_pf_run_create(self, request, target_fixture_name: str): + test_case: CSharpProject = request.getfixturevalue(target_fixture_name) cmd = [ "run", "create", "--flow", - test_case.flow_dir, + test_case["flow_dir"], "--data", - test_case.data, + test_case["data"], ] - if os.path.exists(test_case.init): - cmd.extend(["--init", test_case.init]) + if os.path.exists(test_case["init"]): + cmd.extend(["--init", test_case["init"]]) run_pf_command(*cmd) @pytest.mark.parametrize( - "test_case", + "target_fixture_name", [ - pytest.param(root_test_cases.basic, id="basic"), - pytest.param(root_test_cases.basic_chat, id="basic_chat"), - pytest.param(root_test_cases.function_mode_basic, id="function_mode_basic"), - pytest.param(root_test_cases.class_init_flex_flow, id="class_init_flex_flow"), + pytest.param("csharp_test_project_basic", id="basic"), + pytest.param("csharp_test_project_basic_chat", id="basic_chat"), + pytest.param("csharp_test_project_function_mode_basic", id="function_mode_basic"), + pytest.param("csharp_test_project_class_init_flex_flow", id="class_init_flex_flow"), ], ) - def test_pf_flow_test(self, test_case: TestCase): - with open(test_case.data, "r") as f: + def test_pf_flow_test(self, request, target_fixture_name: str): + test_case: CSharpProject = request.getfixturevalue(target_fixture_name) + with open(test_case["data"], "r") as f: lines = f.readlines() if len(lines) == 0: pytest.skip("No data provided for the test case.") @@ -147,7 +90,7 @@ def test_pf_flow_test(self, test_case: TestCase): "flow", "test", "--flow", - test_case.flow_dir, + test_case["flow_dir"], "--inputs", ] for key, value in inputs.items(): @@ -157,12 +100,12 @@ def test_pf_flow_test(self, test_case: TestCase): value = f'"{value}"' cmd.extend([f"{key}={value}"]) - if os.path.exists(test_case.init): - cmd.extend(["--init", test_case.init]) + if os.path.exists(test_case["init"]): + cmd.extend(["--init", test_case["init"]]) run_pf_command(*cmd) - def test_flow_chat(self, monkeypatch, capsys): - flow_dir = root_test_cases.basic_chat.flow_dir + def test_flow_chat(self, monkeypatch, capsys, csharp_test_project_basic_chat: CSharpProject): + flow_dir = csharp_test_project_basic_chat["flow_dir"] # mock user input with pop so make chat list reversed chat_list = ["what is chat gpt?", "hi"] @@ -192,32 +135,21 @@ def mock_input(*args, **kwargs): assert "Hello world round 1: what is chat gpt?" in outerr.out @pytest.mark.skip(reason="need to update the test case") - def test_pf_run_create_with_connection_override(self): + def test_pf_run_create_with_connection_override(self, csharp_test_project_basic): run_pf_command( "run", "create", "--flow", - root_test_cases.basic_with_builtin_llm.flow_dir, + csharp_test_project_basic["flow_dir"], "--data", - root_test_cases.basic_with_builtin_llm.data, + csharp_test_project_basic["data"], "--connections", "get_answer.connection=azure_open_ai_connection", ) @pytest.mark.skip(reason="need to update the test case") def test_flow_chat_ui_streaming(self): - """Note that this test won't pass. Instead, it will hang and pop up a web page for user input. - Leave it here for debugging purpose. - """ - # The test need to interact with user input in ui - flow_dir = f"{root_test_cases}\\examples\\BasicChatFlowWithBuiltinLLM\\bin\\Debug\\net6.0" - run_pf_command( - "flow", - "test", - "--flow", - flow_dir, - "--ui", - ) + pass @pytest.mark.skip(reason="need to update the test case") def test_flow_run_from_resume(self): diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py new file mode 100644 index 00000000000..0d66d21bd2e --- /dev/null +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py @@ -0,0 +1,55 @@ +from typing import TypedDict + +import pytest + +from promptflow._sdk._load_functions import load_flow +from promptflow._sdk._pf_client import PFClient + +_client = PFClient() + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +@pytest.mark.usefixtures( + "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" +) +@pytest.mark.sdk_test +@pytest.mark.e2etest +@pytest.mark.csharp +class TestCSharpSdk: + @pytest.mark.parametrize( + "expected_signature", + [ + pytest.param( + { + "init": {}, + "inputs": { + "language": {"default": "chinese", "type": "string"}, + "topic": {"default": "ocean", "type": "string"}, + }, + "outputs": {"output": {"type": "object"}}, + }, + id="function_mode_basic", + ), + pytest.param( + { + "init": {"connection": {"type": "AzureOpenAIConnection"}, "name": {"type": "string"}}, + "inputs": {"question": {"default": "What is Promptflow?", "type": "string"}}, + "outputs": {"output": {"type": "object"}}, + }, + id="class_init_flex_flow", + ), + ], + ) + def test_pf_run_create(self, request, expected_signature: dict): + test_case: CSharpProject = request.getfixturevalue(f"csharp_test_project_{request.node.callspec.id}") + flow = load_flow(test_case["flow_dir"]) + signature = _client.flows._infer_signature( + flow, + include_primitive_output=True, + ) + assert signature == expected_signature diff --git a/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py new file mode 100644 index 00000000000..e4c889509b0 --- /dev/null +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py @@ -0,0 +1,50 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +import pytest + +from ..utils import PFSOperations, check_activity_end_telemetry + + +@pytest.mark.csharp +@pytest.mark.e2etest +class TestCSharp: + def test_get_flow_yaml(self, pfs_op: PFSOperations, csharp_test_project_basic) -> None: + with check_activity_end_telemetry(expected_activities=[]): + flow_yaml_from_pfs = pfs_op.get_flow_yaml(flow_path=csharp_test_project_basic["flow_dir"]).data.decode( + "utf-8" + ) + assert flow_yaml_from_pfs == ( + "$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json\n" + "display_name: Basic\n" + "language: csharp\n" + "inputs:\n" + " question:\n" + " type: string\n" + " default: what is promptflow?\n" + "outputs:\n" + " answer:\n" + " type: string\n" + " reference: ${get_answer.output}\n" + "nodes:\n" + "- name: get_answer\n" + " type: csharp\n" + " source:\n" + " type: package\n" + " tool: (Basic)Basic.Flow.HelloWorld\n" + " inputs:\n" + " question: ${inputs.question}\n" + ) + + def test_get_eager_flow_yaml(self, pfs_op: PFSOperations, csharp_test_project_function_mode_basic) -> None: + with check_activity_end_telemetry(expected_activities=[]): + flow_yaml_from_pfs = pfs_op.get_flow_yaml( + flow_path=csharp_test_project_function_mode_basic["flow_dir"] + ).data.decode("utf-8") + assert flow_yaml_from_pfs == ( + "$schema: https://azuremlschemas.azureedge.net/promptflow/latest/flow.schema.json\n" + "\n" + "language: csharp\n" + "entry: (FunctionModeBasic)FunctionModeBasic.MyEntry.WritePoemReturnObjectAsync\n" + ) diff --git a/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml b/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml new file mode 100644 index 00000000000..c5cb0d2d2f1 --- /dev/null +++ b/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml @@ -0,0 +1,615 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '3678' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.033' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?api-version=2023-04-01-preview&count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1372' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.057' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore?api-version=2023-04-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.080' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2023-04-01-preview + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.148' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:03:59 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/data.jsonl + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '122' + content-md5: + - ADhs+/4nCwfVwpMYGRMDNQ== + content-type: + - application/octet-stream + last-modified: + - Fri, 19 Apr 2024 10:02:27 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Fri, 19 Apr 2024 10:02:26 GMT + x-ms-meta-name: + - 592b9e51-3a0b-4e5d-a6b5-ce54e92aaea5 + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - 1f2929ae-366f-4153-a8a5-609a2e769f07 + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:01 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/data.jsonl + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore?api-version=2023-04-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.072' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2023-04-01-preview + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.074' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:06 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/net6.0/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '2762' + content-md5: + - QymQjkUtk24EtHQGHDOiIQ== + content-type: + - application/octet-stream + last-modified: + - Fri, 19 Apr 2024 10:02:43 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Fri, 19 Apr 2024 10:02:37 GMT + x-ms-meta-name: + - 81d0ee87-a9db-4cb3-8c1e-b5eb42183fec + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - '1' + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:07 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/net6.0/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: '{"flowDefinitionDataStoreName": "workspaceblobstore", "flowDefinitionBlobPath": + "LocalUpload/000000000000000000000000000000000000/net6.0/flow.dag.yaml", "runId": + "name", "runDisplayName": "name", "runExperimentName": "", "sessionId": "000000000000000000000000000000000000000000000000", + "sessionSetupMode": "SystemWait", "flowLineageId": "0000000000000000000000000000000000000000000000000000000000000000", + "runDisplayNameGenerationType": "UserProvidedMacro", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/000000000000000000000000000000000000/data.jsonl"}, + "inputsMapping": {}, "environmentVariables": {}, "connections": {}, "runtimeName": + "fake-runtime-name"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '755' + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/submit + response: + body: + string: '"name"' + headers: + connection: + - keep-alive + content-length: + - '38' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '4.005' + status: + code: 200 + message: OK +- request: + body: '{"runId": "name", "selectRunMetadata": true, "selectRunDefinition": true, + "selectJobSpecification": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://eastus.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/rundata + response: + body: + string: '{"runMetadata": {"runNumber": 1713521054, "rootRunId": "name", "createdUtc": + "2024-04-19T10:04:14.0934449+00:00", "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "10032001D9C91417", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Xingzhi Zhang", "upn": + null}, "userId": "00000000-0000-0000-0000-000000000000", "token": null, "tokenExpiryTimeUtc": + null, "error": null, "warnings": null, "revision": 1, "statusRevision": 0, + "runUuid": "a656e9bc-91bf-427e-8950-e21ca2065d67", "parentRunUuid": null, + "rootRunUuid": "a656e9bc-91bf-427e-8950-e21ca2065d67", "lastStartTimeUtc": + null, "currentComputeTime": "00:00:00", "computeDuration": null, "effectiveStartTimeUtc": + null, "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "10032001D9C91417", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Xingzhi Zhang", "upn": + null}, "lastModifiedUtc": "2024-04-19T10:04:14.0934449+00:00", "duration": + null, "cancelationReason": null, "currentAttemptId": 1, "runId": "name", "parentRunId": + null, "experimentId": "36dad2bf-add3-491b-8ad4-1c85de215cfa", "status": "NotStarted", + "startTimeUtc": null, "endTimeUtc": null, "scheduleId": null, "displayName": + "name", "name": null, "dataContainerId": "dcid.name", "description": null, + "hidden": false, "runType": "azureml.promptflow.FlowRun", "runTypeV2": {"orchestrator": + null, "traits": [], "attribution": "PromptFlow", "computeType": null}, "properties": + {"azureml.promptflow.runtime_name": "automatic", "azureml.promptflow.input_data": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/07a51eb4af1ca3b9af2b51008b387aab/data.jsonl", + "azureml.promptflow.disable_trace": "false", "azureml.promptflow.session_id": + "c3329398d4d114830396f085268d91e6ccc4d57b70e80e55", "azureml.promptflow.definition_file_name": + "flow.dag.yaml", "azureml.promptflow.flow_lineage_id": "fbbdc468be255ca5fd167cf97724ad8446094632682e031eda2669d9604572a4", + "azureml.promptflow.flow_definition_datastore_name": "workspaceblobstore", + "azureml.promptflow.flow_definition_blob_path": "LocalUpload/56ddfd1ec40da6b8b0b2e65a99b55e56/net6.0/flow.dag.yaml", + "_azureml.evaluation_run": "promptflow.BatchRun"}, "parameters": {}, "actionUris": + {}, "scriptName": null, "target": null, "uniqueChildRunComputeTargets": [], + "tags": {}, "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": + [], "runDefinition": null, "jobSpecification": null, "primaryMetricName": + null, "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}, "runDefinition": + null, "jobSpecification": null, "systemSettings": null}' + headers: + connection: + - keep-alive + content-length: + - '3599' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.067' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/name + response: + body: + string: '{"flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/name/flowRuns/name", + "flowRunId": "name", "flowRunDisplayName": "name", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/07a51eb4af1ca3b9af2b51008b387aab/data.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/name/flow_artifacts", "sessionId": "c3329398d4d114830396f085268d91e6ccc4d57b70e80e55", + "studioPortalEndpoint": "https://ml.azure.com/runs/name?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '975' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.241' + status: + code: 200 + message: OK +version: 1