From 458ce307ba2796de498b96420f0fa53d9d63a057 Mon Sep 17 00:00:00 2001 From: Robben Wang <350053002@qq.com> Date: Mon, 22 Apr 2024 15:01:44 +0800 Subject: [PATCH 01/14] Add span cancel handling for flex flow and node execution (#2918) # Description Record exception and set error status for each span when cancel job. For otlp's source code, they only handle `Exception` but not `BaseException`, so we need this PR for `KeyboardInterrupt` https://github.com/open-telemetry/opentelemetry-python/blob/47d5ad7aae5aef31238ca66e55dc550b307c7b35/opentelemetry-api/src/opentelemetry/trace/__init__.py#L547 Handle root span's status in this PR: https://github.com/microsoft/promptflow/pull/2889 ![image](https://github.com/microsoft/promptflow/assets/17527303/44be9ba9-8d7b-4347-8bd6-f0d8049b82bf) # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --------- Co-authored-by: robbenwang --- .../promptflow/tracing/_trace.py | 22 +++++++++++---- .../tests/unittests/test_trace.py | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/promptflow-tracing/promptflow/tracing/_trace.py b/src/promptflow-tracing/promptflow/tracing/_trace.py index a77c1c3fbe3..1101b5bc487 100644 --- a/src/promptflow-tracing/promptflow/tracing/_trace.py +++ b/src/promptflow-tracing/promptflow/tracing/_trace.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import contextlib import functools import inspect import json @@ -13,7 +14,7 @@ import opentelemetry.trace as otel_trace from opentelemetry.sdk.trace import ReadableSpan -from opentelemetry.trace import Link +from opentelemetry.trace import Link, Span from opentelemetry.trace.span import NonRecordingSpan, format_trace_id from opentelemetry.trace.status import StatusCode @@ -27,6 +28,17 @@ IS_LEGACY_OPENAI = version("openai").startswith("0.") +@contextlib.contextmanager +def _record_keyboard_interrupt_to_span(span: Span): + try: + yield + except KeyboardInterrupt as ex: + if span.is_recording(): + span.record_exception(ex) + span.set_status(StatusCode.ERROR, "Execution cancelled.") + raise + + class TokenCollector: _lock = Lock() @@ -175,7 +187,7 @@ def traced_generator(original_span: ReadableSpan, inputs, generator): with otel_tracer.start_as_current_span( f"Iterated({original_span.name})", links=[link], - ) as span: + ) as span, _record_keyboard_interrupt_to_span(span): enrich_span_with_original_attributes(span, original_span.attributes) # Enrich the new span with input before generator iteration to prevent loss of input information. # The input is as an event within this span. @@ -199,7 +211,7 @@ async def traced_async_generator(original_span: ReadableSpan, inputs, generator) with otel_tracer.start_as_current_span( f"Iterated({original_span.name})", links=[link], - ) as span: + ) as span, _record_keyboard_interrupt_to_span(span): enrich_span_with_original_attributes(span, original_span.attributes) # Enrich the new span with input before generator iteration to prevent loss of input information. # The input is as an event within this span. @@ -376,7 +388,7 @@ async def wrapped(*args, **kwargs): span_name = get_node_name_from_context(used_for_span_name=True) or trace.name # need to get everytime to ensure tracer is latest otel_tracer = otel_trace.get_tracer("promptflow") - with otel_tracer.start_as_current_span(span_name) as span: + with otel_tracer.start_as_current_span(span_name) as span, _record_keyboard_interrupt_to_span(span): # Store otel trace id in context for correlation OperationContext.get_instance()["otel_trace_id"] = f"0x{format_trace_id(span.get_span_context().trace_id)}" enrich_span_with_trace(span, trace) @@ -442,7 +454,7 @@ def wrapped(*args, **kwargs): span_name = get_node_name_from_context(used_for_span_name=True) or trace.name # need to get everytime to ensure tracer is latest otel_tracer = otel_trace.get_tracer("promptflow") - with otel_tracer.start_as_current_span(span_name) as span: + with otel_tracer.start_as_current_span(span_name) as span, _record_keyboard_interrupt_to_span(span): # Store otel trace id in context for correlation OperationContext.get_instance()["otel_trace_id"] = f"0x{format_trace_id(span.get_span_context().trace_id)}" enrich_span_with_trace(span, trace) diff --git a/src/promptflow-tracing/tests/unittests/test_trace.py b/src/promptflow-tracing/tests/unittests/test_trace.py index c6616d1521a..43cdd06fa41 100644 --- a/src/promptflow-tracing/tests/unittests/test_trace.py +++ b/src/promptflow-tracing/tests/unittests/test_trace.py @@ -6,11 +6,13 @@ import opentelemetry import pytest from openai.types.create_embedding_response import CreateEmbeddingResponse, Embedding, Usage +from opentelemetry.trace.status import StatusCode from promptflow.tracing._experimental import enrich_prompt_template from promptflow.tracing._operation_context import OperationContext from promptflow.tracing._trace import ( TokenCollector, + _record_keyboard_interrupt_to_span, enrich_span_with_context, enrich_span_with_embedding, enrich_span_with_input, @@ -31,6 +33,9 @@ def __init__(self, span_context, parent=None, raise_exception_for_attr=False): self.raise_exception_for_attr = raise_exception_for_attr self.attributes = {} self.events = [] + self.status = None + self.description = None + self.exception = None def get_span_context(self): return self.span_context @@ -50,6 +55,16 @@ def set_attributes(self, attributes): def add_event(self, name: str, attributes=None, timestamp=None): self.events.append(MockEvent(name, attributes, timestamp)) + def set_status(self, status=None, description=None): + self.status = status + self.description = description + + def record_exception(self, exception): + self.exception = exception + + def is_recording(self): + return True + def __enter__(self): return self @@ -298,3 +313,16 @@ def test_set_enrich_prompt_template(): assert template == mock_span.attributes["prompt.template"] assert variables == json.loads(mock_span.attributes["prompt.variables"]) + + +@pytest.mark.unitests +def test_record_keyboard_interrupt_to_span(): + mock_span = MockSpan(MockSpanContext(1)) + try: + with _record_keyboard_interrupt_to_span(mock_span): + raise KeyboardInterrupt + except KeyboardInterrupt: + pass + assert mock_span.status == StatusCode.ERROR + assert "Execution cancelled" in mock_span.description + assert isinstance(mock_span.exception, KeyboardInterrupt) From 6d97a0d01f8c919aefb597f4501688d8de4c87d5 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Mon, 22 Apr 2024 15:18:58 +0800 Subject: [PATCH 02/14] [SDK] Support model config in class based flow test & batch run (#2850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. This pull request primarily focuses on enhancing the model configuration in the `promptflow` package. The changes introduce class methods to create model configurations from connections, add support for model configurations in the script executor, and include tests for these new features. The most important changes are as follows: Related spec: [Model config for promptflow SDK by luigiw · Pull Request #1147 · Azure/azureml_run_specification (github.com)](https://github.com/Azure/azureml_run_specification/pull/1147) Model Configuration Enhancements: * [`src/promptflow-core/promptflow/core/_model_configuration.py`](diffhunk://#diff-109c791ad8cfd402aba59a4dbe5cf688648cd20deff67efb249cfdf005795f75R1-R13): Added `from_connection` class methods to `ModelConfiguration`, `OpenAIModelConfiguration`, and `AzureOpenAIModelConfiguration` classes. This allows model configurations to be created from connection objects. [[1]](diffhunk://#diff-109c791ad8cfd402aba59a4dbe5cf688648cd20deff67efb249cfdf005795f75R1-R13) [[2]](diffhunk://#diff-109c791ad8cfd402aba59a4dbe5cf688648cd20deff67efb249cfdf005795f75R32-R43) [[3]](diffhunk://#diff-109c791ad8cfd402aba59a4dbe5cf688648cd20deff67efb249cfdf005795f75L39-R67) * [`src/promptflow-core/promptflow/core/_model_configuration.py`](diffhunk://#diff-109c791ad8cfd402aba59a4dbe5cf688648cd20deff67efb249cfdf005795f75R106-R108): Added `MODEL_CONFIG_NAMES` constant to hold the names of model configuration classes. Script Executor Changes: * [`src/promptflow-core/promptflow/executor/_script_executor.py`](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4R25): Imported `MODEL_CONFIG_NAMES` and added logic in `_resolve_init_kwargs` method to resolve model configuration parameters. This allows the script executor to support model configurations. [[1]](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4R25) [[2]](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4R222-R249) Test Additions: * [`src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py`](diffhunk://#diff-b125543ffc0d08b4dd6bf8d327df9bc6cfc3c58d3696fe031871d2346e11eb94R9-R19): Added tests for flows with model configurations, including a test for a scenario where the wrong connection type is used. [[1]](diffhunk://#diff-b125543ffc0d08b4dd6bf8d327df9bc6cfc3c58d3696fe031871d2346e11eb94R9-R19) [[2]](diffhunk://#diff-b125543ffc0d08b4dd6bf8d327df9bc6cfc3c58d3696fe031871d2346e11eb94R437-R486) New Flow for Testing: * [`src/promptflow/tests/test_configs/eager_flows/basic_model_config/class_with_model_config.py`](diffhunk://#diff-ced31a7acbdc54327f1c0df0d5e64b2e0d668752e1e90272de069e23fd9e6860R1-R44): Added a new flow `MyFlow` that takes model configurations as input. This is used in the new tests. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../e2etests/test_run_operations.py | 39 + .../unittests/test_flow_entity.py | 18 + .../promptflow/_core/tool_meta_generator.py | 14 +- .../promptflow/_utils/tool_utils.py | 15 +- .../promptflow/core/_model_configuration.py | 51 +- .../promptflow/executor/_errors.py | 4 + .../promptflow/executor/_script_executor.py | 65 +- src/promptflow-core/tests/conftest.py | 16 + .../tests/core/e2etests/test_eager_flow.py | 52 +- .../promptflow/_sdk/entities/_run.py | 14 +- .../_sdk/operations/_flow_operations.py | 2 +- .../promptflow/_sdk/schemas/_flow.py | 2 + .../sdk_cli_test/e2etests/test_flow_run.py | 45 +- .../sdk_cli_test/e2etests/test_flow_test.py | 72 +- ...lowRun_test_model_config_dict_in_init.yaml | 1093 +++++++++++++++++ ...FlowRun_test_model_config_obj_in_init.yaml | 1093 +++++++++++++++++ .../class_with_model_config.py | 47 + .../basic_model_config/flow.flex.yaml | 4 + .../basic_model_config/inputs.jsonl | 2 + .../basic_model_config/requirements.txt | 1 + .../eager_flows/basic_model_config/run.yaml | 5 + 21 files changed, 2613 insertions(+), 41 deletions(-) create mode 100644 src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_dict_in_init.yaml create mode 100644 src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_obj_in_init.yaml create mode 100644 src/promptflow/tests/test_configs/eager_flows/basic_model_config/class_with_model_config.py create mode 100644 src/promptflow/tests/test_configs/eager_flows/basic_model_config/flow.flex.yaml create mode 100644 src/promptflow/tests/test_configs/eager_flows/basic_model_config/inputs.jsonl create mode 100644 src/promptflow/tests/test_configs/eager_flows/basic_model_config/requirements.txt create mode 100644 src/promptflow/tests/test_configs/eager_flows/basic_model_config/run.yaml diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py index 82fe2a9a7b4..1a19efe8db7 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py @@ -37,6 +37,7 @@ ) from promptflow.azure._entities._flow import Flow from promptflow.azure._load_functions import load_flow +from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration from promptflow.exceptions import UserErrorException from promptflow.recording.record_mode import is_live @@ -1346,6 +1347,44 @@ def assert_func(details_dict): assert_batch_run_result(run, pf, assert_func) + @pytest.mark.skipif(not is_live(), reason="Content change in submission time which lead to recording issue.") + def test_model_config_obj_in_init(self, pf): + def assert_func(details_dict): + return details_dict["outputs.azure_open_ai_model_config_azure_endpoint"] != [None, None,] and details_dict[ + "outputs.azure_open_ai_model_config_connection" + ] == [None, None] + + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_model_config") + # init with model config object + config1 = AzureOpenAIModelConfiguration(azure_deployment="my_deployment", connection="azure_open_ai") + config2 = OpenAIModelConfiguration(model="my_model", base_url="fake_base_url") + run = pf.run( + flow=flow_path, + data=f"{EAGER_FLOWS_DIR}/basic_model_config/inputs.jsonl", + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert "azure_open_ai_model_config" in run.properties["azureml.promptflow.init_kwargs"] + assert_batch_run_result(run, pf, assert_func) + + @pytest.mark.skipif(not is_live(), reason="Content change in submission time which lead to recording issue.") + def test_model_config_dict_in_init(self, pf): + def assert_func(details_dict): + return details_dict["outputs.azure_open_ai_model_config_azure_endpoint"] != [None, None,] and details_dict[ + "outputs.azure_open_ai_model_config_connection" + ] == [None, None] + + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_model_config") + # init with model config dict + config1 = dict(azure_deployment="my_deployment", connection="azure_open_ai") + config2 = dict(model="my_model", base_url="fake_base_url") + run = pf.run( + flow=flow_path, + data=f"{EAGER_FLOWS_DIR}/basic_model_config/inputs.jsonl", + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert "azure_open_ai_model_config" in run.properties["azureml.promptflow.init_kwargs"] + assert_batch_run_result(run, pf, assert_func) + def assert_batch_run_result(run: Run, pf: PFClient, assert_func): run = pf.runs.stream(run) diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py index 91aed2fbe72..946f0d6117b 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_flow_entity.py @@ -307,3 +307,21 @@ def test_flex_flow_run_unsupported_types(self, exception_type, data, error_messa data=data, ) assert error_message in str(e.value) + + def test_model_config_resolve_signature(self): + Flow._resolve_signature( + code=Path(f"{EAGER_FLOWS_DIR}/basic_model_config"), + data={ + "entry": "class_with_model_config:MyFlow", + "init": { + "azure_open_ai_model_config": {"type": "AzureOpenAIModelConfiguration"}, + "open_ai_model_config": {"type": "OpenAIModelConfiguration"}, + }, + "inputs": {"func_input": {"type": "string"}}, + "outputs": { + "func_input": {"type": "string"}, + "obj_id": {"type": "string"}, + "obj_input": {"type": "string"}, + }, + }, + ) diff --git a/src/promptflow-core/promptflow/_core/tool_meta_generator.py b/src/promptflow-core/promptflow/_core/tool_meta_generator.py index e6f0dcc1432..537e147fbf6 100644 --- a/src/promptflow-core/promptflow/_core/tool_meta_generator.py +++ b/src/promptflow-core/promptflow/_core/tool_meta_generator.py @@ -47,6 +47,7 @@ resolve_complicated_type, ) from promptflow.contracts.tool import InputDefinition, Tool, ToolType, ValueType +from promptflow.core._model_configuration import MODEL_CONFIG_NAME_2_CLASS from promptflow.exceptions import ErrorTarget, UserErrorException @@ -160,6 +161,7 @@ def _parse_tool_from_function( gen_custom_type_conn=False, skip_prompt_template=False, include_outputs=False, + support_model_config=False, ): try: tool_type = getattr(f, "__type", None) or ToolType.PYTHON @@ -177,6 +179,7 @@ def _parse_tool_from_function( initialize_inputs=initialize_inputs, gen_custom_type_conn=gen_custom_type_conn, skip_prompt_template=skip_prompt_template, + support_model_config=support_model_config, ) except Exception as e: error_type_and_message = f"({e.__class__.__name__}) {e}" @@ -505,12 +508,15 @@ def generate_tool_meta_in_subprocess( return tool_dict, exception_dict -def validate_interface(ports, fields, tool_name, port_type): +def validate_interface(ports, fields, tool_name, port_type, support_model_config=False): """Validate if the interface are serializable.""" + supported_types = [dict, inspect.Signature.empty] + if support_model_config: + supported_types += list(MODEL_CONFIG_NAME_2_CLASS.values()) for k, v in ports.items(): if ValueType.OBJECT not in v.type: continue - if resolve_annotation(fields[k]) not in [dict, inspect.Signature.empty]: + if resolve_annotation(fields[k]) not in supported_types: raise BadFunctionInterface( message_format=( "Parse interface for tool '{tool_name}' failed: " @@ -553,13 +559,15 @@ def generate_flow_meta_dict_by_object(f, cls): # Include data in generated meta to avoid flow definition's fields(e.g. environment variable) missing. flow_meta = {} if cls: - init_tool = _parse_tool_from_function(cls.__init__, include_outputs=False) + # TODO(3129057): check if we can not depend on tools' util + init_tool = _parse_tool_from_function(cls.__init__, include_outputs=False, support_model_config=True) init_inputs = init_tool.inputs validate_interface( init_inputs, {k: v.annotation for k, v in inspect.signature(cls.__init__).parameters.items()}, tool.name, "input", + support_model_config=True, ) else: init_inputs = None diff --git a/src/promptflow-core/promptflow/_utils/tool_utils.py b/src/promptflow-core/promptflow/_utils/tool_utils.py index 47613bb87e9..a74d6117e68 100644 --- a/src/promptflow-core/promptflow/_utils/tool_utils.py +++ b/src/promptflow-core/promptflow/_utils/tool_utils.py @@ -17,6 +17,7 @@ from promptflow._core._errors import DuplicateToolMappingError from promptflow._utils.context_utils import _change_working_dir from promptflow._utils.utils import is_json_serializable +from promptflow.core._model_configuration import MODEL_CONFIG_NAME_2_CLASS from promptflow.exceptions import ErrorTarget, UserErrorException from ..contracts.tool import ( @@ -62,7 +63,7 @@ def resolve_annotation(anno) -> Union[str, list]: return args[0] if len(args) == 1 else args -def param_to_definition(param, gen_custom_type_conn=False) -> (InputDefinition, bool): +def param_to_definition(param, gen_custom_type_conn=False, support_model_config=False) -> (InputDefinition, bool): default_value = param.default # Get value type and enum from annotation value_type = resolve_annotation(param.annotation) @@ -104,6 +105,8 @@ def param_to_definition(param, gen_custom_type_conn=False) -> (InputDefinition, custom_connection_added = True typ.append(t.__name__) is_connection = True + elif support_model_config and value_type in MODEL_CONFIG_NAME_2_CLASS.values(): + typ = [value_type.__name__] else: typ = [ValueType.from_type(value_type)] @@ -137,7 +140,11 @@ def resolve_complicated_type(return_type): def function_to_interface( - f: Callable, initialize_inputs=None, gen_custom_type_conn=False, skip_prompt_template=False + f: Callable, + initialize_inputs=None, + gen_custom_type_conn=False, + skip_prompt_template=False, + support_model_config=False, ) -> tuple: sign = inspect.signature(f) all_inputs = {} @@ -163,7 +170,9 @@ def function_to_interface( if skip_prompt_template and value_type is PromptTemplate: # custom llm tool has prompt template as input, skip it continue - input_def, is_connection = param_to_definition(v, gen_custom_type_conn=gen_custom_type_conn) + input_def, is_connection = param_to_definition( + v, gen_custom_type_conn=gen_custom_type_conn, support_model_config=support_model_config + ) input_defs[k] = input_def if is_connection: connection_types.append(input_def.type) diff --git a/src/promptflow-core/promptflow/core/_model_configuration.py b/src/promptflow-core/promptflow/core/_model_configuration.py index b2bc5059443..1f1353dbc1f 100644 --- a/src/promptflow-core/promptflow/core/_model_configuration.py +++ b/src/promptflow-core/promptflow/core/_model_configuration.py @@ -1,12 +1,18 @@ +import abc from dataclasses import dataclass from typing import Union from promptflow._constants import ConnectionType +from promptflow.core._connection import AzureOpenAIConnection, OpenAIConnection from promptflow.core._errors import InvalidConnectionError class ModelConfiguration: - pass + @classmethod + @abc.abstractmethod + def from_connection(cls, connection, **kwargs): + """Create a model configuration from a connection.""" + pass @dataclass @@ -15,15 +21,30 @@ class AzureOpenAIModelConfiguration(ModelConfiguration): azure_endpoint: str = None api_version: str = None api_key: str = None - organization: str = None # connection and model configs are exclusive. connection: str = None def __post_init__(self): self._type = ConnectionType.AZURE_OPEN_AI - if any([self.azure_endpoint, self.api_key, self.api_version, self.organization]) and self.connection: + if any([self.azure_endpoint, self.api_key, self.api_version]) and self.connection: raise InvalidConnectionError("Cannot configure model config and connection at the same time.") + @classmethod + def from_connection(cls, connection: AzureOpenAIConnection, azure_deployment: str): + """Create a model configuration from an Azure OpenAI connection. + + :param connection: AzureOpenAI Connection object. + :type connection: promptflow.connections.AzureOpenAIConnection + :param azure_deployment: Azure deployment name. + :type azure_deployment: str + """ + return cls( + azure_deployment=azure_deployment, + azure_endpoint=connection.api_base, + api_version=connection.api_version, + api_key=connection.api_key, + ) + @dataclass class OpenAIModelConfiguration(ModelConfiguration): @@ -36,9 +57,25 @@ class OpenAIModelConfiguration(ModelConfiguration): def __post_init__(self): self._type = ConnectionType.OPEN_AI - if any([self.base_url, self.api_key, self.api_version, self.organization]) and self.connection: + if any([self.base_url, self.api_key, self.organization]) and self.connection: raise InvalidConnectionError("Cannot configure model config and connection at the same time.") + @classmethod + def from_connection(cls, connection: OpenAIConnection, model: str): + """Create a model configuration from an OpenAI connection. + + :param connection: OpenAI Connection object. + :type connection: promptflow.connections.OpenAIConnection + :param model: model name. + :type model: str + """ + return cls( + model=model, + base_url=connection.base_url, + api_key=connection.api_key, + organization=connection.organization, + ) + @dataclass class PromptyModelConfiguration: @@ -77,3 +114,9 @@ def __post_init__(self): self._model = self.configuration.model elif isinstance(self.configuration, AzureOpenAIModelConfiguration): self._model = self.configuration.azure_deployment + + +MODEL_CONFIG_NAME_2_CLASS = { + AzureOpenAIModelConfiguration.__name__: AzureOpenAIModelConfiguration, + OpenAIModelConfiguration.__name__: OpenAIModelConfiguration, +} diff --git a/src/promptflow-core/promptflow/executor/_errors.py b/src/promptflow-core/promptflow/executor/_errors.py index 51ccfcf88d1..83bcd6d822f 100644 --- a/src/promptflow-core/promptflow/executor/_errors.py +++ b/src/promptflow-core/promptflow/executor/_errors.py @@ -329,5 +329,9 @@ class InvalidFlexFlowEntry(ValidationException): pass +class InvalidModelConfigValueType(ValidationException): + pass + + class InvalidAggregationFunction(UserErrorException): pass diff --git a/src/promptflow-core/promptflow/executor/_script_executor.py b/src/promptflow-core/promptflow/executor/_script_executor.py index 6edbeb18526..f03d79368c2 100644 --- a/src/promptflow-core/promptflow/executor/_script_executor.py +++ b/src/promptflow-core/promptflow/executor/_script_executor.py @@ -24,8 +24,13 @@ from promptflow.contracts.flow import Flow from promptflow.contracts.tool import ConnectionType from promptflow.core import log_metric +from promptflow.core._model_configuration import ( + MODEL_CONFIG_NAME_2_CLASS, + AzureOpenAIModelConfiguration, + OpenAIModelConfiguration, +) from promptflow.exceptions import ErrorTarget -from promptflow.executor._errors import InvalidFlexFlowEntry +from promptflow.executor._errors import InvalidFlexFlowEntry, InvalidModelConfigValueType from promptflow.executor._result import AggregationResult, LineResult from promptflow.storage import AbstractRunStorage from promptflow.storage._run_storage import DefaultRunStorage @@ -268,18 +273,71 @@ def get_inputs_definition(self): def _resolve_init_kwargs(self, c: type, init_kwargs: dict): """Resolve init kwargs, the connection names will be resolved to connection objects.""" + logger.debug(f"Resolving init kwargs: {init_kwargs.keys()}.") sig = inspect.signature(c.__init__) connection_params = [] + model_config_param_name_2_cls = {} + # TODO(3117908): support connection & model config from YAML signature. for key, param in sig.parameters.items(): if ConnectionType.is_connection_class_name(param.annotation.__name__): connection_params.append(key) - if not connection_params: + elif param.annotation.__name__ in MODEL_CONFIG_NAME_2_CLASS.keys(): + model_config_param_name_2_cls[key] = MODEL_CONFIG_NAME_2_CLASS[param.annotation.__name__] + if not connection_params and not model_config_param_name_2_cls: return init_kwargs resolved_init_kwargs = {k: v for k, v in init_kwargs.items()} + if connection_params: + self._resolve_connection_params( + connection_params=connection_params, init_kwargs=init_kwargs, resolved_init_kwargs=resolved_init_kwargs + ) + if model_config_param_name_2_cls: + self._resolve_model_config_params( + model_config_param_name_2_cls=model_config_param_name_2_cls, + init_kwargs=init_kwargs, + resolved_init_kwargs=resolved_init_kwargs, + ) + + return resolved_init_kwargs + + @classmethod + def _resolve_connection_params(cls, connection_params: list, init_kwargs: dict, resolved_init_kwargs: dict): provider = ConnectionProvider.get_instance() + # parse connection + logger.debug(f"Resolving connection params: {connection_params}") for key in connection_params: resolved_init_kwargs[key] = provider.get(init_kwargs[key]) - return resolved_init_kwargs + + @classmethod + def _resolve_model_config_params( + cls, model_config_param_name_2_cls: dict, init_kwargs: dict, resolved_init_kwargs: dict + ): + # parse model config + logger.debug(f"Resolving model config params: {model_config_param_name_2_cls}") + for key, model_config_cls in model_config_param_name_2_cls.items(): + model_config_val = init_kwargs[key] + if isinstance(model_config_val, dict): + logger.debug(f"Recovering model config object from dict: {model_config_val}.") + model_config_val = model_config_cls(**model_config_val) + if not isinstance(model_config_val, model_config_cls): + raise InvalidModelConfigValueType( + message_format="Model config value is not an instance of {model_config_cls}, got {value_type}", + model_config_cls=model_config_cls, + value_type=type(model_config_val), + ) + if getattr(model_config_val, "connection", None): + logger.debug(f"Getting connection {model_config_val.connection} for model config.") + provider = ConnectionProvider.get_instance() + connection_obj = provider.get(model_config_val.connection) + + if isinstance(model_config_val, AzureOpenAIModelConfiguration): + model_config_val = AzureOpenAIModelConfiguration.from_connection( + connection=connection_obj, azure_deployment=model_config_val.azure_deployment + ) + elif isinstance(model_config_val, OpenAIModelConfiguration): + model_config_val = OpenAIModelConfiguration.from_connection( + connection=connection_obj, model=model_config_val.model + ) + resolved_init_kwargs[key] = model_config_val @property def is_function_entry(self): @@ -311,6 +369,7 @@ def _parse_entry_func(self): resolved_init_kwargs = self._resolve_init_kwargs(func, self._init_kwargs) obj = func(**resolved_init_kwargs) except Exception as e: + # TODO: scrub secrets in init kwarg values raise FlowEntryInitializationError(init_kwargs=self._init_kwargs, ex=e) from e func = getattr(obj, "__call__") self._initialize_aggr_function(obj) diff --git a/src/promptflow-core/tests/conftest.py b/src/promptflow-core/tests/conftest.py index dc895b3cf87..026fc330c9c 100644 --- a/src/promptflow-core/tests/conftest.py +++ b/src/promptflow-core/tests/conftest.py @@ -1,9 +1,11 @@ import json from pathlib import Path +from unittest.mock import patch import pytest from promptflow._utils.flow_utils import resolve_flow_path +from promptflow.core._connection_provider._dict_connection_provider import DictConnectionProvider TEST_CONFIG_ROOT = Path(__file__).parent.parent.parent / "promptflow" / "tests" / "test_configs" FLOW_ROOT = TEST_CONFIG_ROOT / "flows" @@ -30,3 +32,17 @@ def get_yaml_file(folder_name, root: str = FLOW_ROOT, file_name: str = None) -> def dev_connections() -> dict: with open(CONNECTION_FILE, "r") as f: return json.load(f) + + +@pytest.fixture +def mock_dict_azure_open_ai_connection(dev_connections): + connection = dev_connections["azure_open_ai_connection"] + # TODO(3128519): Remove this after the connection type is added to github secrets + if "type" not in connection: + connection["type"] = "AzureOpenAIConnection" + + with patch( + "promptflow.connections.ConnectionProvider.get_instance", + return_value=DictConnectionProvider({"azure_open_ai_connection": connection}), + ): + yield diff --git a/src/promptflow-core/tests/core/e2etests/test_eager_flow.py b/src/promptflow-core/tests/core/e2etests/test_eager_flow.py index 81914e5cea9..c8e3fe98d5b 100644 --- a/src/promptflow-core/tests/core/e2etests/test_eager_flow.py +++ b/src/promptflow-core/tests/core/e2etests/test_eager_flow.py @@ -1,12 +1,11 @@ import asyncio from dataclasses import is_dataclass -from unittest.mock import patch import pytest from promptflow._core.tool_meta_generator import PythonLoadError from promptflow.contracts.run_info import Status -from promptflow.core._connection_provider._dict_connection_provider import DictConnectionProvider +from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration from promptflow.executor._errors import FlowEntryInitializationError, InvalidFlexFlowEntry from promptflow.executor._result import LineResult from promptflow.executor._script_executor import ScriptExecutor @@ -65,9 +64,32 @@ class TestEagerFlow: lambda x: x["func_input"] == "func_input", {"obj_input": "obj_input"}, ), + ( + "basic_model_config", + {"func_input": "input"}, + lambda x: x["azure_open_ai_model_config_azure_endpoint"] == "fake_endpoint", + { + "azure_open_ai_model_config": AzureOpenAIModelConfiguration( + azure_deployment="my_deployment", azure_endpoint="fake_endpoint" + ), + "open_ai_model_config": OpenAIModelConfiguration(model="my_model", base_url="fake_base_url"), + }, + ), + ( + "basic_model_config", + {"func_input": "input"}, + lambda x: x["azure_open_ai_model_config_azure_endpoint"] is not None + and x["open_ai_model_config_connection"] is None, + { + "azure_open_ai_model_config": AzureOpenAIModelConfiguration( + azure_deployment="my_deployment", connection="azure_open_ai_connection" + ), + "open_ai_model_config": OpenAIModelConfiguration(model="my_model", base_url="fake_base_url"), + }, + ), ], ) - def test_flow_run(self, flow_folder, inputs, ensure_output, init_kwargs): + def test_flow_run(self, flow_folder, inputs, ensure_output, init_kwargs, mock_dict_azure_open_ai_connection): flow_file = get_yaml_file(flow_folder, root=EAGER_FLOW_ROOT) # Test submitting eager flow to script executor @@ -90,23 +112,15 @@ def test_flow_run(self, flow_folder, inputs, ensure_output, init_kwargs): line_result2 = executor.exec_line(inputs=inputs, index=0) assert line_result1.output == line_result2.output - def test_flow_run_with_openai_chat(self, dev_connections): + def test_flow_run_with_openai_chat(self, mock_dict_azure_open_ai_connection): flow_file = get_yaml_file("callable_class_with_openai", root=EAGER_FLOW_ROOT, file_name="flow.flex.yaml") - connection = dev_connections["azure_open_ai_connection"] - # TODO: Remove this after the connection type is added to github secrets - if "type" not in connection: - connection["type"] = "AzureOpenAIConnection" - - with patch( - "promptflow.connections.ConnectionProvider.get_instance", - return_value=DictConnectionProvider({"azure_open_ai_connection": connection}), - ): - executor = ScriptExecutor(flow_file=flow_file, init_kwargs={"connection": "azure_open_ai_connection"}) - line_result = executor.exec_line(inputs={"question": "Hello", "stream": False}, index=0) - assert line_result.run_info.status == Status.Completed, line_result.run_info.error - token_names = ["prompt_tokens", "completion_tokens", "total_tokens"] - for token_name in token_names: - assert token_name in line_result.run_info.api_calls[0]["children"][0]["system_metrics"] + + executor = ScriptExecutor(flow_file=flow_file, init_kwargs={"connection": "azure_open_ai_connection"}) + line_result = executor.exec_line(inputs={"question": "Hello", "stream": False}, index=0) + assert line_result.run_info.status == Status.Completed, line_result.run_info.error + token_names = ["prompt_tokens", "completion_tokens", "total_tokens"] + for token_name in token_names: + assert token_name in line_result.run_info.api_calls[0]["children"][0]["system_metrics"] @pytest.mark.parametrize("entry, inputs, expected_output", function_entries) def test_flow_run_with_function_entry(self, entry, inputs, expected_output): diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py index 63c0bf86dee..b504e3dea3a 100644 --- a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py +++ b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py @@ -1,11 +1,12 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- - +import dataclasses import datetime import functools import json import uuid +from dataclasses import asdict from os import PathLike from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -376,7 +377,7 @@ def _to_orm_object(self) -> ORMRun: display_name=display_name, description=self.description, tags=json.dumps(self.tags) if self.tags else None, - properties=json.dumps(self.properties), + properties=json.dumps(self.properties, default=asdict), data=Path(self.data).resolve().absolute().as_posix() if self.data else None, run_source=self._run_source, ) @@ -533,6 +534,13 @@ def _get_flow_dir(self) -> Path: def _get_schema_cls(self): return RunSchema + @classmethod + def _to_rest_init(cls, init): + """Convert init to rest object.""" + if not init: + return None + return {k: asdict(v) if dataclasses.is_dataclass(v) else v for k, v in init.items()} + def _to_rest_object(self): try: from azure.ai.ml._utils._storage_utils import AzureMLDatastorePathUri @@ -616,7 +624,7 @@ def _to_rest_object(self): compute_name=compute_name, identity=identity_resource_id, enable_multi_container=is_multi_container_enabled(), - init_k_wargs=self.init, + init_k_wargs=self._to_rest_init(self.init), ) # use when uploading a local existing run to cloud diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py index 7025a9ad179..04483203ff5 100644 --- a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py +++ b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py @@ -1306,10 +1306,10 @@ def _update_signatures(self, code: Path, data: dict) -> bool: include_primitive_output=True, ) merged_signatures = self._merge_signature(extracted=signatures, signature_overrides=data) - FlexFlow(path=code / FLOW_FLEX_YAML, code=code, data=data, entry=entry)._validate(raise_error=True) updated = False for field in ["inputs", "outputs", "init"]: if merged_signatures.get(field) != data.get(field): updated = True data.update(merged_signatures) + FlexFlow(path=code / FLOW_FLEX_YAML, code=code, data=data, entry=entry)._validate(raise_error=True) return updated diff --git a/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py b/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py index c64cf2cc3a3..05ec83d48fe 100644 --- a/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py +++ b/src/promptflow-devkit/promptflow/_sdk/schemas/_flow.py @@ -10,6 +10,7 @@ from promptflow._sdk.schemas._base import PatchedSchemaMeta, YamlFileSchema from promptflow._sdk.schemas._fields import NestedField from promptflow.contracts.tool import ValueType +from promptflow.core._model_configuration import MODEL_CONFIG_NAME_2_CLASS class FlowInputSchema(metaclass=PatchedSchemaMeta): @@ -87,6 +88,7 @@ class FlexFlowInitSchema(FlowInputSchema): + list( map(lambda x: f"{x.value}Connection", filter(lambda x: x != ConnectionType._NOT_SET, ConnectionType)) ) + + list(MODEL_CONFIG_NAME_2_CLASS.keys()) ), ) diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py index a6f6757dcc5..9a4ca44580c 100644 --- a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py @@ -38,6 +38,7 @@ from promptflow._utils.yaml_utils import load_yaml from promptflow.client import PFClient from promptflow.connections import AzureOpenAIConnection +from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration from promptflow.exceptions import UserErrorException TEST_ROOT = PROMPTFLOW_ROOT / "tests" @@ -1708,8 +1709,6 @@ def assert_func(details_dict): # set code folder to avoid snapshot too big code=f"{EAGER_FLOWS_DIR}/basic_callable_class", ) - assert run.status == "Completed" - assert "error" not in run._to_dict() assert_batch_run_result(run, pf, assert_func) @@ -1719,11 +1718,49 @@ def assert_func(details_dict): # set code folder to avoid snapshot too big code=f"{EAGER_FLOWS_DIR}/basic_callable_class", ) - assert run.status == "Completed" - assert "error" not in run._to_dict() assert_batch_run_result(run, pf, assert_func) + def test_model_config_in_init(self, pf): + def assert_func(details_dict): + keys_to_omit = [ + "inputs.line_number", + "inputs.func_input", + "outputs.func_input", + "outputs.obj_id", + "outputs.azure_open_ai_model_config_azure_endpoint", + ] + omitted_output = {k: v for k, v in details_dict.items() if k not in keys_to_omit} + return omitted_output == { + "outputs.azure_open_ai_model_config_deployment": ["my_deployment", "my_deployment"], + "outputs.azure_open_ai_model_config_connection": ["(Failed)", "(Failed)"], + "outputs.open_ai_model_config_model": ["my_model", "my_model"], + "outputs.open_ai_model_config_base_url": ["fake_base_url", "fake_base_url"], + "outputs.open_ai_model_config_connection": ["(Failed)", "(Failed)"], + } + + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_model_config") + + # init with model config object + config1 = AzureOpenAIModelConfiguration(azure_deployment="my_deployment", connection="azure_open_ai_connection") + config2 = OpenAIModelConfiguration(model="my_model", base_url="fake_base_url") + run = pf.run( + flow=flow_path, + data=f"{EAGER_FLOWS_DIR}/basic_model_config/inputs.jsonl", + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert_batch_run_result(run, pf, assert_func) + + # init with model config dict + config1 = dict(azure_deployment="my_deployment", connection="azure_open_ai_connection") + config2 = dict(model="my_model", base_url="fake_base_url") + run = pf.run( + flow=flow_path, + data=f"{EAGER_FLOWS_DIR}/basic_model_config/inputs.jsonl", + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert_batch_run_result(run, pf, assert_func) + def assert_batch_run_result(run: Run, pf: PFClient, assert_func): assert run.status == "Completed" diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py index ecda3ebbb51..310df87c01e 100644 --- a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_test.py @@ -6,14 +6,17 @@ from types import GeneratorType import papermill +import pydash import pytest from _constants import PROMPTFLOW_ROOT from marshmallow import ValidationError from promptflow._sdk._constants import LOGGER_NAME from promptflow._sdk._pf_client import PFClient +from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration from promptflow.core._utils import init_executable from promptflow.exceptions import UserErrorException +from promptflow.executor._errors import FlowEntryInitializationError TEST_ROOT = PROMPTFLOW_ROOT / "tests" CONNECTION_FILE = (PROMPTFLOW_ROOT / "connections.json").resolve().absolute().as_posix() @@ -360,6 +363,23 @@ def test_eager_flow_with_evc(self): "inputs": {"input_val": {"default": "gpt", "type": "string"}}, }, ), + ( + "basic_model_config", + { + "init": { + "azure_open_ai_model_config": {"type": "AzureOpenAIModelConfiguration"}, + "open_ai_model_config": {"type": "OpenAIModelConfiguration"}, + }, + "inputs": {"func_input": {"type": "string"}}, + "outputs": { + "func_input": {"type": "string"}, + "obj_id": {"type": "string"}, + "obj_input": {"type": "string"}, + }, + "entry": "class_with_model_config:MyFlow", + "function": "__call__", + }, + ), ], ) def test_generate_flow_meta(self, flow_path, expected_meta): @@ -367,7 +387,8 @@ def test_generate_flow_meta(self, flow_path, expected_meta): clear_module_cache("my_module.entry") flow_path = Path(f"{EAGER_FLOWS_DIR}/{flow_path}").absolute() flow_meta = _client._flows._generate_flow_meta(flow_path) - assert flow_meta == expected_meta + omitted_meta = pydash.omit(flow_meta, "environment") + assert omitted_meta == expected_meta def test_generate_flow_meta_exception(self): flow_path = Path(f"{EAGER_FLOWS_DIR}/incorrect_entry/").absolute() @@ -431,3 +452,52 @@ def test_flex_flow_with_init(self, pf): result2 = pf.test(flow=flow_path, inputs={"func_input": "input"}, init={"obj_input": "val"}) assert result2["func_input"] == "input" assert result1["obj_id"] != result2["obj_id"] + + def test_flex_flow_with_model_config(self, pf): + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_model_config") + config1 = AzureOpenAIModelConfiguration(azure_deployment="my_deployment", azure_endpoint="fake_endpoint") + config2 = OpenAIModelConfiguration(model="my_model", base_url="fake_base_url") + result1 = pf.test( + flow=flow_path, + inputs={"func_input": "input"}, + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert pydash.omit(result1, "obj_id") == { + "azure_open_ai_model_config_azure_endpoint": "fake_endpoint", + "azure_open_ai_model_config_connection": None, + "azure_open_ai_model_config_deployment": "my_deployment", + "func_input": "input", + "open_ai_model_config_base_url": "fake_base_url", + "open_ai_model_config_connection": None, + "open_ai_model_config_model": "my_model", + } + + config1 = AzureOpenAIModelConfiguration(azure_deployment="my_deployment", connection="azure_open_ai_connection") + config2 = OpenAIModelConfiguration(model="my_model", base_url="fake_base_url") + result2 = pf.test( + flow=flow_path, + inputs={"func_input": "input"}, + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert pydash.omit(result2, "obj_id", "azure_open_ai_model_config_azure_endpoint") == { + "azure_open_ai_model_config_connection": None, + "azure_open_ai_model_config_deployment": "my_deployment", + "func_input": "input", + "open_ai_model_config_base_url": "fake_base_url", + "open_ai_model_config_connection": None, + "open_ai_model_config_model": "my_model", + } + assert result1["obj_id"] != result2["obj_id"] + + def test_model_config_wrong_connection_type(self, pf): + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_model_config") + config1 = AzureOpenAIModelConfiguration(azure_deployment="my_deployment", azure_endpoint="fake_endpoint") + # using azure open ai connection to initialize open ai model config + config2 = OpenAIModelConfiguration(model="my_model", connection="azure_open_ai_connection") + with pytest.raises(FlowEntryInitializationError) as e: + pf.test( + flow=flow_path, + inputs={"func_input": "input"}, + init={"azure_open_ai_model_config": config1, "open_ai_model_config": config2}, + ) + assert "'AzureOpenAIConnection' object has no attribute 'base_url'" in str(e.value) diff --git a/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_dict_in_init.yaml b/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_dict_in_init.yaml new file mode 100644 index 00000000000..1020ea581b7 --- /dev/null +++ b/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_dict_in_init.yaml @@ -0,0 +1,1093 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '3678' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.046' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1372' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.074' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.087' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.114' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:16:08 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/inputs.jsonl + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '60' + content-md5: + - tiCwcZnEReXDaVrgfN3qAQ== + content-type: + - application/octet-stream + last-modified: + - Thu, 18 Apr 2024 06:39:03 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Thu, 18 Apr 2024 06:39:02 GMT + x-ms-meta-name: + - b71e395a-c049-4e64-8360-c72dd242006d + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - ce6436cb-e73a-4d3a-8a12-774ed1e58516 + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:16:12 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/inputs.jsonl + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.101' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.102' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:16:21 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/basic_model_config/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '1731' + content-md5: + - uep0kZH5H8Dt1nyxSWHg7Q== + content-type: + - application/octet-stream + last-modified: + - Thu, 18 Apr 2024 07:56:09 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Thu, 18 Apr 2024 07:56:08 GMT + x-ms-meta-name: + - c4a517e8-b572-4b41-beb7-1c44702214bc + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - '1' + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:16:24 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/basic_model_config/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: '{"flowDefinitionDataStoreName": "workspaceblobstore", "flowDefinitionBlobPath": + "LocalUpload/000000000000000000000000000000000000/basic_model_config/flow.flex.yaml", + "runId": "basic_model_config_variant_0_20240418_161558_474262", "runDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "runExperimentName": + "", "sessionId": "000000000000000000000000000000000000000000000000", "sessionSetupMode": + "SystemWait", "flowLineageId": "0000000000000000000000000000000000000000000000000000000000000000", + "runDisplayNameGenerationType": "UserProvidedMacro", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/000000000000000000000000000000000000/inputs.jsonl"}, + "inputsMapping": {}, "environmentVariables": {}, "initKWargs": {"azure_open_ai_model_config": + {"azure_deployment": "my_deployment", "connection": "azure_open_ai"}, "open_ai_model_config": + {"model": "my_model", "base_url": "fake_base_url"}}, "connections": {}, "runtimeName": + "fake-runtime-name"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1004' + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/submit + response: + body: + string: '"basic_model_config_variant_0_20240418_161558_474262"' + headers: + connection: + - keep-alive + content-length: + - '53' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '4.245' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161558_474262/flowRuns/basic_model_config_variant_0_20240418_161558_474262", + "flowRunId": "basic_model_config_variant_0_20240418_161558_474262", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "93696879-cef6-477f-9ca9-2013d3161f5d", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161558_474262?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.182' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161558_474262/flowRuns/basic_model_config_variant_0_20240418_161558_474262", + "flowRunId": "basic_model_config_variant_0_20240418_161558_474262", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "93696879-cef6-477f-9ca9-2013d3161f5d", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161558_474262?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.168' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161558_474262/flowRuns/basic_model_config_variant_0_20240418_161558_474262", + "flowRunId": "basic_model_config_variant_0_20240418_161558_474262", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "93696879-cef6-477f-9ca9-2013d3161f5d", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161558_474262?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.222' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161558_474262/flowRuns/basic_model_config_variant_0_20240418_161558_474262", + "flowRunId": "basic_model_config_variant_0_20240418_161558_474262", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "93696879-cef6-477f-9ca9-2013d3161f5d", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161558_474262?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.170' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161558_474262/flowRuns/basic_model_config_variant_0_20240418_161558_474262", + "flowRunId": "basic_model_config_variant_0_20240418_161558_474262", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161558_474262", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "93696879-cef6-477f-9ca9-2013d3161f5d", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161558_474262?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.171' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262/childRuns?endIndex=24&startIndex=0 + response: + body: + string: '[{"run_id": "basic_model_config_variant_0_20240418_161558_474262_0", + "status": "Completed", "error": null, "inputs": {"func_input": "func_input1", + "line_number": 0}, "output": {"azure_open_ai_model_config_deployment": "my_deployment", + "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input1", "obj_id": 139647551047376}, "metrics": + null, "request": null, "parent_run_id": "basic_model_config_variant_0_20240418_161558_474262", + "root_run_id": "basic_model_config_variant_0_20240418_161558_474262", "source_run_id": + null, "flow_id": "default_flow_id", "start_time": "2024-04-18T08:17:01.458005Z", + "end_time": "2024-04-18T08:17:01.485959Z", "index": 0, "api_calls": [{"name": + "MyFlow.__call__", "type": "Function", "inputs": {"func_input": "func_input1"}, + "output": {"azure_open_ai_model_config_deployment": "my_deployment", "azure_open_ai_model_config_azure_endpoint": + "https://gpt-test-eus.openai.azure.com/", "azure_open_ai_model_config_connection": + null, "open_ai_model_config_model": "my_model", "open_ai_model_config_base_url": + "fake_base_url", "open_ai_model_config_connection": null, "func_input": "func_input1", + "obj_id": 139647551047376}, "start_time": 1713428221.459362, "end_time": 1713428221.485111, + "error": null, "children": [], "node_name": null, "parent_id": "", "id": "f8c68398-5abf-4704-bb8c-3ca81a2cabc4", + "function": "MyFlow.__call__"}], "name": "", "description": "", "tags": null, + "system_metrics": {"duration": 0.027954}, "result": {"azure_open_ai_model_config_deployment": + "my_deployment", "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input1", "obj_id": 139647551047376}, "upload_metrics": + false, "otel_trace_id": "a5beda712fa7223ab646f404fbf3d447", "message_format": + "basic"}, {"run_id": "basic_model_config_variant_0_20240418_161558_474262_1", + "status": "Completed", "error": null, "inputs": {"func_input": "func_input2", + "line_number": 1}, "output": {"azure_open_ai_model_config_deployment": "my_deployment", + "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input2", "obj_id": 139647551047376}, "metrics": + null, "request": null, "parent_run_id": "basic_model_config_variant_0_20240418_161558_474262", + "root_run_id": "basic_model_config_variant_0_20240418_161558_474262", "source_run_id": + null, "flow_id": "default_flow_id", "start_time": "2024-04-18T08:17:04.576501Z", + "end_time": "2024-04-18T08:17:04.603291Z", "index": 1, "api_calls": [{"name": + "MyFlow.__call__", "type": "Function", "inputs": {"func_input": "func_input2"}, + "output": {"azure_open_ai_model_config_deployment": "my_deployment", "azure_open_ai_model_config_azure_endpoint": + "https://gpt-test-eus.openai.azure.com/", "azure_open_ai_model_config_connection": + null, "open_ai_model_config_model": "my_model", "open_ai_model_config_base_url": + "fake_base_url", "open_ai_model_config_connection": null, "func_input": "func_input2", + "obj_id": 139647551047376}, "start_time": 1713428224.577688, "end_time": 1713428224.602491, + "error": null, "children": [], "node_name": null, "parent_id": "", "id": "a23ee8b6-d204-497c-9242-b18926bae0be", + "function": "MyFlow.__call__"}], "name": "", "description": "", "tags": null, + "system_metrics": {"duration": 0.02679}, "result": {"azure_open_ai_model_config_deployment": + "my_deployment", "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input2", "obj_id": 139647551047376}, "upload_metrics": + false, "otel_trace_id": "2e8e509655715a15d493587ec8a3fe9f", "message_format": + "basic"}]' + headers: + connection: + - keep-alive + content-length: + - '4120' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.471' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262/childRuns?endIndex=49&startIndex=25 + response: + body: + string: '[]' + headers: + connection: + - keep-alive + content-length: + - '2' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.590' + status: + code: 200 + message: OK +- request: + body: '{"runId": "basic_model_config_variant_0_20240418_161558_474262", "selectRunMetadata": + true, "selectRunDefinition": true, "selectJobSpecification": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '152' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://eastus.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/rundata + response: + body: + string: '{"runMetadata": {"runNumber": 1713428200, "rootRunId": "basic_model_config_variant_0_20240418_161558_474262", + "createdUtc": "2024-04-18T08:16:40.1541935+00:00", "createdBy": {"userObjectId": + "00000000-0000-0000-0000-000000000000", "userPuId": "100320005227D154", "userIdp": + null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Han Wang", + "upn": null}, "userId": "00000000-0000-0000-0000-000000000000", "token": null, + "tokenExpiryTimeUtc": null, "error": null, "warnings": null, "revision": 7, + "statusRevision": 3, "runUuid": "4f9399c4-1c01-4b6d-bcb6-63a032e68a41", "parentRunUuid": + null, "rootRunUuid": "4f9399c4-1c01-4b6d-bcb6-63a032e68a41", "lastStartTimeUtc": + null, "currentComputeTime": null, "computeDuration": "00:00:16.3246911", "effectiveStartTimeUtc": + null, "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "100320005227D154", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Han Wang", "upn": "username@microsoft.com"}, + "lastModifiedUtc": "2024-04-18T08:17:08.1123593+00:00", "duration": "00:00:16.3246911", + "cancelationReason": null, "currentAttemptId": 1, "runId": "basic_model_config_variant_0_20240418_161558_474262", + "parentRunId": null, "experimentId": "d80a6560-645c-446b-a505-eb0cf106d0b7", + "status": "Completed", "startTimeUtc": "2024-04-18T08:16:52.09787+00:00", + "endTimeUtc": "2024-04-18T08:17:08.4225611+00:00", "scheduleId": null, "displayName": + "basic_model_config_variant_0_20240418_161558_474262", "name": null, "dataContainerId": + "dcid.basic_model_config_variant_0_20240418_161558_474262", "description": + null, "hidden": false, "runType": "azureml.promptflow.FlowRun", "runTypeV2": + {"orchestrator": null, "traits": [], "attribution": "PromptFlow", "computeType": + null}, "properties": {"azureml.promptflow.runtime_name": "automatic", "azureml.promptflow.input_data": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl", + "azureml.promptflow.init_kwargs": "{\"azure_open_ai_model_config\":{\"azure_deployment\":\"my_deployment\",\"connection\":\"azure_open_ai\"},\"open_ai_model_config\":{\"model\":\"my_model\",\"base_url\":\"fake_base_url\"}}", + "azureml.promptflow.disable_trace": "false", "azureml.promptflow.session_id": + "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "azureml.promptflow.definition_file_name": + "flow.flex.yaml", "azureml.promptflow.flow_lineage_id": "3f23fa85b23e79109dded73208030690bce5daac03c90aa3b9776c17e5e47d11", + "azureml.promptflow.flow_definition_datastore_name": "workspaceblobstore", + "azureml.promptflow.flow_definition_blob_path": "LocalUpload/f9695f65269ef735088e55e21754dcdc/basic_model_config/flow.flex.yaml", + "_azureml.evaluation_run": "promptflow.BatchRun", "azureml.promptflow.snapshot_id": + "93696879-cef6-477f-9ca9-2013d3161f5d", "azureml.promptflow.run_mode": "Eager", + "azureml.promptflow.runtime_version": "pr-1316971-v19", "azureml.promptflow.total_tokens": + "0", "_azureml.evaluate_artifacts": "[{\"path\": \"instance_results.jsonl\", + \"type\": \"table\"}]"}, "parameters": {}, "actionUris": {}, "scriptName": + null, "target": null, "uniqueChildRunComputeTargets": [], "tags": {}, "settings": + {}, "services": {}, "inputDatasets": [], "outputDatasets": [], "runDefinition": + null, "jobSpecification": null, "primaryMetricName": null, "createdFrom": + null, "cancelUri": null, "completeUri": null, "diagnosticsUri": null, "computeRequest": + null, "compute": null, "retainForLifetimeOfWorkspace": false, "queueingInfo": + null, "inputs": null, "outputs": {"debug_info": {"assetId": "azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161558_474262_output_data_debug_info/versions/1", + "type": "UriFolder"}, "flow_outputs": {"assetId": "azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161558_474262_output_data_flow_outputs/versions/1", + "type": "UriFolder"}}}, "runDefinition": null, "jobSpecification": null, "systemSettings": + null}' + headers: + connection: + - keep-alive + content-length: + - '4847' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.041' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161558_474262/logContent + response: + body: + string: '"2024-04-18 08:16:41 +0000 138 promptflow-runtime INFO [basic_model_config_variant_0_20240418_161558_474262] + Receiving v2 bulk run request a7f9551d-23bd-4bda-b520-6217510545c2: {\"flow_id\": + \"basic_model_config_variant_0_20240418_161558_474262\", \"flow_run_id\": + \"basic_model_config_variant_0_20240418_161558_474262\", \"flow_source\": + {\"flow_source_type\": 1, \"flow_source_info\": {\"snapshot_id\": \"93696879-cef6-477f-9ca9-2013d3161f5d\"}, + \"flow_dag_file\": \"flow.flex.yaml\"}, \"log_path\": \"https://promptfloweast4063704120.blob.core.windows.net/azureml/ExperimentRun/dcid.basic_model_config_variant_0_20240418_161558_474262/logs/azureml/executionlogs.txt?sv=2019-07-07&sr=b&sig=**data_scrubbed**&skoid=55b92eba-d7c7-4afd-ab76-7bb1cd345283&sktid=00000000-0000-0000-0000-000000000000&skt=2024-04-18T03%3A36%3A59Z&ske=2024-04-19T11%3A46%3A59Z&sks=b&skv=2019-07-07&st=2024-04-18T08%3A06%3A41Z&se=2024-04-18T16%3A16%3A41Z&sp=rcw\", + \"app_insights_instrumentation_key\": \"InstrumentationKey=**data_scrubbed**;IngestionEndpoint=https://eastus-6.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/\", + \"flow_name\": \"basic_model_config_variant_0_20240418_161558_474262\", \"batch_timeout_sec\": + 36000, \"data_inputs\": {\"data\": \"azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl\"}, + \"azure_storage_setting\": {\"azure_storage_mode\": 1, \"storage_account_name\": + \"promptfloweast4063704120\", \"blob_container_name\": \"azureml-blobstore-3e123da1-f9a5-4c91-9234-8d9ffbb39ff5\", + \"flow_artifacts_root_path\": \"promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161558_474262\", + \"blob_container_sas_token\": \"?sv=2019-07-07&sr=c&sig=**data_scrubbed**&skoid=55b92eba-d7c7-4afd-ab76-7bb1cd345283&sktid=00000000-0000-0000-0000-000000000000&skt=2024-04-18T08%3A16%3A41Z&ske=2024-04-25T08%3A16%3A41Z&sks=b&skv=2019-07-07&se=2024-04-25T08%3A16%3A41Z&sp=racwl\", + \"output_datastore_name\": \"workspaceblobstore\"}, \"init_kwargs\": {\"azure_open_ai_model_config\": + {\"azure_deployment\": \"my_deployment\", \"connection\": \"azure_open_ai\"}, + \"open_ai_model_config\": {\"model\": \"my_model\", \"base_url\": \"fake_base_url\"}}}\n2024-04-18 + 08:16:42 +0000 138 promptflow-runtime INFO Runtime version: pr-1316971-v19. + PromptFlow version: 1.9.0.dev124219171\n2024-04-18 08:16:42 +0000 138 + promptflow-runtime INFO Updating basic_model_config_variant_0_20240418_161558_474262 + to Status.Preparing...\n2024-04-18 08:16:42 +0000 138 promptflow-runtime + INFO Downloading snapshot to /mnt/host/service/app/37347/requests/basic_model_config_variant_0_20240418_161558_474262\n2024-04-18 + 08:16:42 +0000 138 promptflow-runtime INFO Get snapshot sas url for + 93696879-cef6-477f-9ca9-2013d3161f5d.\n2024-04-18 08:16:42 +0000 138 promptflow-runtime + INFO Snapshot 93696879-cef6-477f-9ca9-2013d3161f5d contains 9 files.\n2024-04-18 + 08:16:42 +0000 138 promptflow-runtime INFO Download snapshot 93696879-cef6-477f-9ca9-2013d3161f5d + completed.\n2024-04-18 08:16:42 +0000 138 promptflow-runtime INFO Successfully + download snapshot to /mnt/host/service/app/37347/requests/basic_model_config_variant_0_20240418_161558_474262\n2024-04-18 + 08:16:42 +0000 138 promptflow-runtime INFO About to execute a python + flow.\n2024-04-18 08:16:43 +0000 138 promptflow-runtime INFO The run + basic_model_config_variant_0_20240418_161558_474262 needs to start trace.\n2024-04-18 + 08:16:43 +0000 138 promptflow-runtime INFO Set otlp_endpoint to http://127.0.0.1:37347//aml-api/v1.0/traces\n2024-04-18 + 08:16:43 +0000 138 promptflow-runtime INFO Use spawn method to start + child process.\n2024-04-18 08:16:43 +0000 138 promptflow-runtime INFO Starting + to check process 532 status for run basic_model_config_variant_0_20240418_161558_474262\n2024-04-18 + 08:16:43 +0000 138 promptflow-runtime INFO Start checking run status + for run basic_model_config_variant_0_20240418_161558_474262\n2024-04-18 08:16:49 + +0000 532 promptflow-runtime INFO [138--532] Start processing flowV2......\n2024-04-18 + 08:16:49 +0000 532 promptflow-runtime INFO Runtime version: pr-1316971-v19. + PromptFlow version: 1.9.0.dev124219171\n2024-04-18 08:16:49 +0000 532 + promptflow-runtime INFO Setting mlflow tracking uri...\n2024-04-18 08:16:49 + +0000 532 promptflow-runtime INFO Validating ''AzureML Data Scientist'' + user authentication...\n2024-04-18 08:16:49 +0000 532 promptflow-runtime + INFO Successfully validated ''AzureML Data Scientist'' user authentication.\n2024-04-18 + 08:16:49 +0000 532 promptflow-runtime INFO Using AzureMLRunStorageV2\n2024-04-18 + 08:16:49 +0000 532 promptflow-runtime INFO Setting mlflow tracking + uri to ''azureml://eastus.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus''\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Setting mlflow tracking + uri to ''azureml://eastus.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus''\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Creating unregistered output + Asset for Run basic_model_config_variant_0_20240418_161558_474262...\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Created debug_info Asset: + azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161558_474262_output_data_debug_info/versions/1\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Creating unregistered output + Asset for Run basic_model_config_variant_0_20240418_161558_474262...\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Created flow_outputs output + Asset: azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161558_474262_output_data_flow_outputs/versions/1\n2024-04-18 + 08:16:50 +0000 532 promptflow-runtime INFO Patching basic_model_config_variant_0_20240418_161558_474262...\n2024-04-18 + 08:16:51 +0000 532 promptflow-runtime INFO Resolve data from url finished + in 1.2249866009997277 seconds\n2024-04-18 08:16:51 +0000 532 promptflow-runtime + INFO Starting the aml run ''basic_model_config_variant_0_20240418_161558_474262''...\n2024-04-18 + 08:16:52 +0000 532 execution WARNING Starting run without column + mapping may lead to unexpected results. Please consult the following documentation + for more information: https://aka.ms/pf/column-mapping\n2024-04-18 08:16:52 + +0000 532 execution.bulk INFO The timeout for the batch run is + 36000 seconds.\n2024-04-18 08:16:53 +0000 532 execution.bulk INFO Set + process count to 2 by taking the minimum value among the factors of {''default_worker_count'': + 4, ''row_count'': 2}.\n2024-04-18 08:17:01 +0000 532 execution.bulk INFO Process + name(ForkProcess-2:2:1)-Process id(615)-Line number(0) start execution.\n2024-04-18 + 08:17:01 +0000 532 execution.bulk INFO Process name(ForkProcess-2:2:2)-Process + id(624)-Line number(1) start execution.\n2024-04-18 08:17:01 +0000 532 + execution.bulk INFO Process name(ForkProcess-2:2:1)-Process id(615)-Line + number(0) completed.\n2024-04-18 08:17:02 +0000 532 execution.bulk INFO Finished + 1 / 2 lines.\n2024-04-18 08:17:02 +0000 532 execution.bulk INFO Average + execution time for completed lines: 9.01 seconds. Estimated time for incomplete + lines: 9.01 seconds.\n2024-04-18 08:17:04 +0000 532 execution.bulk INFO Process + name(ForkProcess-2:2:2)-Process id(624)-Line number(1) completed.\n2024-04-18 + 08:17:05 +0000 532 execution.bulk INFO Finished 2 / 2 lines.\n2024-04-18 + 08:17:05 +0000 532 execution.bulk INFO Average execution time + for completed lines: 6.03 seconds. Estimated time for incomplete lines: 0.0 + seconds.\n2024-04-18 08:17:05 +0000 532 execution.bulk INFO The + thread monitoring the process [615-ForkProcess-2:2:1] will be terminated.\n2024-04-18 + 08:17:05 +0000 532 execution.bulk INFO The thread monitoring the + process [624-ForkProcess-2:2:2] will be terminated.\n2024-04-18 08:17:05 +0000 615 + execution.bulk INFO The process [615] has received a terminate signal.\n2024-04-18 + 08:17:05 +0000 624 execution.bulk INFO The process [624] has received + a terminate signal.\n2024-04-18 08:17:07 +0000 532 promptflow-runtime + INFO Post processing batch result...\n2024-04-18 08:17:08 +0000 532 + execution.bulk INFO Upload status summary metrics for run basic_model_config_variant_0_20240418_161558_474262 + finished in 0.7782254149997243 seconds\n2024-04-18 08:17:08 +0000 532 + promptflow-runtime INFO Successfully write run properties {\"azureml.promptflow.total_tokens\": + 0, \"_azureml.evaluate_artifacts\": \"[{\\\"path\\\": \\\"instance_results.jsonl\\\", + \\\"type\\\": \\\"table\\\"}]\"} with run id ''basic_model_config_variant_0_20240418_161558_474262''\n2024-04-18 + 08:17:08 +0000 532 execution.bulk INFO Upload RH properties for + run basic_model_config_variant_0_20240418_161558_474262 finished in 0.07885484100006579 + seconds\n2024-04-18 08:17:08 +0000 532 promptflow-runtime INFO Creating + Artifact for Run basic_model_config_variant_0_20240418_161558_474262...\n2024-04-18 + 08:17:08 +0000 532 promptflow-runtime INFO Created instance_results.jsonl + Artifact.\n2024-04-18 08:17:08 +0000 532 promptflow-runtime INFO Ending + the aml run ''basic_model_config_variant_0_20240418_161558_474262'' with status + ''Completed''...\n"' + headers: + connection: + - keep-alive + content-length: + - '9882' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.426' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_obj_in_init.yaml b/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_obj_in_init.yaml new file mode 100644 index 00000000000..fee8e7d4cea --- /dev/null +++ b/src/promptflow-recording/recordings/azure/test_run_operations_TestFlowRun_test_model_config_obj_in_init.yaml @@ -0,0 +1,1093 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '3678' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.029' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1372' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.047' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.074' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.106' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:12:11 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/inputs.jsonl + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '60' + content-md5: + - tiCwcZnEReXDaVrgfN3qAQ== + content-type: + - application/octet-stream + last-modified: + - Thu, 18 Apr 2024 06:39:03 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Thu, 18 Apr 2024 06:39:02 GMT + x-ms-meta-name: + - b71e395a-c049-4e64-8360-c72dd242006d + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - ce6436cb-e73a-4d3a-8a12-774ed1e58516 + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:12:15 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/inputs.jsonl + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.090' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.156' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:12:26 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/basic_model_config/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '1731' + content-md5: + - uep0kZH5H8Dt1nyxSWHg7Q== + content-type: + - application/octet-stream + last-modified: + - Thu, 18 Apr 2024 07:56:09 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Thu, 18 Apr 2024 07:56:08 GMT + x-ms-meta-name: + - c4a517e8-b572-4b41-beb7-1c44702214bc + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - '1' + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.0 Python/3.11.5 (Windows-10-10.0.22621-SP0) + x-ms-date: + - Thu, 18 Apr 2024 08:12:30 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/basic_model_config/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: '{"flowDefinitionDataStoreName": "workspaceblobstore", "flowDefinitionBlobPath": + "LocalUpload/000000000000000000000000000000000000/basic_model_config/flow.flex.yaml", + "runId": "basic_model_config_variant_0_20240418_161201_761862", "runDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "runExperimentName": + "", "sessionId": "000000000000000000000000000000000000000000000000", "sessionSetupMode": + "SystemWait", "flowLineageId": "0000000000000000000000000000000000000000000000000000000000000000", + "runDisplayNameGenerationType": "UserProvidedMacro", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/000000000000000000000000000000000000/inputs.jsonl"}, + "inputsMapping": {}, "environmentVariables": {}, "initKWargs": {"azure_open_ai_model_config": + {"azure_deployment": "my_deployment", "azure_endpoint": null, "api_version": + null, "api_key": null, "organization": null, "connection": "azure_open_ai"}, + "open_ai_model_config": {"model": "my_model", "base_url": "fake_base_url", "api_key": + null, "organization": null, "connection": null}}, "connections": {}, "runtimeName": + "fake-runtime-name"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1147' + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: POST + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/submit + response: + body: + string: '"basic_model_config_variant_0_20240418_161201_761862"' + headers: + connection: + - keep-alive + content-length: + - '53' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '5.847' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161201_761862/flowRuns/basic_model_config_variant_0_20240418_161201_761862", + "flowRunId": "basic_model_config_variant_0_20240418_161201_761862", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161201_761862?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.252' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161201_761862/flowRuns/basic_model_config_variant_0_20240418_161201_761862", + "flowRunId": "basic_model_config_variant_0_20240418_161201_761862", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161201_761862?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.164' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161201_761862/flowRuns/basic_model_config_variant_0_20240418_161201_761862", + "flowRunId": "basic_model_config_variant_0_20240418_161201_761862", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161201_761862?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.151' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161201_761862/flowRuns/basic_model_config_variant_0_20240418_161201_761862", + "flowRunId": "basic_model_config_variant_0_20240418_161201_761862", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161201_761862?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.191' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862 + response: + body: + string: '{"flowGraph": {"inputs": {"func_input": {"type": "string", "is_chat_input": + false}}, "outputs": {"obj_input": {"type": "string", "evaluation_only": false, + "is_chat_output": false}, "func_input": {"type": "string", "evaluation_only": + false, "is_chat_output": false}, "obj_id": {"type": "string", "evaluation_only": + false, "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/basic_model_config_variant_0_20240418_161201_761862/flowRuns/basic_model_config_variant_0_20240418_161201_761862", + "flowRunId": "basic_model_config_variant_0_20240418_161201_761862", "flowRunDisplayName": + "basic_model_config_variant_0_20240418_161201_761862", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", + "sessionId": "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "studioPortalEndpoint": + "https://ml.azure.com/runs/basic_model_config_variant_0_20240418_161201_761862?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '1485' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.191' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862/childRuns?endIndex=24&startIndex=0 + response: + body: + string: '[{"run_id": "basic_model_config_variant_0_20240418_161201_761862_0", + "status": "Completed", "error": null, "inputs": {"func_input": "func_input1", + "line_number": 0}, "output": {"azure_open_ai_model_config_deployment": "my_deployment", + "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input1", "obj_id": 139968831760080}, "metrics": + null, "request": null, "parent_run_id": "basic_model_config_variant_0_20240418_161201_761862", + "root_run_id": "basic_model_config_variant_0_20240418_161201_761862", "source_run_id": + null, "flow_id": "default_flow_id", "start_time": "2024-04-18T08:13:08.137761Z", + "end_time": "2024-04-18T08:13:08.164414Z", "index": 0, "api_calls": [{"name": + "MyFlow.__call__", "type": "Function", "inputs": {"func_input": "func_input1"}, + "output": {"azure_open_ai_model_config_deployment": "my_deployment", "azure_open_ai_model_config_azure_endpoint": + "https://gpt-test-eus.openai.azure.com/", "azure_open_ai_model_config_connection": + null, "open_ai_model_config_model": "my_model", "open_ai_model_config_base_url": + "fake_base_url", "open_ai_model_config_connection": null, "func_input": "func_input1", + "obj_id": 139968831760080}, "start_time": 1713427988.139056, "end_time": 1713427988.163505, + "error": null, "children": [], "node_name": null, "parent_id": "", "id": "2b4a5c59-2bfe-4612-98ec-04121ca55631", + "function": "MyFlow.__call__"}], "name": "", "description": "", "tags": null, + "system_metrics": {"duration": 0.026653}, "result": {"azure_open_ai_model_config_deployment": + "my_deployment", "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input1", "obj_id": 139968831760080}, "upload_metrics": + false, "otel_trace_id": "afd33d05cca5a0a29a1db8a2f0b0749a", "message_format": + "basic"}, {"run_id": "basic_model_config_variant_0_20240418_161201_761862_1", + "status": "Completed", "error": null, "inputs": {"func_input": "func_input2", + "line_number": 1}, "output": {"azure_open_ai_model_config_deployment": "my_deployment", + "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input2", "obj_id": 139968831760080}, "metrics": + null, "request": null, "parent_run_id": "basic_model_config_variant_0_20240418_161201_761862", + "root_run_id": "basic_model_config_variant_0_20240418_161201_761862", "source_run_id": + null, "flow_id": "default_flow_id", "start_time": "2024-04-18T08:13:08.259313Z", + "end_time": "2024-04-18T08:13:08.262171Z", "index": 1, "api_calls": [{"name": + "MyFlow.__call__", "type": "Function", "inputs": {"func_input": "func_input2"}, + "output": {"azure_open_ai_model_config_deployment": "my_deployment", "azure_open_ai_model_config_azure_endpoint": + "https://gpt-test-eus.openai.azure.com/", "azure_open_ai_model_config_connection": + null, "open_ai_model_config_model": "my_model", "open_ai_model_config_base_url": + "fake_base_url", "open_ai_model_config_connection": null, "func_input": "func_input2", + "obj_id": 139968831760080}, "start_time": 1713427988.25989, "end_time": 1713427988.261461, + "error": null, "children": [], "node_name": null, "parent_id": "", "id": "ce41b1a1-1be8-4a28-87d0-ee0682d42dd1", + "function": "MyFlow.__call__"}], "name": "", "description": "", "tags": null, + "system_metrics": {"duration": 0.002858}, "result": {"azure_open_ai_model_config_deployment": + "my_deployment", "azure_open_ai_model_config_azure_endpoint": "https://gpt-test-eus.openai.azure.com/", + "azure_open_ai_model_config_connection": null, "open_ai_model_config_model": + "my_model", "open_ai_model_config_base_url": "fake_base_url", "open_ai_model_config_connection": + null, "func_input": "func_input2", "obj_id": 139968831760080}, "upload_metrics": + false, "otel_trace_id": "da07ead7139d6f6fe56a2e64d859a8ea", "message_format": + "basic"}]' + headers: + connection: + - keep-alive + content-length: + - '4120' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.472' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862/childRuns?endIndex=49&startIndex=25 + response: + body: + string: '[]' + headers: + connection: + - keep-alive + content-length: + - '2' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.478' + status: + code: 200 + message: OK +- request: + body: '{"runId": "basic_model_config_variant_0_20240418_161201_761862", "selectRunMetadata": + true, "selectRunDefinition": true, "selectJobSpecification": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '152' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://eastus.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/rundata + response: + body: + string: '{"runMetadata": {"runNumber": 1713427965, "rootRunId": "basic_model_config_variant_0_20240418_161201_761862", + "createdUtc": "2024-04-18T08:12:45.4503985+00:00", "createdBy": {"userObjectId": + "00000000-0000-0000-0000-000000000000", "userPuId": "100320005227D154", "userIdp": + null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Han Wang", + "upn": null}, "userId": "00000000-0000-0000-0000-000000000000", "token": null, + "tokenExpiryTimeUtc": null, "error": null, "warnings": null, "revision": 7, + "statusRevision": 3, "runUuid": "ce248b68-f9bf-4e99-8889-3f4f2a9bbbcc", "parentRunUuid": + null, "rootRunUuid": "ce248b68-f9bf-4e99-8889-3f4f2a9bbbcc", "lastStartTimeUtc": + null, "currentComputeTime": null, "computeDuration": "00:00:11.3695647", "effectiveStartTimeUtc": + null, "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "100320005227D154", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Han Wang", "upn": "username@microsoft.com"}, + "lastModifiedUtc": "2024-04-18T08:13:11.4374418+00:00", "duration": "00:00:11.3695647", + "cancelationReason": null, "currentAttemptId": 1, "runId": "basic_model_config_variant_0_20240418_161201_761862", + "parentRunId": null, "experimentId": "d80a6560-645c-446b-a505-eb0cf106d0b7", + "status": "Completed", "startTimeUtc": "2024-04-18T08:13:00.3734882+00:00", + "endTimeUtc": "2024-04-18T08:13:11.7430529+00:00", "scheduleId": null, "displayName": + "basic_model_config_variant_0_20240418_161201_761862", "name": null, "dataContainerId": + "dcid.basic_model_config_variant_0_20240418_161201_761862", "description": + null, "hidden": false, "runType": "azureml.promptflow.FlowRun", "runTypeV2": + {"orchestrator": null, "traits": [], "attribution": "PromptFlow", "computeType": + null}, "properties": {"azureml.promptflow.runtime_name": "automatic", "azureml.promptflow.input_data": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl", + "azureml.promptflow.init_kwargs": "{\"azure_open_ai_model_config\":{\"azure_deployment\":\"my_deployment\",\"azure_endpoint\":null,\"api_version\":null,\"api_key\":null,\"organization\":null,\"connection\":\"azure_open_ai\"},\"open_ai_model_config\":{\"model\":\"my_model\",\"base_url\":\"fake_base_url\",\"api_key\":null,\"organization\":null,\"connection\":null}}", + "azureml.promptflow.disable_trace": "false", "azureml.promptflow.session_id": + "251ed427322fde47d867169f708e706292530ffe4b1b2f06", "azureml.promptflow.definition_file_name": + "flow.flex.yaml", "azureml.promptflow.flow_lineage_id": "3f23fa85b23e79109dded73208030690bce5daac03c90aa3b9776c17e5e47d11", + "azureml.promptflow.flow_definition_datastore_name": "workspaceblobstore", + "azureml.promptflow.flow_definition_blob_path": "LocalUpload/f9695f65269ef735088e55e21754dcdc/basic_model_config/flow.flex.yaml", + "_azureml.evaluation_run": "promptflow.BatchRun", "azureml.promptflow.snapshot_id": + "b0af1303-07c4-410e-bf78-c1d0ed5d3b23", "azureml.promptflow.run_mode": "Eager", + "azureml.promptflow.runtime_version": "pr-1316971-v19", "azureml.promptflow.total_tokens": + "0", "_azureml.evaluate_artifacts": "[{\"path\": \"instance_results.jsonl\", + \"type\": \"table\"}]"}, "parameters": {}, "actionUris": {}, "scriptName": + null, "target": null, "uniqueChildRunComputeTargets": [], "tags": {}, "settings": + {}, "services": {}, "inputDatasets": [], "outputDatasets": [], "runDefinition": + null, "jobSpecification": null, "primaryMetricName": null, "createdFrom": + null, "cancelUri": null, "completeUri": null, "diagnosticsUri": null, "computeRequest": + null, "compute": null, "retainForLifetimeOfWorkspace": false, "queueingInfo": + null, "inputs": null, "outputs": {"debug_info": {"assetId": "azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161201_761862_output_data_debug_info/versions/1", + "type": "UriFolder"}, "flow_outputs": {"assetId": "azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161201_761862_output_data_flow_outputs/versions/1", + "type": "UriFolder"}}}, "runDefinition": null, "jobSpecification": null, "systemSettings": + null}' + headers: + connection: + - keep-alive + content-length: + - '4992' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.032' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22621-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/basic_model_config_variant_0_20240418_161201_761862/logContent + response: + body: + string: '"2024-04-18 08:12:49 +0000 231 promptflow-runtime INFO [basic_model_config_variant_0_20240418_161201_761862] + Receiving v2 bulk run request 7ca632f2-4cf9-41ec-bc22-44495d167dff: {\"flow_id\": + \"basic_model_config_variant_0_20240418_161201_761862\", \"flow_run_id\": + \"basic_model_config_variant_0_20240418_161201_761862\", \"flow_source\": + {\"flow_source_type\": 1, \"flow_source_info\": {\"snapshot_id\": \"b0af1303-07c4-410e-bf78-c1d0ed5d3b23\"}, + \"flow_dag_file\": \"flow.flex.yaml\"}, \"log_path\": \"https://promptfloweast4063704120.blob.core.windows.net/azureml/ExperimentRun/dcid.basic_model_config_variant_0_20240418_161201_761862/logs/azureml/executionlogs.txt?sv=2019-07-07&sr=b&sig=**data_scrubbed**&skoid=55b92eba-d7c7-4afd-ab76-7bb1cd345283&sktid=00000000-0000-0000-0000-000000000000&skt=2024-04-18T08%3A02%3A44Z&ske=2024-04-19T16%3A12%3A44Z&sks=b&skv=2019-07-07&st=2024-04-18T08%3A02%3A47Z&se=2024-04-18T16%3A12%3A47Z&sp=rcw\", + \"app_insights_instrumentation_key\": \"InstrumentationKey=**data_scrubbed**;IngestionEndpoint=https://eastus-6.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/\", + \"flow_name\": \"basic_model_config_variant_0_20240418_161201_761862\", \"batch_timeout_sec\": + 36000, \"data_inputs\": {\"data\": \"azureml://datastores/workspaceblobstore/paths/LocalUpload/162dde470936d99746b01ac5ccd17fd7/inputs.jsonl\"}, + \"azure_storage_setting\": {\"azure_storage_mode\": 1, \"storage_account_name\": + \"promptfloweast4063704120\", \"blob_container_name\": \"azureml-blobstore-3e123da1-f9a5-4c91-9234-8d9ffbb39ff5\", + \"flow_artifacts_root_path\": \"promptflow/PromptFlowArtifacts/basic_model_config_variant_0_20240418_161201_761862\", + \"blob_container_sas_token\": \"?sv=2019-07-07&sr=c&sig=**data_scrubbed**&skoid=55b92eba-d7c7-4afd-ab76-7bb1cd345283&sktid=00000000-0000-0000-0000-000000000000&skt=2024-04-18T08%3A12%3A47Z&ske=2024-04-25T08%3A12%3A47Z&sks=b&skv=2019-07-07&se=2024-04-25T08%3A12%3A47Z&sp=racwl\", + \"output_datastore_name\": \"workspaceblobstore\"}, \"init_kwargs\": {\"azure_open_ai_model_config\": + {\"azure_deployment\": \"my_deployment\", \"azure_endpoint\": null, \"api_version\": + null, \"api_key\": null, \"organization\": null, \"connection\": \"azure_open_ai\"}, + \"open_ai_model_config\": {\"model\": \"my_model\", \"base_url\": \"fake_base_url\", + \"api_key\": null, \"organization\": null, \"connection\": null}}}\n2024-04-18 + 08:12:49 +0000 231 promptflow-runtime INFO Runtime version: pr-1316971-v19. + PromptFlow version: 1.9.0.dev124219171\n2024-04-18 08:12:49 +0000 231 + promptflow-runtime INFO Updating basic_model_config_variant_0_20240418_161201_761862 + to Status.Preparing...\n2024-04-18 08:12:49 +0000 231 promptflow-runtime + INFO Downloading snapshot to /mnt/host/service/app/40807/requests/basic_model_config_variant_0_20240418_161201_761862\n2024-04-18 + 08:12:49 +0000 231 promptflow-runtime INFO Get snapshot sas url for + b0af1303-07c4-410e-bf78-c1d0ed5d3b23.\n2024-04-18 08:12:49 +0000 231 promptflow-runtime + INFO Snapshot b0af1303-07c4-410e-bf78-c1d0ed5d3b23 contains 9 files.\n2024-04-18 + 08:12:50 +0000 231 promptflow-runtime INFO Download snapshot b0af1303-07c4-410e-bf78-c1d0ed5d3b23 + completed.\n2024-04-18 08:12:50 +0000 231 promptflow-runtime INFO Successfully + download snapshot to /mnt/host/service/app/40807/requests/basic_model_config_variant_0_20240418_161201_761862\n2024-04-18 + 08:12:50 +0000 231 promptflow-runtime INFO About to execute a python + flow.\n2024-04-18 08:12:51 +0000 231 promptflow-runtime INFO The run + basic_model_config_variant_0_20240418_161201_761862 needs to start trace.\n2024-04-18 + 08:12:51 +0000 231 promptflow-runtime INFO Set otlp_endpoint to http://127.0.0.1:40807//aml-api/v1.0/traces\n2024-04-18 + 08:12:51 +0000 231 promptflow-runtime INFO Use spawn method to start + child process.\n2024-04-18 08:12:51 +0000 231 promptflow-runtime INFO Starting + to check process 360 status for run basic_model_config_variant_0_20240418_161201_761862\n2024-04-18 + 08:12:51 +0000 231 promptflow-runtime INFO Start checking run status + for run basic_model_config_variant_0_20240418_161201_761862\n2024-04-18 08:12:56 + +0000 360 promptflow-runtime INFO [231--360] Start processing flowV2......\n2024-04-18 + 08:12:57 +0000 360 promptflow-runtime INFO Runtime version: pr-1316971-v19. + PromptFlow version: 1.9.0.dev124219171\n2024-04-18 08:12:57 +0000 360 + promptflow-runtime INFO Setting mlflow tracking uri...\n2024-04-18 08:12:57 + +0000 360 promptflow-runtime INFO Validating ''AzureML Data Scientist'' + user authentication...\n2024-04-18 08:12:57 +0000 360 promptflow-runtime + INFO Successfully validated ''AzureML Data Scientist'' user authentication.\n2024-04-18 + 08:12:57 +0000 360 promptflow-runtime INFO Using AzureMLRunStorageV2\n2024-04-18 + 08:12:57 +0000 360 promptflow-runtime INFO Setting mlflow tracking + uri to ''azureml://eastus.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus''\n2024-04-18 + 08:12:57 +0000 360 promptflow-runtime INFO Setting mlflow tracking + uri to ''azureml://eastus.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus''\n2024-04-18 + 08:12:57 +0000 360 promptflow-runtime INFO Creating unregistered output + Asset for Run basic_model_config_variant_0_20240418_161201_761862...\n2024-04-18 + 08:12:58 +0000 360 promptflow-runtime INFO Created debug_info Asset: + azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161201_761862_output_data_debug_info/versions/1\n2024-04-18 + 08:12:58 +0000 360 promptflow-runtime INFO Creating unregistered output + Asset for Run basic_model_config_variant_0_20240418_161201_761862...\n2024-04-18 + 08:12:58 +0000 360 promptflow-runtime INFO Created flow_outputs output + Asset: azureml://locations/eastus/workspaces/00000/data/azureml_basic_model_config_variant_0_20240418_161201_761862_output_data_flow_outputs/versions/1\n2024-04-18 + 08:12:58 +0000 360 promptflow-runtime INFO Patching basic_model_config_variant_0_20240418_161201_761862...\n2024-04-18 + 08:13:00 +0000 360 promptflow-runtime INFO Resolve data from url finished + in 1.1747502879998137 seconds\n2024-04-18 08:13:00 +0000 360 promptflow-runtime + INFO Starting the aml run ''basic_model_config_variant_0_20240418_161201_761862''...\n2024-04-18 + 08:13:01 +0000 360 execution WARNING Starting run without column + mapping may lead to unexpected results. Please consult the following documentation + for more information: https://aka.ms/pf/column-mapping\n2024-04-18 08:13:01 + +0000 360 execution.bulk INFO The timeout for the batch run is + 36000 seconds.\n2024-04-18 08:13:01 +0000 360 execution.bulk INFO Set + process count to 2 by taking the minimum value among the factors of {''default_worker_count'': + 4, ''row_count'': 2}.\n2024-04-18 08:13:07 +0000 360 execution.bulk INFO Process + name(ForkProcess-2:2:1)-Process id(443)-Line number(0) start execution.\n2024-04-18 + 08:13:08 +0000 360 execution.bulk INFO Process name(ForkProcess-2:2:1)-Process + id(443)-Line number(0) completed.\n2024-04-18 08:13:08 +0000 360 execution.bulk INFO Process + name(ForkProcess-2:2:1)-Process id(443)-Line number(1) start execution.\n2024-04-18 + 08:13:08 +0000 360 execution.bulk INFO Finished 1 / 2 lines.\n2024-04-18 + 08:13:08 +0000 360 execution.bulk INFO Process name(ForkProcess-2:2:1)-Process + id(443)-Line number(1) completed.\n2024-04-18 08:13:08 +0000 360 execution.bulk INFO Average + execution time for completed lines: 7.01 seconds. Estimated time for incomplete + lines: 7.01 seconds.\n2024-04-18 08:13:08 +0000 360 execution.bulk INFO The + thread monitoring the process [447-ForkProcess-2:2:2] will be terminated.\n2024-04-18 + 08:13:08 +0000 360 execution.bulk INFO The thread monitoring the + process [443-ForkProcess-2:2:1] will be terminated.\n2024-04-18 08:13:08 +0000 443 + execution.bulk INFO The process [443] has received a terminate signal.\n2024-04-18 + 08:13:08 +0000 447 execution.bulk INFO The process [447] has received + a terminate signal.\n2024-04-18 08:13:10 +0000 360 promptflow-runtime + INFO Post processing batch result...\n2024-04-18 08:13:11 +0000 360 + execution.bulk INFO Upload status summary metrics for run basic_model_config_variant_0_20240418_161201_761862 + finished in 0.9009329910004453 seconds\n2024-04-18 08:13:11 +0000 360 + promptflow-runtime INFO Successfully write run properties {\"azureml.promptflow.total_tokens\": + 0, \"_azureml.evaluate_artifacts\": \"[{\\\"path\\\": \\\"instance_results.jsonl\\\", + \\\"type\\\": \\\"table\\\"}]\"} with run id ''basic_model_config_variant_0_20240418_161201_761862''\n2024-04-18 + 08:13:11 +0000 360 execution.bulk INFO Upload RH properties for + run basic_model_config_variant_0_20240418_161201_761862 finished in 0.06188116199973592 + seconds\n2024-04-18 08:13:11 +0000 360 promptflow-runtime INFO Creating + Artifact for Run basic_model_config_variant_0_20240418_161201_761862...\n2024-04-18 + 08:13:11 +0000 360 promptflow-runtime INFO Created instance_results.jsonl + Artifact.\n2024-04-18 08:13:11 +0000 360 promptflow-runtime INFO Ending + the aml run ''basic_model_config_variant_0_20240418_161201_761862'' with status + ''Completed''...\n"' + headers: + connection: + - keep-alive + content-length: + - '9783' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.333' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow/tests/test_configs/eager_flows/basic_model_config/class_with_model_config.py b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/class_with_model_config.py new file mode 100644 index 00000000000..1818aee7b87 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/class_with_model_config.py @@ -0,0 +1,47 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +from typing import TypedDict + +from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration + + +class FlowOutput(TypedDict): + obj_input: str + func_input: str + obj_id: str + + +class MyFlow: + def __init__( + self, + azure_open_ai_model_config: AzureOpenAIModelConfiguration, + open_ai_model_config: OpenAIModelConfiguration + ): + self.azure_open_ai_model_config = azure_open_ai_model_config + self.open_ai_model_config = open_ai_model_config + + def __call__(self, func_input: str) -> FlowOutput: + return { + "azure_open_ai_model_config_deployment": self.azure_open_ai_model_config.azure_deployment, + "azure_open_ai_model_config_azure_endpoint": self.azure_open_ai_model_config.azure_endpoint, + "azure_open_ai_model_config_connection": self.azure_open_ai_model_config.connection, + "open_ai_model_config_model": self.open_ai_model_config.model, + "open_ai_model_config_base_url": self.open_ai_model_config.base_url, + "open_ai_model_config_connection": self.open_ai_model_config.connection, + "func_input": func_input, + "obj_id": id(self), + } + + +if __name__ == "__main__": + config1 = AzureOpenAIModelConfiguration( + azure_deployment="my_deployment", + ) + config2 = OpenAIModelConfiguration( + model="my_model", + ) + flow = MyFlow(azure_open_ai_model_config=config1, open_ai_model_config=config2) + result = flow("func_input") + print(result) + diff --git a/src/promptflow/tests/test_configs/eager_flows/basic_model_config/flow.flex.yaml b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/flow.flex.yaml new file mode 100644 index 00000000000..e3d93690a7c --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/flow.flex.yaml @@ -0,0 +1,4 @@ +entry: class_with_model_config:MyFlow +environment: + image: promptflow.azurecr.io/promptflow-runtime:pr-1316971-v19 + python_requirements_txt: requirements.txt diff --git a/src/promptflow/tests/test_configs/eager_flows/basic_model_config/inputs.jsonl b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/inputs.jsonl new file mode 100644 index 00000000000..85246b081fa --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/inputs.jsonl @@ -0,0 +1,2 @@ +{"func_input": "func_input1"} +{"func_input": "func_input2"} \ No newline at end of file diff --git a/src/promptflow/tests/test_configs/eager_flows/basic_model_config/requirements.txt b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/requirements.txt new file mode 100644 index 00000000000..2201c932fb3 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/requirements.txt @@ -0,0 +1 @@ +promptflow \ No newline at end of file diff --git a/src/promptflow/tests/test_configs/eager_flows/basic_model_config/run.yaml b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/run.yaml new file mode 100644 index 00000000000..0721b380328 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/basic_model_config/run.yaml @@ -0,0 +1,5 @@ +description: sample bulk run +flow: ./ +data: ./inputs.jsonl +init: + obj_input: val From 4114d6aa33b71f22cfad30193c51c52ef5aad465 Mon Sep 17 00:00:00 2001 From: Honglin Date: Mon, 22 Apr 2024 15:58:56 +0800 Subject: [PATCH 03/14] [SDK/CLI] Support evaluation run local to cloud (#2914) # Description Support evaluation run local to cloud. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../azure/operations/_run_operations.py | 14 ++- .../e2etests/test_run_upload.py | 38 ++++++++- .../unittests/test_run_operations.py | 1 + .../promptflow/_sdk/entities/_run.py | 85 ++++++++++--------- 4 files changed, 95 insertions(+), 43 deletions(-) diff --git a/src/promptflow-azure/promptflow/azure/operations/_run_operations.py b/src/promptflow-azure/promptflow/azure/operations/_run_operations.py index a163b86af89..88de2e5f92d 100644 --- a/src/promptflow-azure/promptflow/azure/operations/_run_operations.py +++ b/src/promptflow-azure/promptflow/azure/operations/_run_operations.py @@ -946,6 +946,18 @@ def _upload(self, run: Union[str, Run]): ) set_event_loop_policy() + # check if it's evaluation run and make sure the main run is already uploaded + if run.run: + main_run_name = run.run.name if isinstance(run.run, Run) else run.run + try: + self.get(main_run_name) + except RunNotFoundError: + raise UserErrorException( + f"Failed to upload evaluation run {run.name!r} to cloud. It ran against a run {main_run_name!r} " + f"that was not uploaded to cloud. Make sure the previous run is already uploaded to cloud when " + f"uploading an evaluation run." + ) + # upload local run details to cloud run_uploader = AsyncRunUploader._from_run_operations(run=run, run_ops=self) result_dict = async_run_allowing_running_loop(run_uploader.upload) @@ -961,7 +973,7 @@ def _upload(self, run: Union[str, Run]): print(f"Portal url: {self._get_run_portal_url(run_id=run.name)}") def _registry_existing_bulk_run(self, run: Run): - # register the run in the cloud + """Register the run in the cloud""" rest_obj = run._to_rest_object() self._service_caller.create_existing_bulk_run( subscription_id=self._operation_scope.subscription_id, diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py index 1694d0f21aa..2ccefc75d93 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py @@ -43,12 +43,17 @@ def check_local_to_cloud_run(pf: PFClient, run: Run): # check if local run is uploaded cloud_run = pf.runs.get(run.name) assert cloud_run.display_name == run.display_name - assert cloud_run.description == run.description - assert cloud_run.tags == run.tags assert cloud_run.status == run.status assert cloud_run._start_time and cloud_run._end_time assert cloud_run.properties["azureml.promptflow.local_to_cloud"] == "true" assert cloud_run.properties["azureml.promptflow.snapshot_id"] + + # if no description or tags, skip the check, since one could be {} but the other is None + if run.description: + assert cloud_run.description == run.description + if run.tags: + assert cloud_run.tags == run.tags + return cloud_run @@ -182,3 +187,32 @@ def test_upload_run_with_customized_run_properties(self, pf: PFClient, randstr: assert cloud_run.properties[Local2CloudUserProperties.EVAL_ARTIFACTS] == eval_artifacts # check total tokens is recorded assert cloud_run.properties[Local2CloudProperties.TOTAL_TOKENS] + + @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") + @pytest.mark.usefixtures( + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + ) + def test_upload_eval_run(self, pf: PFClient, randstr: Callable[[str], str]): + main_run_name = randstr("main_run_name_for_test_upload_eval_run") + local_pf = Local2CloudTestHelper.get_local_pf(main_run_name) + main_run = local_pf.run( + flow=f"{FLOWS_DIR}/simple_hello_world", + data=f"{DATAS_DIR}/webClassification3.jsonl", + name=main_run_name, + column_mapping={"name": "${data.url}"}, + ) + Local2CloudTestHelper.check_local_to_cloud_run(pf, main_run) + + # run an evaluation run + eval_run_name = randstr("eval_run_name_for_test_upload_eval_run") + local_lpf = Local2CloudTestHelper.get_local_pf(eval_run_name) + eval_run = local_lpf.run( + flow=f"{FLOWS_DIR}/simple_hello_world", + data=f"{DATAS_DIR}/webClassification3.jsonl", + run=main_run_name, + name=eval_run_name, + # column_mapping={"name": "${run.outputs.result}"}, + column_mapping={"name": "${data.url}"}, + ) + eval_run = Local2CloudTestHelper.check_local_to_cloud_run(pf, eval_run) + assert eval_run.properties["azureml.promptflow.variant_run_id"] == main_run_name diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py index 249597935f0..f85ace022af 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/unittests/test_run_operations.py @@ -126,6 +126,7 @@ def test_upload_run_with_invalid_workspace_datastore(self, pf: PFClient, mocker: from promptflow._sdk.operations import RunOperations mocked = mocker.patch.object(RunOperations, "get") + mocked.return_value.run = None mocked.return_value.status = "Completed" mocker.patch.object(pf.runs, "_workspace_default_datastore", "test") with pytest.raises(UserErrorException, match="workspace default datastore is not supported"): diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py index b504e3dea3a..d6425ea7bea 100644 --- a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py +++ b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py @@ -547,7 +547,6 @@ def _to_rest_object(self): from promptflow.azure._restclient.flow.models import ( BatchDataInput, - CreateExistingBulkRunRequest, RunDisplayNameGenerationType, SessionSetupModeEnum, SubmitBulkRunRequest, @@ -657,45 +656,7 @@ def _to_rest_object(self): ) elif local_to_cloud_info: # register local run to cloud - - # parse local_to_cloud_info to get necessary information - flow_artifact_path = local_to_cloud_info[OutputsFolderName.FLOW_ARTIFACTS] - flow_artifact_root_path = Path(flow_artifact_path).parent.as_posix() - log_file_relative_path = local_to_cloud_info[LocalStorageFilenames.LOG] - snapshot_file_path = local_to_cloud_info[LocalStorageFilenames.SNAPSHOT_FOLDER] - - # get the start and end time. Plus "Z" to specify the timezone is UTC, otherwise there will be warning - # when sending the request to the server. - # e.g. WARNING:msrest.serialization:Datetime with no tzinfo will be considered UTC. - # for start_time, switch to "_start_time" once the bug item is fixed: BUG - 3085432. - start_time = self._created_on.isoformat() + "Z" if self._created_on else None - end_time = self._end_time.isoformat() + "Z" if self._end_time else None - - # extract properties that needs to be passed to the request - total_tokens = self.properties[FlowRunProperties.SYSTEM_METRICS].get("total_tokens", 0) - properties = {Local2CloudProperties.TOTAL_TOKENS: total_tokens} - for property_key in Local2CloudUserProperties.get_all_values(): - value = self.properties.get(property_key, None) - if value is not None: - properties[property_key] = value - - return CreateExistingBulkRunRequest( - run_id=self.name, - run_status=self.status, - start_time_utc=start_time, - end_time_utc=end_time, - run_display_name=self._get_default_display_name(), - description=self.description, - tags=self.tags, - properties=properties, - run_experiment_name=self._experiment_name, - run_display_name_generation_type=RunDisplayNameGenerationType.USER_PROVIDED_MACRO, - output_data_store=CloudDatastore.DEFAULT, - flow_artifacts_root_path=flow_artifact_root_path, - log_file_relative_path=log_file_relative_path, - flow_definition_data_store_name=CloudDatastore.DEFAULT, - flow_definition_blob_path=snapshot_file_path, - ) + return self._to_rest_object_for_local_to_cloud(local_to_cloud_info, variant) else: # upload via CodeOperations.create_or_update # submit with param FlowDefinitionDataUri @@ -703,6 +664,50 @@ def _to_rest_object(self): flow_definition_data_uri=str(self.flow), ) + def _to_rest_object_for_local_to_cloud(self, local_to_cloud_info: dict, variant_run_id=None): + """Convert run object to CreateExistingBulkRunRequest object for local to cloud operation.""" + from promptflow.azure._restclient.flow.models import CreateExistingBulkRunRequest, RunDisplayNameGenerationType + + # parse local_to_cloud_info to get necessary information + flow_artifact_path = local_to_cloud_info[OutputsFolderName.FLOW_ARTIFACTS] + flow_artifact_root_path = Path(flow_artifact_path).parent.as_posix() + log_file_relative_path = local_to_cloud_info[LocalStorageFilenames.LOG] + snapshot_file_path = local_to_cloud_info[LocalStorageFilenames.SNAPSHOT_FOLDER] + + # get the start and end time. Plus "Z" to specify the timezone is UTC, otherwise there will be warning + # when sending the request to the server. + # e.g. WARNING:msrest.serialization:Datetime with no tzinfo will be considered UTC. + # for start_time, switch to "_start_time" once the bug item is fixed: BUG - 3085432. + start_time = self._created_on.isoformat() + "Z" if self._created_on else None + end_time = self._end_time.isoformat() + "Z" if self._end_time else None + + # extract properties that needs to be passed to the request + total_tokens = self.properties[FlowRunProperties.SYSTEM_METRICS].get("total_tokens", 0) + properties = {Local2CloudProperties.TOTAL_TOKENS: total_tokens} + for property_key in Local2CloudUserProperties.get_all_values(): + value = self.properties.get(property_key, None) + if value is not None: + properties[property_key] = value + + return CreateExistingBulkRunRequest( + run_id=self.name, + run_status=self.status, + start_time_utc=start_time, + end_time_utc=end_time, + run_display_name=self._get_default_display_name(), + description=self.description, + tags=self.tags, + variant_run_id=variant_run_id, + properties=properties, + run_experiment_name=self._experiment_name, + run_display_name_generation_type=RunDisplayNameGenerationType.USER_PROVIDED_MACRO, + output_data_store=CloudDatastore.DEFAULT, + flow_artifacts_root_path=flow_artifact_root_path, + log_file_relative_path=log_file_relative_path, + flow_definition_data_store_name=CloudDatastore.DEFAULT, + flow_definition_blob_path=snapshot_file_path, + ) + def _check_run_status_is_completed(self) -> None: if self.status != RunStatus.COMPLETED: error_message = f"Run {self.name!r} is not completed, the status is {self.status!r}." From 17d13b27878d8a876093cbedbd875cfc63fccccf Mon Sep 17 00:00:00 2001 From: Clement Wang <47586720+wangchao1230@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:11:05 +0800 Subject: [PATCH 04/14] Fix devcontainer config: resolve conflict between azure-cli and promtpflow-azure (#2896) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .devcontainer/Dockerfile | 1 + .devcontainer/devcontainer.json | 47 +++++++++++++++++++-------------- .devcontainer/requirements.txt | 3 +-- examples/requirements.txt | 2 +- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0f272ea7a0d..42b2f449e7a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update \ && apt-get -y install build-essential \ && apt-get -y install docker.io +# Install notebook depenency RUN pip install ipython ipykernel RUN ipython kernel install --user --name promptflow diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a8c0bd1e87..1b0f2ac6b8c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,22 +1,29 @@ { - "name": "Promptflow-Python39", - // "context" is the path that the Codespaces docker build command should be run from, relative to devcontainer.json - "context": ".", - "dockerFile": "Dockerfile", - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-toolsai.vscode-ai", - "ms-toolsai.jupyter", - "redhat.vscode-yaml", - "prompt-flow.prompt-flow" - ], - - "runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"] + "name": "Promptflow-Python39", + // "context" is the path that the Codespaces docker build command should be run from, relative to devcontainer.json + "context": ".", + "dockerFile": "Dockerfile", + "runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"], + "customizations": { + "codespaces": { + "openFiles": ["README.md", "examples/README.md"] + }, + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-toolsai.vscode-ai", + "ms-toolsai.jupyter", + "redhat.vscode-yaml", + "prompt-flow.prompt-flow" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": {} + } } diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt index 7e457e37f76..7aa4db48f47 100644 --- a/.devcontainer/requirements.txt +++ b/.devcontainer/requirements.txt @@ -1,3 +1,2 @@ -azure-cli -promptflow[azure] +promptflow[azure]>=1.9.0 promptflow-tools \ No newline at end of file diff --git a/examples/requirements.txt b/examples/requirements.txt index f7ca3edd40c..21832de3282 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,4 +1,4 @@ -promptflow[azure]==1.9.0 +promptflow[azure] promptflow-tools python-dotenv bs4 From b105ae1011394b135feb8cb198a301daf764a099 Mon Sep 17 00:00:00 2001 From: Xingzhi Zhang <37076709+elliotzh@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:01:03 +0800 Subject: [PATCH 05/14] fix: chat ui need to infer signature of csharp (#2898) # Description 1. fix for csharp flex flow chat ui. 2. refactor current csharp tests - give them a specific marker and skip the marker in regular tests Hard to add corresponding tests for now given it's wired to do a copy for all api test utils/conftest # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes].** - [x] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../promptflow-csharp-e2e-test-env-setup.yml | 55 ++ .../promptflow-csharp-e2e-test-tests.yml | 47 ++ .../pipelines/promptflow-csharp-e2e-test.yml | 131 +--- .github/workflows/promptflow-sdk-cli-test.yml | 2 +- .../sdk-cli-azure-test-pull-request.yml | 2 +- .../tests/sdk_cli_azure_test/conftest.py | 25 +- .../e2etests/test_csharp_cli.py | 24 - .../e2etests/test_csharp_sdk.py | 33 + .../promptflow/_utils/flow_utils.py | 2 +- src/promptflow-core/promptflow/core/_utils.py | 4 +- .../_proxy/_python_executor_proxy.py | 4 +- .../_proxy/_python_inspector_proxy.py | 4 +- .../_sdk/operations/_flow_operations.py | 33 +- src/promptflow-devkit/pyproject.toml | 3 +- src/promptflow-devkit/tests/conftest.py | 42 ++ .../sdk_cli_test/e2etests/test_csharp_cli.py | 144 ++-- .../sdk_cli_test/e2etests/test_csharp_sdk.py | 55 ++ .../sdk_pfs_test/e2etests/test_csharp.py | 50 ++ ...sdk_TestCSharpSdk_test_basic_run_bulk.yaml | 615 ++++++++++++++++++ 19 files changed, 1005 insertions(+), 270 deletions(-) create mode 100644 .github/pipelines/promptflow-csharp-e2e-test-env-setup.yml create mode 100644 .github/pipelines/promptflow-csharp-e2e-test-tests.yml delete mode 100644 src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py create mode 100644 src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py create mode 100644 src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py create mode 100644 src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py create mode 100644 src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml diff --git a/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml b/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml new file mode 100644 index 00000000000..d84bcaa71dd --- /dev/null +++ b/.github/pipelines/promptflow-csharp-e2e-test-env-setup.yml @@ -0,0 +1,55 @@ +parameters: +- name: promptflowCsPat + displayName: "PAT to clone csharp repository" + type: string +- name: flowProjectRelativePath + displayName: "Flow Project Relative Path" + type: string + +steps: + - task: UseDotNet@2 + inputs: + version: '6.x' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9.x' + architecture: 'x64' + + - task: PowerShell@2 + displayName: 'Install promptflow cli' + inputs: + targetType: 'inline' + script: | + Set-PSDebug -Trace 1 + pip install -r src/promptflow/dev_requirements.txt + pip install src/promptflow-tracing + pip install src/promptflow-core[executor-service] + pip install src/promptflow-devkit + pip install src/promptflow-azure + pip install src/promptflow-recording + pip freeze + + - task: PowerShell@2 + displayName: 'Clone csharp repository' + inputs: + targetType: 'inline' + script: | + git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp + + - task: NuGetAuthenticate@1 + + - task: DotNetCoreCLI@2 + inputs: + command: 'restore' + projects: '$(flowProjectRelativePath)/**/*.csproj' + feedsToUse: 'config' + nugetConfigPath: '$(flowProjectRelativePath)/nuget.config' + displayName: 'dotnet restore' + + - task: DotNetCoreCLI@2 + inputs: + command: 'build' + projects: '$(flowProjectRelativePath)/**/*.csproj' + feedsToUse: 'config' + nugetConfigPath: '$(flowProjectRelativePath)/nuget.config' + displayName: 'dotnet build' diff --git a/.github/pipelines/promptflow-csharp-e2e-test-tests.yml b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml new file mode 100644 index 00000000000..c88884a1875 --- /dev/null +++ b/.github/pipelines/promptflow-csharp-e2e-test-tests.yml @@ -0,0 +1,47 @@ +parameters: +- name: azureOpenAiApiKey + displayName: "Azure OpenAI API Key" + type: string +- name: azureOpenAiApiBase + displayName: "Azure OpenAI API Base" + type: string +- name: flowProjectRelativePath + displayName: "Flow Project Relative Path" + type: string + +steps: +- task: PowerShell@2 + displayName: 'Copy local connections for ci pipeline' + inputs: + targetType: 'inline' + script: | + Copy-Item dev-connections.json.example connections.json + workingDirectory: $(Build.SourcesDirectory)/src/promptflow + +- task: PowerShell@2 + displayName: 'Run sdk cli tests' + inputs: + targetType: 'inline' + script: | + pytest tests/ -m "csharp" + workingDirectory: $(Build.SourcesDirectory)/src/promptflow-devkit + env: + CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) + AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey) + AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase) + IS_IN_CI_PIPELINE: true + +# enable this after we fix recording issue +#- task: PowerShell@2 +# displayName: 'Run azure sdk cli tests' +# inputs: +# targetType: 'inline' +# script: | +# pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py +# workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure +# env: +# CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) +# AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey) +# AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase) +# PROMPT_FLOW_TEST_MODE: "replay" +# IS_IN_CI_PIPELINE: true diff --git a/.github/pipelines/promptflow-csharp-e2e-test.yml b/.github/pipelines/promptflow-csharp-e2e-test.yml index 585cdc04d62..7af220811ba 100644 --- a/.github/pipelines/promptflow-csharp-e2e-test.yml +++ b/.github/pipelines/promptflow-csharp-e2e-test.yml @@ -44,25 +44,6 @@ jobs: pool: name: promptflow-1ES-ubuntu20 steps: - - task: UseDotNet@2 - inputs: - version: '6.x' - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9.x' - architecture: 'x64' - - task: Bash@3 - displayName: 'Install promptflow cli' - inputs: - targetType: 'inline' - script: | - Set-PSDebug -Trace 1 - pip install -r ./src/promptflow/dev_requirements.txt - pip install ./src/promptflow-tracing - pip install ./src/promptflow-core[executor-service] - pip install ./src/promptflow-devkit - pip install ./src/promptflow-azure - pip freeze - task: Bash@3 displayName: 'Set environment variables' inputs: @@ -74,43 +55,16 @@ jobs: export ACS_CONNECTION=$(ACS_CONNECTION) export IS_IN_CI_PIPELINE=true - - task: Bash@3 - displayName: 'Clone csharp repository' - inputs: - targetType: 'inline' - script: | - git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp - - - task: NuGetAuthenticate@1 + - template: promptflow-csharp-e2e-test-env-setup.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + promptflowCsPat: '$(PROMPTFLOW_CS_PAT)' - - task: DotNetCoreCLI@2 - inputs: - command: 'restore' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet restore' - - - task: DotNetCoreCLI@2 - inputs: - command: 'build' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet build' - - - task: Bash@3 - displayName: 'Run tests' - inputs: - targetType: 'inline' - script: | - cd ./src/promptflow-devkit - pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py - env: - CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) - AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - IS_IN_CI_PIPELINE: true + - template: promptflow-csharp-e2e-test-tests.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)' + azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)' - publish: $(flowProjectRelativePath) condition: always() @@ -119,28 +73,6 @@ jobs: pool: name: promptflow-1ES-win steps: - - task: UseDotNet@2 - inputs: - version: '6.x' - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8.x' - architecture: 'x64' - - - task: PowerShell@2 - displayName: 'Install promptflow cli' - inputs: - targetType: 'inline' - script: | - Set-PSDebug -Trace 1 - pip install -r ./src/promptflow/dev_requirements.txt - pip install ./src/promptflow-tracing - pip install ./src/promptflow-core[executor-service] - pip install ./src/promptflow-devkit - pip install ./src/promptflow-azure - pip freeze - - task: PowerShell@2 displayName: 'Set environment variables' inputs: @@ -152,43 +84,16 @@ jobs: setx ACS_CONNECTION $(ACS_CONNECTION) setx IS_IN_CI_PIPELINE true - - task: PowerShell@2 - displayName: 'Clone csharp repository' - inputs: - targetType: 'inline' - script: | - git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp + - template: promptflow-csharp-e2e-test-env-setup.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + promptflowCsPat: '$(PROMPTFLOW_CS_PAT)' - - task: NuGetAuthenticate@1 - - - task: DotNetCoreCLI@2 - inputs: - command: 'restore' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet restore' - - - task: DotNetCoreCLI@2 - inputs: - command: 'build' - projects: '$(flowProjectRelativePath)/**/*.csproj' - feedsToUse: 'config' - nugetConfigPath: 'csharp/src/TestProjects/nuget.config' - displayName: 'dotnet build' - - - task: PowerShell@2 - displayName: 'Run tests' - inputs: - targetType: 'inline' - script: | - cd ./src/promptflow-devkit - pytest tests/sdk_cli_test/e2etests/test_csharp_cli.py - env: - CSHARP_TEST_CASES_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath) - AZURE_OPENAI_API_KEY: $(AZURE_OPENAI_API_KEY) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - IS_IN_CI_PIPELINE: true + - template: promptflow-csharp-e2e-test-tests.yml + parameters: + flowProjectRelativePath: '$(flowProjectRelativePath)' + azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)' + azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)' - publish: $(flowProjectRelativePath) condition: always() diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 6dec7fb6e60..e7955629903 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -72,7 +72,7 @@ jobs: - name: run devkit tests run: | poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \ - --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \ + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \ --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py working-directory: ${{ env.WORKING_DIRECTORY }} - name: Upload Test Results diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml index a989e8c7d19..31d1222b5cd 100644 --- a/.github/workflows/sdk-cli-azure-test-pull-request.yml +++ b/.github/workflows/sdk-cli-azure-test-pull-request.yml @@ -70,7 +70,7 @@ jobs: working-directory: ${{ env.WORKING_DIRECTORY }} run: | poetry run pytest ./tests/sdk_cli_azure_test --cov=promptflow --cov-config=pyproject.toml ` - --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" + --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" - name: Upload Test Results if: always() diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py index df0c93adcb2..e6643bd60a8 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py @@ -7,7 +7,7 @@ import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from typing import Callable +from typing import Callable, TypedDict from unittest.mock import patch import jwt @@ -626,3 +626,26 @@ def mock_trace_provider_to_cloud( ) with patch("promptflow._sdk._configuration.Configuration.get_trace_provider", return_value=trace_provider): yield + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +def construct_csharp_test_project(flow_name: str) -> CSharpProject: + root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None) + if not root_of_test_cases: + pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.") + root_of_test_cases = Path(root_of_test_cases) + return { + "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(), + "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(), + "init": (root_of_test_cases / flow_name / "init.json").as_posix(), + } + + +@pytest.fixture +def csharp_test_project_basic() -> CSharpProject: + return construct_csharp_test_project("Basic") diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py deleted file mode 100644 index ed014372187..00000000000 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_cli.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import os.path -from typing import Callable - -import pytest - -from promptflow.azure import PFClient - - -def get_repo_base_path(): - return os.getenv("CSHARP_REPO_BASE_PATH", None) - - -@pytest.mark.usefixtures( - "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" -) -@pytest.mark.cli_test -@pytest.mark.e2etest -@pytest.mark.skipif(get_repo_base_path() is None, reason="available locally only before csharp support go public") -class TestCSharpCli: - def test_eager_flow_run_without_yaml(self, pf: PFClient, randstr: Callable[[str], str]): - pf.run( - flow=f"{get_repo_base_path()}\\src\\PromptflowCSharp\\Sample\\Basic\\bin\\Debug\\net6.0", - ) diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py new file mode 100644 index 00000000000..f1ffc69754b --- /dev/null +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py @@ -0,0 +1,33 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +from typing import Callable + +import pytest + +from promptflow._sdk.entities import Run +from promptflow.azure import PFClient + +from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD + + +@pytest.mark.timeout(timeout=DEFAULT_TEST_TIMEOUT, method=PYTEST_TIMEOUT_METHOD) +@pytest.mark.e2etest +@pytest.mark.csharp +@pytest.mark.usefixtures( + "mock_set_headers_with_user_aml_token", + "single_worker_thread_pool", + "vcr_recording", +) +class TestCSharpSdk: + def test_basic_run_bulk(self, pf: PFClient, randstr: Callable[[str], str], csharp_test_project_basic): + name = randstr("name") + + run = pf.run( + flow=csharp_test_project_basic["flow_dir"], + data=csharp_test_project_basic["data"], + name=name, + ) + assert isinstance(run, Run) + assert run.name == name diff --git a/src/promptflow-core/promptflow/_utils/flow_utils.py b/src/promptflow-core/promptflow/_utils/flow_utils.py index 1fba097f265..0962a675de8 100644 --- a/src/promptflow-core/promptflow/_utils/flow_utils.py +++ b/src/promptflow-core/promptflow/_utils/flow_utils.py @@ -194,7 +194,7 @@ def is_prompty_flow(file_path: Union[str, Path], raise_error: bool = False): return Path(file_path).suffix.lower() == PROMPTY_EXTENSION -def resolve_entry_file(entry: str, working_dir: Path) -> Optional[str]: +def resolve_python_entry_file(entry: str, working_dir: Path) -> Optional[str]: """Resolve entry file from entry. If entry is a local file, e.g. my.local.file:entry_function, return the local file: my/local/file.py and executor will import it from local file. diff --git a/src/promptflow-core/promptflow/core/_utils.py b/src/promptflow-core/promptflow/core/_utils.py index b4b92e757eb..b97d1ced01b 100644 --- a/src/promptflow-core/promptflow/core/_utils.py +++ b/src/promptflow-core/promptflow/core/_utils.py @@ -9,7 +9,7 @@ from jinja2 import Template from promptflow._constants import AZURE_WORKSPACE_REGEX_FORMAT -from promptflow._utils.flow_utils import is_flex_flow, resolve_entry_file, resolve_flow_path +from promptflow._utils.flow_utils import is_flex_flow, resolve_flow_path, resolve_python_entry_file from promptflow._utils.logger_utils import LoggerFactory from promptflow._utils.utils import _match_reference from promptflow._utils.yaml_utils import load_yaml @@ -44,7 +44,7 @@ def init_executable(*, flow_dag: dict = None, flow_path: Path = None, working_di if is_flex_flow(yaml_dict=flow_dag): entry = flow_dag.get("entry") - entry_file = resolve_entry_file(entry=entry, working_dir=working_dir) + entry_file = resolve_python_entry_file(entry=entry, working_dir=working_dir) from promptflow._core.entry_meta_generator import generate_flow_meta diff --git a/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py b/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py index 50f1e699de3..e607bb39d1f 100644 --- a/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py +++ b/src/promptflow-devkit/promptflow/_proxy/_python_executor_proxy.py @@ -10,7 +10,7 @@ from promptflow._core.run_tracker import RunTracker from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT, FLOW_TOOLS_JSON_GEN_TIMEOUT from promptflow._sdk._utils import can_accept_kwargs -from promptflow._utils.flow_utils import resolve_entry_file +from promptflow._utils.flow_utils import resolve_python_entry_file from promptflow._utils.logger_utils import bulk_logger from promptflow._utils.yaml_utils import load_yaml from promptflow.contracts.run_mode import RunMode @@ -44,7 +44,7 @@ def _generate_flow_json( # generate flow.json only for eager flow for now return generate_flow_meta( flow_directory=working_dir, - source_path=resolve_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), + source_path=resolve_python_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), data=flow_dag, dump=dump, timeout=timeout, diff --git a/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py b/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py index b18d45f9519..ccb5b051e5b 100644 --- a/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py +++ b/src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py @@ -5,7 +5,7 @@ from promptflow._constants import FlowEntryRegex from promptflow._core.entry_meta_generator import _generate_flow_meta from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT -from promptflow._utils.flow_utils import is_flex_flow, resolve_entry_file +from promptflow._utils.flow_utils import is_flex_flow, resolve_python_entry_file from ._base_inspector_proxy import AbstractInspectorProxy @@ -41,7 +41,7 @@ def get_entry_meta( # generate flow.json only for eager flow for now return _generate_flow_meta( flow_directory=working_dir, - source_path=resolve_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), + source_path=resolve_python_entry_file(entry=flow_dag.get("entry"), working_dir=working_dir), data=flow_dag, timeout=timeout, load_in_subprocess=load_in_subprocess, diff --git a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py index 04483203ff5..67f1223ce1f 100644 --- a/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py +++ b/src/promptflow-devkit/promptflow/_sdk/operations/_flow_operations.py @@ -39,7 +39,6 @@ _get_additional_includes, _merge_local_code_and_additional_includes, copy_tree_respect_template_and_ignore_file, - entry_string_to_callable, format_signature_type, generate_flow_tools_json, generate_random_string, @@ -55,7 +54,6 @@ is_flex_flow, is_prompty_flow, parse_variant, - resolve_entry_file, ) from promptflow._utils.yaml_utils import dump_yaml, load_yaml from promptflow.exceptions import ErrorTarget, UserErrorException @@ -1038,15 +1036,20 @@ def _infer_signature(entry: Union[Callable, FlexFlow, Flow, Prompty], include_pr flow_meta["init"] = init_dict format_signature_type(flow_meta) elif isinstance(entry, FlexFlow): - # TODO: this part will fail for csharp - entry_file = resolve_entry_file(entry=entry.entry, working_dir=entry.code) - entry_func = entry_string_to_callable(entry_file, entry.entry) + # non-python flow depends on dumped flow meta to infer signature + ProxyFactory().create_inspector_proxy(language=entry.language).prepare_metadata( + flow_file=entry.path, + working_dir=entry.code, + ) flow_meta, _, _ = FlowOperations._infer_signature_flex_flow( - entry=entry_func, language=entry.language, include_primitive_output=include_primitive_output + entry=entry.entry, + code=entry.code.as_posix(), + language=entry.language, + include_primitive_output=include_primitive_output, ) elif inspect.isclass(entry) or inspect.isfunction(entry): flow_meta, _, _ = FlowOperations._infer_signature_flex_flow( - entry=entry, include_primitive_output=include_primitive_output + entry=entry, include_primitive_output=include_primitive_output, language=FlowLanguage.Python ) else: # TODO support to get infer signature of dag flow @@ -1057,16 +1060,13 @@ def _infer_signature(entry: Union[Callable, FlexFlow, Flow, Prompty], include_pr def _infer_signature_flex_flow( entry: Union[Callable, str], *, + language: str, code: str = None, keep_entry: bool = False, validate: bool = True, - language: str = FlowLanguage.Python, include_primitive_output: bool = False, ) -> Tuple[dict, Path, List[str]]: - """Infer signature of a flow entry. - - Note that this is a Python only feature. - """ + """Infer signature of a flow entry.""" snapshot_list = None # resolve entry and code if isinstance(entry, str): @@ -1117,7 +1117,6 @@ def _infer_signature_flex_flow( # this path is actually not used flow = FlexFlow(path=code / FLOW_FLEX_YAML, code=code, data=flow_meta, entry=flow_meta["entry"]) flow._validate(raise_error=True) - flow_meta.pop("language", None) if include_primitive_output and "outputs" not in flow_meta: flow_meta["outputs"] = { @@ -1126,9 +1125,11 @@ def _infer_signature_flex_flow( } } - if not keep_entry: - flow_meta.pop("entry", None) - return flow_meta, code, snapshot_list + keys_to_keep = ["inputs", "outputs", "init"] + if keep_entry: + keys_to_keep.append("entry") + filtered_meta = {k: flow_meta[k] for k in keys_to_keep if k in flow_meta} + return filtered_meta, code, snapshot_list @monitor_operation(activity_name="pf.flows.infer_signature", activity_type=ActivityType.PUBLICAPI) def infer_signature(self, entry: Union[Callable, FlexFlow, Flow, Prompty], **kwargs) -> dict: diff --git a/src/promptflow-devkit/pyproject.toml b/src/promptflow-devkit/pyproject.toml index 78c2e135051..b4169508782 100644 --- a/src/promptflow-devkit/pyproject.toml +++ b/src/promptflow-devkit/pyproject.toml @@ -121,7 +121,8 @@ pf = "promptflow._cli.pf:main" [tool.pytest.ini_options] markers = [ "unittest", - "e2etest" + "e2etest", + "csharp" ] # junit - analyse and publish test results (https://github.com/EnricoMi/publish-unit-test-result-action) # durations - list the slowest test durations diff --git a/src/promptflow-devkit/tests/conftest.py b/src/promptflow-devkit/tests/conftest.py index a8184a3ab78..d366f7953cf 100644 --- a/src/promptflow-devkit/tests/conftest.py +++ b/src/promptflow-devkit/tests/conftest.py @@ -4,6 +4,7 @@ import tempfile from multiprocessing import Lock from pathlib import Path +from typing import TypedDict from unittest.mock import MagicMock, patch import pytest @@ -256,3 +257,44 @@ def reset_tracer_provider(): "opentelemetry.trace._TRACER_PROVIDER_SET_ONCE._done", False ): yield + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +def construct_csharp_test_project(flow_name: str) -> CSharpProject: + root_of_test_cases = os.getenv("CSHARP_TEST_PROJECTS_ROOT", None) + if not root_of_test_cases: + pytest.skip(reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT.") + root_of_test_cases = Path(root_of_test_cases) + return { + "flow_dir": (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix(), + "data": (root_of_test_cases / flow_name / "data.jsonl").as_posix(), + "init": (root_of_test_cases / flow_name / "init.json").as_posix(), + } + + +@pytest.fixture +def csharp_test_project_basic() -> CSharpProject: + return construct_csharp_test_project("Basic") + + +@pytest.fixture +def csharp_test_project_basic_chat() -> CSharpProject: + return construct_csharp_test_project("BasicChat") + + +@pytest.fixture +def csharp_test_project_function_mode_basic() -> CSharpProject: + return construct_csharp_test_project("FunctionModeBasic") + + +@pytest.fixture +def csharp_test_project_class_init_flex_flow() -> CSharpProject: + is_in_ci_pipeline = os.getenv("IS_IN_CI_PIPELINE", "false").lower() == "true" + if is_in_ci_pipeline: + pytest.skip(reason="need to avoid fetching connection from local pfs to enable this in ci") + return construct_csharp_test_project("ClassInitFlexFlow") diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py index d4eab61a457..3493f1f0d36 100644 --- a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_cli.py @@ -1,78 +1,15 @@ -import dataclasses import json import os import os.path import sys from pathlib import Path -from typing import Optional +from typing import TypedDict import pytest from promptflow._cli._pf.entry import main -class TestCase: - def __init__(self, root_of_test_cases: Path, flow_name: str, skip_reason: str = None): - self._flow_dir = (root_of_test_cases / flow_name / "bin" / "Debug" / "net6.0").as_posix() - self.data = (root_of_test_cases / flow_name / "data.jsonl").as_posix() - self.init = (root_of_test_cases / flow_name / "init.json").as_posix() - self._skip_reason = skip_reason - - @property - def flow_dir(self): - if self._skip_reason: - pytest.skip(self._skip_reason) - return self._flow_dir - - -@dataclasses.dataclass -class TestCases: - basic: TestCase - function_mode_basic: TestCase - basic_with_builtin_llm: TestCase - class_init_flex_flow: TestCase - basic_chat: TestCase - - def __init__(self, root_of_test_cases: Path): - is_in_ci_pipeline = os.getenv("IS_IN_CI_PIPELINE", "false").lower() == "true" - self.basic = TestCase(root_of_test_cases, "Basic") - self.function_mode_basic = TestCase(root_of_test_cases, "FunctionModeBasic") - self.class_init_flex_flow = TestCase( - root_of_test_cases, - "ClassInitFlexFlow", - "need to avoid fetching connection from local pfs to enable this in ci" if is_in_ci_pipeline else None, - ) - self.basic_chat = TestCase(root_of_test_cases, "BasicChat") - - package_root = Path(__file__).parent.parent.parent.parent.parent / "promptflow" - dev_connections_path = package_root / "connections.json" - - if is_in_ci_pipeline and not dev_connections_path.exists(): - dev_connections_path.write_text( - json.dumps( - { - "azure_open_ai_connection": { - "type": "AzureOpenAIConnection", - "value": { - "api_key": os.getenv("AZURE_OPENAI_API_KEY", "00000000000000000000000000000000"), - "api_base": os.getenv("AZURE_OPENAI_ENDPOINT", "https://openai.azure.com/"), - "api_type": "azure", - "api_version": "2023-07-01-preview", - }, - "module": "promptflow.connections", - } - } - ) - ) - print(f"Using dev connections file: {dev_connections_path}") - - -def get_root_test_cases() -> Optional[TestCases]: - target_path = os.getenv("CSHARP_TEST_CASES_ROOT", None) - target_path = Path(target_path or Path(__file__).parent) - return TestCases(target_path) - - # TODO: move this to a shared utility module def run_pf_command(*args, cwd=None): """Run a pf command with the given arguments and working directory. @@ -91,51 +28,57 @@ def run_pf_command(*args, cwd=None): os.chdir(origin_cwd) -root_test_cases = get_root_test_cases() +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str @pytest.mark.usefixtures( - "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" + "use_secrets_config_file", + "recording_injection", + "setup_local_connection", + "install_custom_tool_pkg", ) @pytest.mark.cli_test @pytest.mark.e2etest -@pytest.mark.skipif( - not os.getenv("CSHARP_TEST_CASES_ROOT", None), reason="No C# test cases found, please set CSHARP_TEST_CASES_ROOT." -) +@pytest.mark.csharp class TestCSharpCli: @pytest.mark.parametrize( - "test_case", + "target_fixture_name", [ - pytest.param(root_test_cases.basic, id="basic"), - pytest.param(root_test_cases.basic_chat, id="basic_chat"), - pytest.param(root_test_cases.function_mode_basic, id="function_mode_basic"), - pytest.param(root_test_cases.class_init_flex_flow, id="class_init_flex_flow"), + pytest.param("csharp_test_project_basic", id="basic"), + pytest.param("csharp_test_project_basic_chat", id="basic_chat"), + pytest.param("csharp_test_project_function_mode_basic", id="function_mode_basic"), + pytest.param("csharp_test_project_class_init_flex_flow", id="class_init_flex_flow"), ], ) - def test_pf_run_create(self, test_case: TestCase): + def test_pf_run_create(self, request, target_fixture_name: str): + test_case: CSharpProject = request.getfixturevalue(target_fixture_name) cmd = [ "run", "create", "--flow", - test_case.flow_dir, + test_case["flow_dir"], "--data", - test_case.data, + test_case["data"], ] - if os.path.exists(test_case.init): - cmd.extend(["--init", test_case.init]) + if os.path.exists(test_case["init"]): + cmd.extend(["--init", test_case["init"]]) run_pf_command(*cmd) @pytest.mark.parametrize( - "test_case", + "target_fixture_name", [ - pytest.param(root_test_cases.basic, id="basic"), - pytest.param(root_test_cases.basic_chat, id="basic_chat"), - pytest.param(root_test_cases.function_mode_basic, id="function_mode_basic"), - pytest.param(root_test_cases.class_init_flex_flow, id="class_init_flex_flow"), + pytest.param("csharp_test_project_basic", id="basic"), + pytest.param("csharp_test_project_basic_chat", id="basic_chat"), + pytest.param("csharp_test_project_function_mode_basic", id="function_mode_basic"), + pytest.param("csharp_test_project_class_init_flex_flow", id="class_init_flex_flow"), ], ) - def test_pf_flow_test(self, test_case: TestCase): - with open(test_case.data, "r") as f: + def test_pf_flow_test(self, request, target_fixture_name: str): + test_case: CSharpProject = request.getfixturevalue(target_fixture_name) + with open(test_case["data"], "r") as f: lines = f.readlines() if len(lines) == 0: pytest.skip("No data provided for the test case.") @@ -147,7 +90,7 @@ def test_pf_flow_test(self, test_case: TestCase): "flow", "test", "--flow", - test_case.flow_dir, + test_case["flow_dir"], "--inputs", ] for key, value in inputs.items(): @@ -157,12 +100,12 @@ def test_pf_flow_test(self, test_case: TestCase): value = f'"{value}"' cmd.extend([f"{key}={value}"]) - if os.path.exists(test_case.init): - cmd.extend(["--init", test_case.init]) + if os.path.exists(test_case["init"]): + cmd.extend(["--init", test_case["init"]]) run_pf_command(*cmd) - def test_flow_chat(self, monkeypatch, capsys): - flow_dir = root_test_cases.basic_chat.flow_dir + def test_flow_chat(self, monkeypatch, capsys, csharp_test_project_basic_chat: CSharpProject): + flow_dir = csharp_test_project_basic_chat["flow_dir"] # mock user input with pop so make chat list reversed chat_list = ["what is chat gpt?", "hi"] @@ -192,32 +135,21 @@ def mock_input(*args, **kwargs): assert "Hello world round 1: what is chat gpt?" in outerr.out @pytest.mark.skip(reason="need to update the test case") - def test_pf_run_create_with_connection_override(self): + def test_pf_run_create_with_connection_override(self, csharp_test_project_basic): run_pf_command( "run", "create", "--flow", - root_test_cases.basic_with_builtin_llm.flow_dir, + csharp_test_project_basic["flow_dir"], "--data", - root_test_cases.basic_with_builtin_llm.data, + csharp_test_project_basic["data"], "--connections", "get_answer.connection=azure_open_ai_connection", ) @pytest.mark.skip(reason="need to update the test case") def test_flow_chat_ui_streaming(self): - """Note that this test won't pass. Instead, it will hang and pop up a web page for user input. - Leave it here for debugging purpose. - """ - # The test need to interact with user input in ui - flow_dir = f"{root_test_cases}\\examples\\BasicChatFlowWithBuiltinLLM\\bin\\Debug\\net6.0" - run_pf_command( - "flow", - "test", - "--flow", - flow_dir, - "--ui", - ) + pass @pytest.mark.skip(reason="need to update the test case") def test_flow_run_from_resume(self): diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py new file mode 100644 index 00000000000..0d66d21bd2e --- /dev/null +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_csharp_sdk.py @@ -0,0 +1,55 @@ +from typing import TypedDict + +import pytest + +from promptflow._sdk._load_functions import load_flow +from promptflow._sdk._pf_client import PFClient + +_client = PFClient() + + +class CSharpProject(TypedDict): + flow_dir: str + data: str + init: str + + +@pytest.mark.usefixtures( + "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg" +) +@pytest.mark.sdk_test +@pytest.mark.e2etest +@pytest.mark.csharp +class TestCSharpSdk: + @pytest.mark.parametrize( + "expected_signature", + [ + pytest.param( + { + "init": {}, + "inputs": { + "language": {"default": "chinese", "type": "string"}, + "topic": {"default": "ocean", "type": "string"}, + }, + "outputs": {"output": {"type": "object"}}, + }, + id="function_mode_basic", + ), + pytest.param( + { + "init": {"connection": {"type": "AzureOpenAIConnection"}, "name": {"type": "string"}}, + "inputs": {"question": {"default": "What is Promptflow?", "type": "string"}}, + "outputs": {"output": {"type": "object"}}, + }, + id="class_init_flex_flow", + ), + ], + ) + def test_pf_run_create(self, request, expected_signature: dict): + test_case: CSharpProject = request.getfixturevalue(f"csharp_test_project_{request.node.callspec.id}") + flow = load_flow(test_case["flow_dir"]) + signature = _client.flows._infer_signature( + flow, + include_primitive_output=True, + ) + assert signature == expected_signature diff --git a/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py new file mode 100644 index 00000000000..e4c889509b0 --- /dev/null +++ b/src/promptflow-devkit/tests/sdk_pfs_test/e2etests/test_csharp.py @@ -0,0 +1,50 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +import pytest + +from ..utils import PFSOperations, check_activity_end_telemetry + + +@pytest.mark.csharp +@pytest.mark.e2etest +class TestCSharp: + def test_get_flow_yaml(self, pfs_op: PFSOperations, csharp_test_project_basic) -> None: + with check_activity_end_telemetry(expected_activities=[]): + flow_yaml_from_pfs = pfs_op.get_flow_yaml(flow_path=csharp_test_project_basic["flow_dir"]).data.decode( + "utf-8" + ) + assert flow_yaml_from_pfs == ( + "$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json\n" + "display_name: Basic\n" + "language: csharp\n" + "inputs:\n" + " question:\n" + " type: string\n" + " default: what is promptflow?\n" + "outputs:\n" + " answer:\n" + " type: string\n" + " reference: ${get_answer.output}\n" + "nodes:\n" + "- name: get_answer\n" + " type: csharp\n" + " source:\n" + " type: package\n" + " tool: (Basic)Basic.Flow.HelloWorld\n" + " inputs:\n" + " question: ${inputs.question}\n" + ) + + def test_get_eager_flow_yaml(self, pfs_op: PFSOperations, csharp_test_project_function_mode_basic) -> None: + with check_activity_end_telemetry(expected_activities=[]): + flow_yaml_from_pfs = pfs_op.get_flow_yaml( + flow_path=csharp_test_project_function_mode_basic["flow_dir"] + ).data.decode("utf-8") + assert flow_yaml_from_pfs == ( + "$schema: https://azuremlschemas.azureedge.net/promptflow/latest/flow.schema.json\n" + "\n" + "language: csharp\n" + "entry: (FunctionModeBasic)FunctionModeBasic.MyEntry.WritePoemReturnObjectAsync\n" + ) diff --git a/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml b/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml new file mode 100644 index 00000000000..c5cb0d2d2f1 --- /dev/null +++ b/src/promptflow-recording/recordings/azure/test_csharp_sdk_TestCSharpSdk_test_basic_run_bulk.yaml @@ -0,0 +1,615 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000?api-version=2023-08-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '3678' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.033' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?api-version=2023-04-01-preview&count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1372' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.057' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore?api-version=2023-04-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.080' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2023-04-01-preview + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.148' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:03:59 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/data.jsonl + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '122' + content-md5: + - ADhs+/4nCwfVwpMYGRMDNQ== + content-type: + - application/octet-stream + last-modified: + - Fri, 19 Apr 2024 10:02:27 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Fri, 19 Apr 2024 10:02:26 GMT + x-ms-meta-name: + - 592b9e51-3a0b-4e5d-a6b5-ce54e92aaea5 + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - 1f2929ae-366f-4153-a8a5-609a2e769f07 + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:01 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/data.jsonl + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore?api-version=2023-04-01-preview + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2023-04-08T02:53:06.5886442+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2023-04-08T02:53:07.521127+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1227' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.072' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azure-ai-ml/1.15.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2023-04-01-preview + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.074' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:06 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/LocalUpload/000000000000000000000000000000000000/net6.0/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '2762' + content-md5: + - QymQjkUtk24EtHQGHDOiIQ== + content-type: + - application/octet-stream + last-modified: + - Fri, 19 Apr 2024 10:02:43 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + vary: + - Origin + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Fri, 19 Apr 2024 10:02:37 GMT + x-ms-meta-name: + - 81d0ee87-a9db-4cb3-8c1e-b5eb42183fec + x-ms-meta-upload_status: + - completed + x-ms-meta-version: + - '1' + x-ms-version: + - '2023-11-03' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.19.1 Python/3.11.8 (Windows-10-10.0.22631-SP0) + x-ms-date: + - Fri, 19 Apr 2024 10:04:07 GMT + x-ms-version: + - '2023-11-03' + method: HEAD + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/az-ml-artifacts/000000000000000000000000000000000000/net6.0/.promptflow/flow.detail.json + response: + body: + string: '' + headers: + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + vary: + - Origin + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2023-11-03' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: '{"flowDefinitionDataStoreName": "workspaceblobstore", "flowDefinitionBlobPath": + "LocalUpload/000000000000000000000000000000000000/net6.0/flow.dag.yaml", "runId": + "name", "runDisplayName": "name", "runExperimentName": "", "sessionId": "000000000000000000000000000000000000000000000000", + "sessionSetupMode": "SystemWait", "flowLineageId": "0000000000000000000000000000000000000000000000000000000000000000", + "runDisplayNameGenerationType": "UserProvidedMacro", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/000000000000000000000000000000000000/data.jsonl"}, + "inputsMapping": {}, "environmentVariables": {}, "connections": {}, "runtimeName": + "fake-runtime-name"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '755' + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/submit + response: + body: + string: '"name"' + headers: + connection: + - keep-alive + content-length: + - '38' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '4.005' + status: + code: 200 + message: OK +- request: + body: '{"runId": "name", "selectRunMetadata": true, "selectRunDefinition": true, + "selectJobSpecification": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://eastus.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/rundata + response: + body: + string: '{"runMetadata": {"runNumber": 1713521054, "rootRunId": "name", "createdUtc": + "2024-04-19T10:04:14.0934449+00:00", "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "10032001D9C91417", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Xingzhi Zhang", "upn": + null}, "userId": "00000000-0000-0000-0000-000000000000", "token": null, "tokenExpiryTimeUtc": + null, "error": null, "warnings": null, "revision": 1, "statusRevision": 0, + "runUuid": "a656e9bc-91bf-427e-8950-e21ca2065d67", "parentRunUuid": null, + "rootRunUuid": "a656e9bc-91bf-427e-8950-e21ca2065d67", "lastStartTimeUtc": + null, "currentComputeTime": "00:00:00", "computeDuration": null, "effectiveStartTimeUtc": + null, "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "10032001D9C91417", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Xingzhi Zhang", "upn": + null}, "lastModifiedUtc": "2024-04-19T10:04:14.0934449+00:00", "duration": + null, "cancelationReason": null, "currentAttemptId": 1, "runId": "name", "parentRunId": + null, "experimentId": "36dad2bf-add3-491b-8ad4-1c85de215cfa", "status": "NotStarted", + "startTimeUtc": null, "endTimeUtc": null, "scheduleId": null, "displayName": + "name", "name": null, "dataContainerId": "dcid.name", "description": null, + "hidden": false, "runType": "azureml.promptflow.FlowRun", "runTypeV2": {"orchestrator": + null, "traits": [], "attribution": "PromptFlow", "computeType": null}, "properties": + {"azureml.promptflow.runtime_name": "automatic", "azureml.promptflow.input_data": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/07a51eb4af1ca3b9af2b51008b387aab/data.jsonl", + "azureml.promptflow.disable_trace": "false", "azureml.promptflow.session_id": + "c3329398d4d114830396f085268d91e6ccc4d57b70e80e55", "azureml.promptflow.definition_file_name": + "flow.dag.yaml", "azureml.promptflow.flow_lineage_id": "fbbdc468be255ca5fd167cf97724ad8446094632682e031eda2669d9604572a4", + "azureml.promptflow.flow_definition_datastore_name": "workspaceblobstore", + "azureml.promptflow.flow_definition_blob_path": "LocalUpload/56ddfd1ec40da6b8b0b2e65a99b55e56/net6.0/flow.dag.yaml", + "_azureml.evaluation_run": "promptflow.BatchRun"}, "parameters": {}, "actionUris": + {}, "scriptName": null, "target": null, "uniqueChildRunComputeTargets": [], + "tags": {}, "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": + [], "runDefinition": null, "jobSpecification": null, "primaryMetricName": + null, "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}, "runDefinition": + null, "jobSpecification": null, "systemSettings": null}' + headers: + connection: + - keep-alive + content-length: + - '3599' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.067' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/1.8.0.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.8 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://eastus.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/name + response: + body: + string: '{"flowRunResourceId": "azureml://locations/eastus/workspaces/00000/flows/name/flowRuns/name", + "flowRunId": "name", "flowRunDisplayName": "name", "batchDataInput": {"dataUri": + "azureml://datastores/workspaceblobstore/paths/LocalUpload/07a51eb4af1ca3b9af2b51008b387aab/data.jsonl"}, + "flowRunType": "FlowRun", "flowType": "Default", "runtimeName": "automatic", + "inputsMapping": {}, "outputDatastoreName": "workspaceblobstore", "childRunBasePath": + "promptflow/PromptFlowArtifacts/name/flow_artifacts", "sessionId": "c3329398d4d114830396f085268d91e6ccc4d57b70e80e55", + "studioPortalEndpoint": "https://ml.azure.com/runs/name?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000"}' + headers: + connection: + - keep-alive + content-length: + - '975' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.241' + status: + code: 200 + message: OK +version: 1 From e6fd0d25479724690adee2e25ccae368eb96aa6d Mon Sep 17 00:00:00 2001 From: Zhengfei Wang <38847871+zhengfeiwang@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:01:49 +0800 Subject: [PATCH 06/14] [trace][azure] Fix inappropriate Azure trace UI links (#2926) # Description This pull request fixes inappropriate Azure trace UI links: remove int region, and remove an unexpected flight. Besides, fix a missing rename in test. # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [x] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [x] Title of the pull request is clear and informative. - [x] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../tests/sdk_cli_azure_test/conftest.py | 12 ++++-------- .../sdk_cli_azure_test/e2etests/test_run_upload.py | 12 ++++++------ src/promptflow-devkit/promptflow/_sdk/_tracing.py | 6 +++--- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py index e6643bd60a8..a7632c359ab 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py @@ -614,17 +614,13 @@ def mock_check_latest_version() -> None: @pytest.fixture -def mock_trace_provider_to_cloud( - subscription_id: str, - resource_group_name: str, - workspace_name: str, -) -> None: - """Mock trace provider to cloud.""" - trace_provider = ( +def mock_trace_destination_to_cloud(subscription_id: str, resource_group_name: str, workspace_name: str): + """Mock trace destination to cloud.""" + trace_destination = ( f"azureml://subscriptions/{subscription_id}/resourceGroups/{resource_group_name}/" f"providers/Microsoft.MachineLearningServices/workspaces/{workspace_name}" ) - with patch("promptflow._sdk._configuration.Configuration.get_trace_provider", return_value=trace_provider): + with patch("promptflow._sdk._configuration.Configuration.get_trace_destination", return_value=trace_destination): yield diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py index 2ccefc75d93..145a7505e70 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py @@ -67,7 +67,7 @@ def check_local_to_cloud_run(pf: PFClient, run: Run): class TestFlowRunUpload: @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_run( self, @@ -94,7 +94,7 @@ def test_upload_run( @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_flex_flow_run_with_yaml(self, pf: PFClient, randstr: Callable[[str], str]): name = randstr("flex_run_name_with_yaml_for_upload") @@ -116,7 +116,7 @@ def test_upload_flex_flow_run_with_yaml(self, pf: PFClient, randstr: Callable[[s @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_flex_flow_run_without_yaml(self, pf: PFClient, randstr: Callable[[str], str]): name = randstr("flex_run_name_without_yaml_for_upload") @@ -139,7 +139,7 @@ def test_upload_flex_flow_run_without_yaml(self, pf: PFClient, randstr: Callable @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_prompty_run(self, pf: PFClient, randstr: Callable[[str], str]): # currently prompty run is skipped for upload, this test should be finished without error @@ -155,7 +155,7 @@ def test_upload_prompty_run(self, pf: PFClient, randstr: Callable[[str], str]): @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_run_with_customized_run_properties(self, pf: PFClient, randstr: Callable[[str], str]): name = randstr("batch_run_name_for_upload_with_customized_properties") @@ -190,7 +190,7 @@ def test_upload_run_with_customized_run_properties(self, pf: PFClient, randstr: @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( - "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_provider_to_cloud" + "mock_isinstance_for_mock_datastore", "mock_get_azure_pf_client", "mock_trace_destination_to_cloud" ) def test_upload_eval_run(self, pf: PFClient, randstr: Callable[[str], str]): main_run_name = randstr("main_run_name_for_test_upload_eval_run") diff --git a/src/promptflow-devkit/promptflow/_sdk/_tracing.py b/src/promptflow-devkit/promptflow/_sdk/_tracing.py index 332420b9599..3b14b0fd1d7 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_tracing.py +++ b/src/promptflow-devkit/promptflow/_sdk/_tracing.py @@ -226,12 +226,12 @@ def _print_tracing_url_from_azure_portal( run: typing.Optional[str] = None, ) -> None: url = ( - "https://int.ml.azure.com/{query}?" + "https://ml.azure.com/{query}?" f"wsid=/subscriptions/{ws_triad.subscription_id}" f"/resourceGroups/{ws_triad.resource_group_name}" "/providers/Microsoft.MachineLearningServices" f"/workspaces/{ws_triad.workspace_name}" - "&flight=PFTrace,PFNewRunDetail" + "&flight=PFTrace" ) if run is None: @@ -247,7 +247,7 @@ def _print_tracing_url_from_azure_portal( query = f"prompts/trace/run/{run}/details" elif AzureWorkspaceKind.is_project(kind): _logger.debug(f"{ws_triad.workspace_name!r} is an Azure AI project.") - url = url.replace("int.ml.azure.com", "int.ai.azure.com") + url = url.replace("ml.azure.com", "ai.azure.com") if run is None: query = f"projecttrace/collection/{collection_id}/list" else: From e12f0e3efdfce4da268a2de5ff160f6b5a55e810 Mon Sep 17 00:00:00 2001 From: Robben Wang <350053002@qq.com> Date: Mon, 22 Apr 2024 17:03:19 +0800 Subject: [PATCH 07/14] Handle job cancellation for root span (#2889) # Description 1. Record KeyboardInterrupt exception for span (otlp's code doesn't handle BaseException but only Exception) 2. Raise KeyboardInterrupt in FlowNodesScheduler 3. Remove `with ThreadPoolExecutor` to guarantee KeyboardInterrupt could be received by outside. Why make this change: **we want to support cancellation for span.** We have 2 schedulers, one is FlowNodesScheduler, another is AsyncNodesScheduler. For most cases, we are using FlowNodesScheduler. In FlowNodesScheduler, there is `with ThreadPoolExecutor`, which will wait all running threads finish, no matter what exception is raised. That's why our cancellation is blocked. ![image](https://github.com/microsoft/promptflow/assets/17527303/b6a1a63a-8425-445c-95ec-bf4d2d833787) So, in this PR, remove the `with ThreadPoolExecutor` to make the KeyboardInterrupt could be received by outside. Why also change other tests: For now, our exception could be received by outside immediately, the exception type and execution time are affected. For test_batch_timeout, situation is very complex. The test set 5 seconds batch run timeout and 600 seconds line run timeout. But actually, for batch run execution, line run timeout is calculated by batch run timeout. So, 600 seconds won't take effect, the real line run timeout is less than 5 seconds. But before this change, even we raise line run timeout exception, we can't exit ThreadPoolExecutor to finish line run. From process perspective, the line run has never finish, so assign one `BatchExecutionTimeoutError`. After this change, line run timeout exception could be raised out soon, so the behavior is changed. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --------- Co-authored-by: robbenwang --- .../executor/_flow_nodes_scheduler.py | 121 ++++++++++++------ .../promptflow/executor/flow_executor.py | 17 ++- .../executor/e2etests/test_batch_timeout.py | 26 +++- .../e2etests/test_concurent_execution.py | 2 +- .../tests/executor/e2etests/test_logs.py | 6 +- 5 files changed, 121 insertions(+), 51 deletions(-) diff --git a/src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py b/src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py index 285ba14b37f..e4b460a512e 100644 --- a/src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py +++ b/src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py @@ -5,7 +5,10 @@ import asyncio import contextvars import inspect +import os +import signal import threading +import time from concurrent import futures from concurrent.futures import Future, ThreadPoolExecutor from typing import Dict, List, Optional, Tuple @@ -13,6 +16,7 @@ from promptflow._core.flow_execution_context import FlowExecutionContext from promptflow._core.tools_manager import ToolsManager from promptflow._utils.logger_utils import flow_logger +from promptflow._utils.thread_utils import ThreadWithContextVars from promptflow._utils.utils import set_context from promptflow.contracts.flow import Node from promptflow.executor._dag_manager import DAGManager @@ -23,6 +27,33 @@ DEFAULT_CONCURRENCY_FLOW = 16 +def signal_handler(sig, frame): + """ + In main thread, we raise KeyboardInterrupt to mark run as cancelled and exit execution. + But the process may not exit immediately because there are worker threads running from ThreadPoolExecutor. + So, start new worker thread and use os._exit to guarantee the process could exit after delay time. + """ + + flow_logger.info(f"Received signal {sig}({signal.Signals(sig).name}), start to exit sync flow execution.") + + def exit_process_with_delay(): + # Adding 3 seconds delay here to let the main thread to finish the cleanup work, such as exporting spans. + time.sleep(3) + # Use os._exit instead of sys.exit, so that the process can stop without + # waiting for the thread created by ThreadPoolExecutor to finish. + # sys.exit: https://docs.python.org/3/library/sys.html#sys.exit + # Raise a SystemExit exception, signaling an intention to exit the interpreter. + # Specifically, it does not exit non-daemon thread + # os._exit https://docs.python.org/3/library/os.html#os._exit + # Exit the process with status n, without calling cleanup handlers, flushing stdio buffers, etc. + # Specifically, it stops process without waiting for non-daemon thread. + os._exit(0) + + monitor = ThreadWithContextVars(target=exit_process_with_delay) + monitor.start() + raise KeyboardInterrupt + + class FlowNodesScheduler: def __init__( self, @@ -51,45 +82,61 @@ def execute( self, line_timeout_sec: Optional[int] = None, ) -> Tuple[dict, dict]: + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + else: + flow_logger.info( + "Current thread is not main thread, skip signal handler registration in AsyncNodesScheduler." + ) parent_context = contextvars.copy_context() - with ThreadPoolExecutor( + # If we use `with ThreadPoolExecutor`, the __exit__ method will be called to wait all threads are finished. + # Then any exception raised in the threads will be blocked, which is unexpected. + # So, we just create the executor and shutdown it manually. + executor = ThreadPoolExecutor( max_workers=self._node_concurrency, initializer=set_context, initargs=(parent_context,) - ) as executor: - self._execute_nodes(executor) - timeout_task = None - event = threading.Event() - if line_timeout_sec is not None: - timeout_task = executor.submit(self.wait_within_timeout, event, line_timeout_sec) - try: - while not self._dag_manager.completed(): - if not self._future_to_node: - raise NoNodeExecutedError("No nodes are ready for execution, but the flow is not completed.") - tasks_to_wait = list(self._future_to_node.keys()) - if timeout_task is not None: - tasks_to_wait.append(timeout_task) - completed_futures_with_wait, _ = futures.wait(tasks_to_wait, return_when=futures.FIRST_COMPLETED) - completed_futures = [f for f in completed_futures_with_wait if f in self._future_to_node] - self._dag_manager.complete_nodes(self._collect_outputs(completed_futures)) - for each_future in completed_futures: - del self._future_to_node[each_future] - if timeout_task and timeout_task.done(): - raise LineExecutionTimeoutError(self._context._line_number, line_timeout_sec) - self._execute_nodes(executor) - except Exception as e: - err_msg = "Flow execution has failed." - if isinstance(e, LineExecutionTimeoutError): - err_msg = f"Line execution timeout after {line_timeout_sec} seconds." - self._context.cancel_node_runs(err_msg) - node_names = ",".join(node.name for node in self._future_to_node.values()) - flow_logger.error(f"{err_msg} Cancelling all running nodes: {node_names}.") - for unfinished_future in self._future_to_node.keys(): - # We can't cancel running tasks here, only pending tasks could be cancelled. - unfinished_future.cancel() - # Even we raise exception here, still need to wait all running jobs finish to exit. - raise e - finally: - # Cancel timeout task no matter the execution is finished or failed. - event.set() + ) + self._execute_nodes(executor) + timeout_task = None + event = threading.Event() + if line_timeout_sec is not None: + timeout_task = executor.submit(self.wait_within_timeout, event, line_timeout_sec) + try: + while not self._dag_manager.completed(): + if not self._future_to_node: + raise NoNodeExecutedError("No nodes are ready for execution, but the flow is not completed.") + tasks_to_wait = list(self._future_to_node.keys()) + if timeout_task is not None: + tasks_to_wait.append(timeout_task) + completed_futures_with_wait, _ = futures.wait(tasks_to_wait, return_when=futures.FIRST_COMPLETED) + completed_futures = [f for f in completed_futures_with_wait if f in self._future_to_node] + self._dag_manager.complete_nodes(self._collect_outputs(completed_futures)) + for each_future in completed_futures: + del self._future_to_node[each_future] + if timeout_task and timeout_task.done(): + raise LineExecutionTimeoutError(self._context._line_number, line_timeout_sec) + self._execute_nodes(executor) + event.set() + executor.shutdown() + except Exception as e: + err_msg = "Flow execution has failed." + if isinstance(e, LineExecutionTimeoutError): + err_msg = f"Line execution timeout after {line_timeout_sec} seconds." + self._context.cancel_node_runs(err_msg) + node_names = ",".join(node.name for node in self._future_to_node.values()) + flow_logger.error(f"{err_msg} Cancelling all running nodes: {node_names}.") + for unfinished_future in self._future_to_node.keys(): + # We can't cancel running tasks here, only pending tasks could be cancelled. + unfinished_future.cancel() + raise e + finally: + # When meet exception, mark event to exit timeout task. + event.set() + # Use shutdown(wait=False) to ignore running threads. + # When meet exception, we want to exit the execution and mark it as failed/cancelled, + # so don't need to wait for running threads. + # If not meet exception, below shutdown will not take any effect. + executor.shutdown(wait=False) for node in self._dag_manager.bypassed_nodes: self._dag_manager.completed_nodes_outputs[node] = None return self._dag_manager.completed_nodes_outputs, self._dag_manager.bypassed_nodes diff --git a/src/promptflow-core/promptflow/executor/flow_executor.py b/src/promptflow-core/promptflow/executor/flow_executor.py index 62106d8b571..09c60be1b69 100644 --- a/src/promptflow-core/promptflow/executor/flow_executor.py +++ b/src/promptflow-core/promptflow/executor/flow_executor.py @@ -18,7 +18,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union import opentelemetry.trace as otel_trace -from opentelemetry.trace.span import format_trace_id +from opentelemetry.trace.span import Span, format_trace_id from opentelemetry.trace.status import StatusCode from promptflow._constants import LINE_NUMBER_KEY @@ -847,7 +847,7 @@ def _exec_inner_with_trace( ): # need to get everytime to ensure tracer is latest otel_tracer = otel_trace.get_tracer("promptflow") - with otel_tracer.start_as_current_span(self._flow.name) as span: + with otel_tracer.start_as_current_span(self._flow.name) as span, self._record_keyboard_interrupt_to_span(span): # Store otel trace id in context for correlation OperationContext.get_instance()["otel_trace_id"] = f"0x{format_trace_id(span.get_span_context().trace_id)}" # initialize span @@ -874,6 +874,16 @@ def _exec_inner_with_trace( span.set_status(StatusCode.OK) return output, aggregation_inputs + @contextlib.contextmanager + def _record_keyboard_interrupt_to_span(self, span: Span): + try: + yield + except KeyboardInterrupt as ex: + if span.is_recording(): + span.record_exception(ex) + span.set_status(StatusCode.ERROR, "Execution cancelled.") + raise + def _exec_inner( self, inputs: Mapping[str, Any], @@ -966,9 +976,6 @@ def _exec( # Update the run info of those running nodes to a canceled status. run_tracker.cancel_node_runs(run_id) run_tracker.end_run(line_run_id, ex=ex) - # If async execution is enabled, ignore this exception and return the partial line results. - if not self._should_use_async(): - raise except Exception as ex: run_tracker.end_run(line_run_id, ex=ex) if self._raise_ex: diff --git a/src/promptflow/tests/executor/e2etests/test_batch_timeout.py b/src/promptflow/tests/executor/e2etests/test_batch_timeout.py index eabc9e933a5..fb4d88fafb2 100644 --- a/src/promptflow/tests/executor/e2etests/test_batch_timeout.py +++ b/src/promptflow/tests/executor/e2etests/test_batch_timeout.py @@ -9,7 +9,7 @@ from promptflow.batch._result import BatchResult, LineError from promptflow.contracts.run_info import Status from promptflow.exceptions import ErrorTarget -from promptflow.executor._errors import BatchExecutionTimeoutError, LineExecutionTimeoutError +from promptflow.executor._errors import LineExecutionTimeoutError from ..utils import MemoryRunStorage, get_flow_folder, get_flow_inputs_file, get_yaml_file @@ -123,13 +123,24 @@ def test_batch_with_one_line_timeout(self, flow_folder, dev_connections): assert len(mem_run_storage._node_runs) == 6, "Node runs are persisted in memory storage." @pytest.mark.parametrize( - "flow_folder, line_timeout_sec, batch_timeout_sec, expected_error, batch_run_status", + "flow_folder, line_timeout_sec, batch_timeout_sec, expected_error_message, batch_run_status", [ - (ONE_LINE_OF_BULK_TEST_TIMEOUT, 600, 5, BatchExecutionTimeoutError(2, 5), Status.Failed), - (ONE_LINE_OF_BULK_TEST_TIMEOUT, 3, 600, LineExecutionTimeoutError(2, 3), Status.Completed), + # For the first case, the line timeout will not take effect + # because the real line timeout is calculated according to batch run timeout. + # So, both cases will raise LineExecutionTimeoutError + (ONE_LINE_OF_BULK_TEST_TIMEOUT, 600, 5, "Line 2 execution timeout for exceeding", Status.Failed), + ( + ONE_LINE_OF_BULK_TEST_TIMEOUT, + 3, + 600, + "Line 2 execution timeout for exceeding 3 seconds", + Status.Completed, + ), ], ) - def test_batch_timeout(self, flow_folder, line_timeout_sec, batch_timeout_sec, expected_error, batch_run_status): + def test_batch_timeout( + self, flow_folder, line_timeout_sec, batch_timeout_sec, expected_error_message, batch_run_status + ): mem_run_storage = MemoryRunStorage() batch_engine = BatchEngine( get_yaml_file(flow_folder), @@ -175,9 +186,10 @@ def test_batch_timeout(self, flow_folder, line_timeout_sec, batch_timeout_sec, e assert isinstance(actual_line_error, LineError) assert actual_line_error.line_number == 2 actual_error_dict = actual_line_error.error - expected_error_dict = ExceptionPresenter.create(expected_error).to_dict() + expected_error_dict = ExceptionPresenter.create(LineExecutionTimeoutError(2, 1)).to_dict() assert actual_error_dict["code"] == expected_error_dict["code"] - assert actual_error_dict["message"] == expected_error_dict["message"] + # We can't assert the exact message because timeout it's dynamic + assert expected_error_message in actual_error_dict["message"] assert actual_error_dict["referenceCode"] == expected_error_dict["referenceCode"] assert actual_error_dict["innerError"]["code"] == expected_error_dict["innerError"]["code"] diff --git a/src/promptflow/tests/executor/e2etests/test_concurent_execution.py b/src/promptflow/tests/executor/e2etests/test_concurent_execution.py index c2a1518ecc1..394e61bf17d 100644 --- a/src/promptflow/tests/executor/e2etests/test_concurent_execution.py +++ b/src/promptflow/tests/executor/e2etests/test_concurent_execution.py @@ -47,7 +47,7 @@ def test_concurrent_run(self): def test_concurrent_run_with_exception(self): executor = FlowExecutor.create(get_yaml_file(FLOW_FOLDER), {}, raise_ex=False) flow_result = executor.exec_line({"input1": "True", "input2": "False", "input3": "False", "input4": "False"}) - assert 2 < flow_result.run_info.system_metrics["duration"] < 4, "Should at least finish the running job." + assert 1 > flow_result.run_info.system_metrics["duration"], "Will finish quickly with exception." error_response = ErrorResponse.from_error_dict(flow_result.run_info.error) assert error_response.error_code_hierarchy == "UserError/ToolExecutionError" diff --git a/src/promptflow/tests/executor/e2etests/test_logs.py b/src/promptflow/tests/executor/e2etests/test_logs.py index 1493ebf1426..21c2c00c915 100644 --- a/src/promptflow/tests/executor/e2etests/test_logs.py +++ b/src/promptflow/tests/executor/e2etests/test_logs.py @@ -133,7 +133,11 @@ def test_batch_run_flow_logs(self, flow_root_dir, flow_folder_name, line_number) assert "execution WARNING" in log_content assert "execution.flow INFO" in log_content assert f"in line {i} (index starts from 0)" in log_content - assert line_number == count_lines(flow_log_file) + # Some monitor logs may not be printed in CI test. + # Assert max line number to avoid printing too many noisy logs. + assert line_number == count_lines( + flow_log_file + ), f"log line count is incorrect, content is {log_content}" @pytest.mark.parametrize( "folder_name", From f8186c501736654d406f4ac3fed5b8c1bb768535 Mon Sep 17 00:00:00 2001 From: Honglin Date: Mon, 22 Apr 2024 17:27:15 +0800 Subject: [PATCH 08/14] [SDK/CLI] Upload instance_result.jsonl and flow_logs (#2924) # Description - Upload `flow_logs`, which is the detailed logs for each line run. - Upload `instance_results.jsonl`, which is a summary of each line run. Cloud run has this to let UI know metrics like `failure rate`. ```jsonl {"line_number": 2, "status": "Completed", "inputs.url": "https://www.youtube.com/watch?v=o5ZQyXaAv1g", "inputs.line_number": 2, "category": "Movie", "evidence": "Text content"} {"line_number": 1, "status": "Completed", "inputs.url": "https://www.youtube.com/watch?v=o5ZQyXaAv1g", "inputs.line_number": 1, "category": "Movie", "evidence": "Text content"} {"line_number": 0, "status": "Completed", "inputs.url": "https://www.youtube.com/watch?v=o5ZQyXaAv1g", "inputs.line_number": 0, "category": "Movie", "evidence": "Text content"} ``` # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../azure/operations/_async_run_uploader.py | 89 +++++++++++++++++-- .../e2etests/test_run_upload.py | 14 ++- .../promptflow/_cli/_utils.py | 45 ++++++++++ .../promptflow/_sdk/_constants.py | 2 + 4 files changed, 139 insertions(+), 11 deletions(-) diff --git a/src/promptflow-azure/promptflow/azure/operations/_async_run_uploader.py b/src/promptflow-azure/promptflow/azure/operations/_async_run_uploader.py index 41b569c40b8..327abfe9cf4 100644 --- a/src/promptflow-azure/promptflow/azure/operations/_async_run_uploader.py +++ b/src/promptflow-azure/promptflow/azure/operations/_async_run_uploader.py @@ -3,12 +3,12 @@ import shutil import tempfile from pathlib import Path -from typing import Dict +from typing import Dict, List from azure.core.exceptions import HttpResponseError, ResourceExistsError from azure.storage.blob.aio import BlobServiceClient -from promptflow._cli._utils import merge_jsonl_files +from promptflow._cli._utils import get_instance_results, merge_jsonl_files from promptflow._constants import OutputsFolderName from promptflow._sdk._constants import ( DEFAULT_ENCODING, @@ -94,8 +94,10 @@ async def upload(self) -> Dict: self._upload_flow_artifacts(), self._upload_node_artifacts(), self._upload_run_outputs(), - self._upload_logs(), + self._upload_logs(), # overall logs self._upload_snapshot(), + self._upload_flow_logs(), # detailed logs for each line run + self._upload_instance_results(), ] results = await asyncio.gather(*tasks) @@ -106,6 +108,8 @@ async def upload(self) -> Dict: OutputsFolderName.FLOW_OUTPUTS: results[2], LocalStorageFilenames.LOG: results[3], LocalStorageFilenames.SNAPSHOT_FOLDER: results[4], + LocalStorageFilenames.FLOW_LOGS_FOLDER: results[5], + Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME: results[6], } return result_dict @@ -130,8 +134,7 @@ async def _upload_flow_artifacts(self) -> str: await self._upload_local_folder_to_blob(temp_local_folder, remote_folder) # upload updated meta.json to cloud await self._upload_meta_json(temp_local_folder) - - return f"{remote_folder}/{OutputsFolderName.FLOW_ARTIFACTS}" + return f"{remote_folder}/{OutputsFolderName.FLOW_ARTIFACTS}" async def _upload_meta_json(self, temp_dir: Path): """ @@ -166,9 +169,9 @@ async def _upload_run_outputs(self) -> str: return f"{remote_folder}/{OutputsFolderName.FLOW_OUTPUTS}" async def _upload_logs(self) -> str: - """Upload logs to cloud. Return the cloud relative path of logs file""" + """Upload overall logs to cloud. Return the cloud relative path of logs file""" logger.debug(f"Uploading logs for run {self.run.name!r}.") - local_file = self.run_output_path / "logs.txt" + local_file = self.run_output_path / LocalStorageFilenames.LOG remote_file = f"{Local2Cloud.BLOB_EXPERIMENT_RUN}/dcid.{self.run.name}/{Local2Cloud.EXECUTION_LOG}" await self._upload_local_file_to_blob(local_file, remote_file, target_datastore=CloudDatastore.ARTIFACT) return remote_file @@ -185,7 +188,7 @@ async def _upload_snapshot(self) -> str: shutil.copytree(local_folder, temp_local_folder) remote_folder = f"{Local2Cloud.BLOB_ROOT_RUNS}" await self._upload_local_folder_to_blob(temp_local_folder, remote_folder) - return f"{remote_folder}/{self.run.name}/{flow_file}" + return f"{remote_folder}/{self.run.name}/{flow_file}" async def _upload_metrics(self) -> None: """Upload run metrics to cloud.""" @@ -194,6 +197,30 @@ async def _upload_metrics(self) -> None: remote_folder = f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}/{Local2Cloud.BLOB_METRICS}/{self.run.name}" await self._upload_local_folder_to_blob(local_folder, remote_folder) + async def _upload_flow_logs(self) -> str: + """Upload flow logs for each line run to cloud.""" + logger.debug(f"Uploading flow logs for run {self.run.name!r}.") + local_folder = self.run_output_path / LocalStorageFilenames.FLOW_LOGS_FOLDER + remote_folder = f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}/{Local2Cloud.BLOB_ARTIFACTS}/{self.run.name}" + await self._upload_local_folder_to_blob(local_folder, remote_folder) + return f"{remote_folder}/{LocalStorageFilenames.FLOW_LOGS_FOLDER}" + + async def _upload_instance_results(self) -> str: + """Upload instance results to cloud.""" + logger.debug(f"Uploading instance results for run {self.run.name!r}.") + flow_artifacts_folder = self.run_output_path / f"{OutputsFolderName.FLOW_ARTIFACTS}" + instance_results = get_instance_results(flow_artifacts_folder) + with tempfile.TemporaryDirectory() as temp_dir: + file_name = Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME + local_file = Path(temp_dir) / file_name + # write instance results to a temp local file + with open(local_file, "w", encoding=DEFAULT_ENCODING) as f: + for line_result in instance_results: + f.write(json.dumps(line_result) + "\n") + remote_file = f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}/{Local2Cloud.BLOB_ARTIFACTS}/{self.run.name}/{file_name}" + await self._upload_local_file_to_blob(local_file, remote_file) + return remote_file + async def _upload_local_folder_to_blob(self, local_folder, remote_folder): """Upload local folder to remote folder in blob. @@ -287,3 +314,49 @@ def _from_run_operations(cls, run: Run, run_ops: "RunOperations"): f"Cannot upload run {run.name!r} because the workspace default datastore is not supported. " f"Supported ones are ['AzureBlobDatastore'], got {type(datastore).__name__!r}." ) + + async def _check_run_details_exist_in_cloud(self, blob_path: List = None): + """Check if run details exist in cloud, mainly for test use.""" + flow_artifacts_prefix = f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}/{Local2Cloud.BLOB_ARTIFACTS}/{self.run.name}" + default_targets = [ + f"{flow_artifacts_prefix}/{OutputsFolderName.FLOW_ARTIFACTS}", + f"{flow_artifacts_prefix}/{OutputsFolderName.NODE_ARTIFACTS}", + f"{flow_artifacts_prefix}/{OutputsFolderName.FLOW_OUTPUTS}", + f"{Local2Cloud.BLOB_EXPERIMENT_RUN}/dcid.{self.run.name}/{Local2Cloud.EXECUTION_LOG}", + f"{Local2Cloud.BLOB_ROOT_RUNS}/{self.run.name}", + f"{flow_artifacts_prefix}/{LocalStorageFilenames.FLOW_LOGS_FOLDER}", + f"{flow_artifacts_prefix}/{Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME}", + ] + targets = blob_path or default_targets + target_files = [item for item in targets if "." in Path(item).name] + target_folders = [item for item in targets if item not in target_files] + results = {target: False for target in targets} + + default_service_client = self.blob_service_client[CloudDatastore.DEFAULT] + artifact_service_client = self.blob_service_client[CloudDatastore.ARTIFACT] + async with default_service_client, artifact_service_client: + default_container_client = default_service_client.get_container_client( + self.datastore[CloudDatastore.DEFAULT].container_name + ) + artifact_container_client = artifact_service_client.get_container_client( + self.datastore[CloudDatastore.ARTIFACT].container_name + ) + async with default_container_client, artifact_container_client: + # check blob existence + for target in target_files: + container_client = ( + default_container_client + if not target.endswith(Local2Cloud.EXECUTION_LOG) + else artifact_container_client + ) + blob_client = container_client.get_blob_client(target) + if await blob_client.exists(): + results[target] = True + + # check folder existence + for target in target_folders: + # all folder targets should be in default container + async for _ in default_container_client.list_blobs(name_starts_with=target): + results[target] = True + break + return results diff --git a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py index 145a7505e70..cfd648a6910 100644 --- a/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py +++ b/src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_upload.py @@ -13,7 +13,9 @@ from promptflow._sdk._errors import RunNotFoundError from promptflow._sdk._pf_client import PFClient as LocalPFClient from promptflow._sdk.entities import Run +from promptflow._utils.async_utils import async_run_allowing_running_loop from promptflow.azure import PFClient +from promptflow.azure.operations._async_run_uploader import AsyncRunUploader from .._azure_utils import DEFAULT_TEST_TIMEOUT, PYTEST_TIMEOUT_METHOD @@ -39,7 +41,7 @@ def get_local_pf(run_name: str) -> LocalPFClient: return local_pf @staticmethod - def check_local_to_cloud_run(pf: PFClient, run: Run): + def check_local_to_cloud_run(pf: PFClient, run: Run, check_run_details_in_cloud: bool = False) -> Run: # check if local run is uploaded cloud_run = pf.runs.get(run.name) assert cloud_run.display_name == run.display_name @@ -54,6 +56,13 @@ def check_local_to_cloud_run(pf: PFClient, run: Run): if run.tags: assert cloud_run.tags == run.tags + # check run details are actually uploaded to cloud + if check_run_details_in_cloud: + run_uploader = AsyncRunUploader._from_run_operations(run=run, run_ops=pf.runs) + result_dict = async_run_allowing_running_loop(run_uploader._check_run_details_exist_in_cloud) + for key, value in result_dict.items(): + assert value is True, f"Run details {key!r} not found in cloud, run name is {run.name!r}" + return cloud_run @@ -86,11 +95,10 @@ def test_upload_run( tags={"sdk-cli-test": "true"}, description="test sdk local to cloud", ) - run = local_pf.runs.stream(run.name) assert run.status == RunStatus.COMPLETED # check the run is uploaded to cloud - Local2CloudTestHelper.check_local_to_cloud_run(pf, run) + Local2CloudTestHelper.check_local_to_cloud_run(pf, run, check_run_details_in_cloud=True) @pytest.mark.skipif(condition=not pytest.is_live, reason="Bug - 3089145 Replay failed for test 'test_upload_run'") @pytest.mark.usefixtures( diff --git a/src/promptflow-devkit/promptflow/_cli/_utils.py b/src/promptflow-devkit/promptflow/_cli/_utils.py index af639e7d04d..36f0c7a86aa 100644 --- a/src/promptflow-devkit/promptflow/_cli/_utils.py +++ b/src/promptflow-devkit/promptflow/_cli/_utils.py @@ -409,6 +409,51 @@ def _try_delete_existing_run_record(run_name: str): pass +def get_instance_results(path: Union[str, Path]) -> List[Dict]: + """Parse flow artifact jsonl files in a directory and return a list of dictionaries. + + This function takes a path to a directory as input. It reads all jsonl files in the directory, + parses the json data, and returns a list of dictionaries. Each dictionary contains the following keys: + 'line_number', 'status', and all keys in 'inputs' and 'output'. + + .. example:: + 000000000_000000000.jsonl + 000000001_000000001.jsonl + 000000002_000000002.jsonl + + Get a list of dict like this: + {"line_number": 0, "status": "Completed", "inputs.name": "hod", "inputs.line_number": 2, "result": "res"} + ... + ... + + Note that inputs keys are prefixed with 'inputs.', but outputs keys are not. + Don't ask me why, because runtime did it this way :p + + Args: + path (Union[str, Path]): The path to the directory containing the jsonl files. + + Returns: + List[Dict]: A list of dictionaries containing the parsed data. + """ + path = Path(path) + result = [] + for file in path.glob("*.jsonl"): + with open(file, "r", encoding=DEFAULT_ENCODING) as f: + for line in f: + data = json.loads(line) + run_info = data.get("run_info", {}) + inputs = run_info.get("inputs", {}) + output = run_info.get("output", {}) + record = { + "line_number": data.get("line_number"), + "status": run_info.get("status"), + } + record.update({f"inputs.{k}": v for k, v in inputs.items()}) + record.update(output) + result.append(record) + return result + + def merge_jsonl_files(source_folder: Union[str, Path], output_folder: Union[str, Path], group_size: int = 25) -> None: """ Merge .jsonl files from a source folder into groups and write the merged files to an output folder. diff --git a/src/promptflow-devkit/promptflow/_sdk/_constants.py b/src/promptflow-devkit/promptflow/_sdk/_constants.py index 2c47c1d0f56..216b6427b4f 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_constants.py +++ b/src/promptflow-devkit/promptflow/_sdk/_constants.py @@ -474,6 +474,8 @@ class Local2Cloud: ASSET_NAME_DEBUG_INFO = "debug_info" ASSET_NAME_FLOW_OUTPUTS = "flow_outputs" EXECUTION_LOG = "logs/azureml/executionlogs.txt" + # instance_results.jsonl contains the inputs and outputs of all lines + FLOW_INSTANCE_RESULTS_FILE_NAME = "instance_results.jsonl" class Local2CloudProperties: From dfe16a49778628ed59630caad33a8e39ba4b722f Mon Sep 17 00:00:00 2001 From: Han Wang Date: Mon, 22 Apr 2024 17:30:39 +0800 Subject: [PATCH 09/14] [Executor] Fix exception in `__aggregate__` (#2925) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. Fix this: ![image](https://github.com/microsoft/promptflow/assets/7776147/49c579da-43b8-4991-b51f-2e81f2ff8cf5) This pull request primarily focuses on improving error handling and logging in the `src/promptflow-core/promptflow/executor/_script_executor.py` file, and adding tests to verify these improvements. The changes also include the addition of a new test case and the enhancement of existing test cases in the `src/promptflow-core/tests/core/e2etests/test_eager_flow.py` and `src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py` files. Here are the key changes: Error handling and logging improvements: * [`src/promptflow-core/promptflow/executor/_script_executor.py`](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4R19): Imported `ExceptionPresenter` to present exceptions in a more readable format. * [`src/promptflow-core/promptflow/executor/_script_executor.py`](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4L193-R194): Modified the `_exec_aggregation` function to initialize `output` and `metrics` at the same time, and to log a warning message when an exception occurs during the execution of the aggregation function. [[1]](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4L193-R194) [[2]](diffhunk://#diff-ee8f37478601ef45fdb2e773f04b280f5912f43868da4d5f1c4cf888a9980ab4L202-R212) Test case additions and enhancements: * [`src/promptflow-core/tests/core/e2etests/test_eager_flow.py`](diffhunk://#diff-162af09d07468e1340c2e7487f5964c9c7a3a54ac6430f11429a788d09e597c4R189-R199): Added a new test case `test_aggregation_error` to verify that the execution of the aggregation function does not fail when an error occurs. * [`src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py`](diffhunk://#diff-6297ac8f85a3d93b1ea97b35fea7723a46fed46163130590f8f124e621fb113dR1681-R1696): Enhanced the `assert_func` function to include a new function `assert_metrics` that verifies the correctness of the metrics. Also, added calls to `assert_run_metrics` in two places to verify the metrics of the run. [[1]](diffhunk://#diff-6297ac8f85a3d93b1ea97b35fea7723a46fed46163130590f8f124e621fb113dR1681-R1696) [[2]](diffhunk://#diff-6297ac8f85a3d93b1ea97b35fea7723a46fed46163130590f8f124e621fb113dR1777-R1783) New test files: * [`src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/aggregation_exception.py`](diffhunk://#diff-5dd9d0875d57a786ed8f6335a5c89875f0308a1f7a7138d3f70dcee206908b54R1-R34): Added a new Python file for testing purposes that includes a class `MyFlow` with an `__aggregate__` method that raises a `NotImplementedError`. * [`src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/flow.flex.yaml`](diffhunk://#diff-0ac09234963626548388379698a74796df5c23add62c041be9e771c931031edcR1-R3): Added a new YAML file that specifies the entry point for the flow. * [`src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/inputs.jsonl`](diffhunk://#diff-40b1d0810a58662fecbb8f37a1963d69759ad90d8aeed10710fba3b503a40114R1-R4): Added a new JSONL file that provides inputs for the flow. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../promptflow/executor/_script_executor.py | 15 ++++++-- .../tests/core/e2etests/test_eager_flow.py | 11 ++++++ .../sdk_cli_test/e2etests/test_flow_run.py | 12 +++++++ .../aggregation_exception.py | 34 +++++++++++++++++++ .../flow.flex.yaml | 3 ++ .../inputs.jsonl | 4 +++ 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/aggregation_exception.py create mode 100644 src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/flow.flex.yaml create mode 100644 src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/inputs.jsonl diff --git a/src/promptflow-core/promptflow/executor/_script_executor.py b/src/promptflow-core/promptflow/executor/_script_executor.py index f03d79368c2..d502b5879e6 100644 --- a/src/promptflow-core/promptflow/executor/_script_executor.py +++ b/src/promptflow-core/promptflow/executor/_script_executor.py @@ -16,6 +16,7 @@ from promptflow._core.tool_meta_generator import PythonLoadError from promptflow._utils.async_utils import async_run_allowing_running_loop from promptflow._utils.dataclass_serializer import convert_eager_flow_output_to_dict +from promptflow._utils.exception_utils import ExceptionPresenter from promptflow._utils.logger_utils import logger from promptflow._utils.multimedia_utils import BasicMultimediaProcessor from promptflow._utils.tool_utils import function_to_interface @@ -190,7 +191,7 @@ def _exec_aggregation( self, inputs: List[Any], ) -> AggregationResult: - output = None + output, metrics = None, {} try: if inspect.iscoroutinefunction(self._aggr_func): output = async_run_allowing_running_loop(self._aggr_func, **{self._aggr_input_name: inputs}) @@ -199,8 +200,16 @@ def _exec_aggregation( metrics = output if isinstance(output, dict) else {"metrics": output} for k, v in metrics.items(): log_metric(k, v) - except Exception: - pass + except Exception as e: + error_type_and_message = f"({e.__class__.__name__}) {e}" + e = ScriptExecutionError( + message_format="Execution failure in '{func_name}': {error_type_and_message}", + func_name=self._aggr_func.__name__, + error_type_and_message=error_type_and_message, + ) + error = ExceptionPresenter.create(e).to_dict(include_debug_info=True) + logger.warning(f"Failed to execute aggregation function with error: {error}") + logger.warning("The flow will have empty metrics.") return AggregationResult(output, metrics, {}) async def exec_line_async( diff --git a/src/promptflow-core/tests/core/e2etests/test_eager_flow.py b/src/promptflow-core/tests/core/e2etests/test_eager_flow.py index c8e3fe98d5b..daff96da76b 100644 --- a/src/promptflow-core/tests/core/e2etests/test_eager_flow.py +++ b/src/promptflow-core/tests/core/e2etests/test_eager_flow.py @@ -186,3 +186,14 @@ def test_execute_func_with_user_error(self, flow_folder, expected_exception, exp with pytest.raises(expected_exception) as e: ScriptExecutor(flow_file=flow_file) assert expected_error_msg in str(e.value) + + def test_aggregation_error(self): + flow_folder = "class_based_flow_with_aggregation_exception" + flow_file = get_yaml_file(flow_folder, root=EAGER_FLOW_ROOT) + executor = ScriptExecutor(flow_file=flow_file, init_kwargs={"obj_input": "obj_input"}) + line_result = executor.exec_line(inputs={"func_input": "func_input"}, index=0) + + if executor.has_aggregation_node: + aggr_result = executor._exec_aggregation(inputs=[line_result.output]) + # exec aggregation won't fail with error + assert aggr_result.metrics == {} diff --git a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py index 9a4ca44580c..81f89fa85a6 100644 --- a/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py +++ b/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_flow_run.py @@ -1678,17 +1678,22 @@ def assert_func(details_dict): "func_input", ] and details_dict["outputs.obj_input"] == ["val", "val", "val", "val"] + def assert_metrics(metrics_dict): + return metrics_dict == {"length": 4} + flow_path = Path(f"{EAGER_FLOWS_DIR}/basic_callable_class") run = pf.run( flow=flow_path, data=f"{EAGER_FLOWS_DIR}/basic_callable_class/inputs.jsonl", init={"obj_input": "val"} ) assert_batch_run_result(run, pf, assert_func) + assert_run_metrics(run, pf, assert_metrics) run = load_run( source=f"{EAGER_FLOWS_DIR}/basic_callable_class/run.yaml", ) run = pf.runs.create_or_update(run=run) assert_batch_run_result(run, pf, assert_func) + assert_run_metrics(run, pf, assert_metrics) def test_run_with_init_class(self, pf): def assert_func(details_dict): @@ -1769,3 +1774,10 @@ def assert_batch_run_result(run: Run, pf: PFClient, assert_func): # convert DataFrame to dict details_dict = details.to_dict(orient="list") assert assert_func(details_dict), details_dict + + +def assert_run_metrics(run: Run, pf: PFClient, assert_func): + assert run.status == "Completed" + assert "error" not in run._to_dict(), run._to_dict()["error"] + metrics = pf.runs.get_metrics(run.name) + assert assert_func(metrics), metrics diff --git a/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/aggregation_exception.py b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/aggregation_exception.py new file mode 100644 index 00000000000..fe0f4a04538 --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/aggregation_exception.py @@ -0,0 +1,34 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +from typing import TypedDict + +from promptflow.tracing import trace + +class FlowOutput(TypedDict): + obj_input: str + func_input: str + obj_id: str + + +class MyFlow: + def __init__(self, obj_input: str): + self.obj_input = obj_input + + @trace + def __call__(self, func_input: str) -> FlowOutput: + return { + "obj_input": self.obj_input, + "func_input": func_input, + "obj_id": id(self), + } + + def __aggregate__(self, results: list) -> dict: + raise NotImplementedError + + +if __name__ == "__main__": + flow = MyFlow("obj_input") + result = flow("func_input") + print(result) + diff --git a/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/flow.flex.yaml b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/flow.flex.yaml new file mode 100644 index 00000000000..fcd8d404e9d --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/flow.flex.yaml @@ -0,0 +1,3 @@ +entry: aggregation_exception:MyFlow +environment: + image: promptflow.azurecr.io/promptflow-runtime:bu-20240403-1452 diff --git a/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/inputs.jsonl b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/inputs.jsonl new file mode 100644 index 00000000000..cf192f44c3e --- /dev/null +++ b/src/promptflow/tests/test_configs/eager_flows/class_based_flow_with_aggregation_exception/inputs.jsonl @@ -0,0 +1,4 @@ +{"func_input": "func_input"} +{"func_input": "func_input"} +{"func_input": "func_input"} +{"func_input": "func_input"} \ No newline at end of file From a1af5b35857745aee78f6726b08de3212ca47e8b Mon Sep 17 00:00:00 2001 From: Ge Gao <49388944+dorisjoy@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:48:29 +0800 Subject: [PATCH 10/14] Support ServerlessConnection in Embedding (#2537) Support ServerlessConnection in Embedding --------- Co-authored-by: Ge Gao Co-authored-by: Meng Lan Co-authored-by: melionel --- src/promptflow-tools/connections.json.example | 8 +++++++ .../promptflow/tools/embedding.py | 22 +++++++++++++++++-- .../promptflow/tools/yamls/embedding.yaml | 2 +- src/promptflow-tools/tests/conftest.py | 5 +++++ src/promptflow-tools/tests/test_embedding.py | 7 ++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/promptflow-tools/connections.json.example b/src/promptflow-tools/connections.json.example index 98a36a1caa1..878484bb3b1 100644 --- a/src/promptflow-tools/connections.json.example +++ b/src/promptflow-tools/connections.json.example @@ -85,5 +85,13 @@ "api_base": "serverless-endpoint-url" }, "module": "promptflow.connections" + }, + "serverless_connection_embedding": { + "type": "ServerlessConnection", + "value": { + "api_key": "serverless-embedding-api-key", + "api_base": "serverless-embedding-endpoint-url" + }, + "module": "promptflow.connections" } } diff --git a/src/promptflow-tools/promptflow/tools/embedding.py b/src/promptflow-tools/promptflow/tools/embedding.py index 1079b82a533..45cbeab4765 100644 --- a/src/promptflow-tools/promptflow/tools/embedding.py +++ b/src/promptflow-tools/promptflow/tools/embedding.py @@ -8,6 +8,13 @@ # since the code here is in promptflow namespace as well from promptflow._internal import tool from promptflow.connections import AzureOpenAIConnection, OpenAIConnection +try: + from promptflow.connections import ServerlessConnection +except ImportError: + # If unable to import ServerlessConnection, define a placeholder class to allow isinstance checks to pass. + # ServerlessConnection was introduced in pf version 1.6.0. + class ServerlessConnection: + pass class EmbeddingModel(str, Enum): @@ -18,8 +25,12 @@ class EmbeddingModel(str, Enum): @tool @handle_openai_error() -def embedding(connection: Union[AzureOpenAIConnection, OpenAIConnection], input: str, deployment_name: str = "", - model: EmbeddingModel = EmbeddingModel.TEXT_EMBEDDING_ADA_002): +def embedding( + connection: Union[AzureOpenAIConnection, OpenAIConnection, ServerlessConnection], + input: str, + deployment_name: str = "", + model: EmbeddingModel = EmbeddingModel.TEXT_EMBEDDING_ADA_002 +): if isinstance(connection, AzureOpenAIConnection): client = init_azure_openai_client(connection) return client.embeddings.create( @@ -33,6 +44,13 @@ def embedding(connection: Union[AzureOpenAIConnection, OpenAIConnection], input: input=input, model=model ).data[0].embedding + elif isinstance(connection, ServerlessConnection): + client = init_openai_client(connection) + return client.embeddings.create( + input=[input], + model=model, + encoding_format="float" + ).data[0].embedding else: error_message = f"Not Support connection type '{type(connection).__name__}' for embedding api. " \ f"Connection type should be in [AzureOpenAIConnection, OpenAIConnection]." diff --git a/src/promptflow-tools/promptflow/tools/yamls/embedding.yaml b/src/promptflow-tools/promptflow/tools/yamls/embedding.yaml index e5c68f361ea..ab67db5d18e 100644 --- a/src/promptflow-tools/promptflow/tools/yamls/embedding.yaml +++ b/src/promptflow-tools/promptflow/tools/yamls/embedding.yaml @@ -6,7 +6,7 @@ promptflow.tools.embedding.embedding: function: embedding inputs: connection: - type: [AzureOpenAIConnection, OpenAIConnection] + type: [AzureOpenAIConnection, OpenAIConnection, ServerlessConnection] deployment_name: type: - string diff --git a/src/promptflow-tools/tests/conftest.py b/src/promptflow-tools/tests/conftest.py index 0c686f619d7..daba229e8af 100644 --- a/src/promptflow-tools/tests/conftest.py +++ b/src/promptflow-tools/tests/conftest.py @@ -65,6 +65,11 @@ def serverless_connection(): return ConnectionManager().get("serverless_connection") +@pytest.fixture +def serverless_connection_embedding(): + return ConnectionManager().get("serverless_connection_embedding") + + def verify_om_llm_custom_connection(connection: CustomConnection) -> bool: '''Verify that there is a MIR endpoint up and available for the Custom Connection. We explicitly do not pass the endpoint key to avoid the delay in generating a response. diff --git a/src/promptflow-tools/tests/test_embedding.py b/src/promptflow-tools/tests/test_embedding.py index 51dfe680950..32fc46856fc 100644 --- a/src/promptflow-tools/tests/test_embedding.py +++ b/src/promptflow-tools/tests/test_embedding.py @@ -21,6 +21,13 @@ def test_embedding_conn_oai(self, open_ai_connection): model="text-embedding-ada-002") assert len(result) == 1536 + @pytest.mark.skip_if_no_api_key("serverless_connection_embedding") + def test_embedding_conn_serverless(self, serverless_connection_embedding): + result = embedding( + connection=serverless_connection_embedding, + input="The food was delicious and the waiter",) + assert len(result) == 1024 + def test_embedding_invalid_connection_type(self, serp_connection): error_codes = "UserError/ToolValidationError/InvalidConnectionType" with pytest.raises(InvalidConnectionType) as exc_info: From 6f3509a32e05b1fecd59f76cf35097690ffa6383 Mon Sep 17 00:00:00 2001 From: Brynn Yin <24237253+brynn-code@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:48:44 +0800 Subject: [PATCH 11/14] [Doc] Build some notebook to doc (#2909) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. ![image](https://github.com/microsoft/promptflow/assets/47586720/23113dcd-6ced-4ee4-b4f6-6f4f1f07d152) # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --------- Signed-off-by: Brynn Yin Co-authored-by: Clement Wang Co-authored-by: Clement Wang <47586720+wangchao1230@users.noreply.github.com> --- .github/workflows/build_doc_ci.yml | 1 + .github/workflows/publish_doc.yml | 1 + .gitignore | 5 +- .../basic/flex-flow-quickstart-azure.ipynb | 14 +++-- .../basic/flex-flow-quickstart.ipynb | 12 ++-- .../chat-with-pdf/chat-with-pdf-azure.ipynb | 8 ++- .../chat/chat-with-pdf/chat-with-pdf.ipynb | 15 +++-- .../pf-visualize-screenshot.png | Bin .../prompty/basic/prompty-quickstart.ipynb | 8 ++- .../chat-basic/chat-with-prompty.ipynb | 6 +- .../get-started/flow-as-function.ipynb | 4 ++ .../get-started/quickstart-azure.ipynb | 16 +++-- .../tutorials/get-started/quickstart.ipynb | 24 +++++--- .../flow-in-pipeline/pf-base-component.png | Bin .../pf-component-parameters.png | Bin .../run-flow-with-pipeline/pipeline.ipynb | 21 ++++--- .../run-management/cloud-run-management.ipynb | 14 +++-- .../run-management/run-management.ipynb | 6 +- .../trace-autogen-groupchat.ipynb | 12 ++-- .../otlp-trace-collector.ipynb | 4 ++ .../tracing/langchain/trace-langchain.ipynb | 10 ++- scripts/docs/conf.py | 45 +++++++++----- scripts/docs/doc_generation.ps1 | 57 +++++++++++++++++- 23 files changed, 207 insertions(+), 76 deletions(-) rename examples/flows/chat/chat-with-pdf/{assets => media/chat-with-pdf}/pf-visualize-screenshot.png (100%) rename {docs => examples/tutorials/run-flow-with-pipeline}/media/cloud/flow-in-pipeline/pf-base-component.png (100%) rename {docs => examples/tutorials/run-flow-with-pipeline}/media/cloud/flow-in-pipeline/pf-component-parameters.png (100%) diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml index 48cb7045dcb..b0d034dcf66 100644 --- a/.github/workflows/build_doc_ci.yml +++ b/.github/workflows/build_doc_ci.yml @@ -8,6 +8,7 @@ on: paths: - 'README.md' - 'docs/**' + - 'examples/**.ipynb' - 'scripts/docs/**' - '.github/workflows/build_doc_ci.yml' - 'src/promptflow-tracing/promptflow/**' diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index 43bc9cdd0c5..c1604ee0ed6 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -9,6 +9,7 @@ on: paths: - 'README.md' - 'docs/**' + - 'examples/**.ipynb' - 'scripts/docs/**' - '.github/workflows/publish_doc.yml' - 'src/promptflow-tracing/promptflow/**' diff --git a/.gitignore b/.gitignore index 36c50ebf596..5c1a9b0ebcf 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,8 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +scripts/docs/_build/ +scripts/docs/jupyter_execute/ # PyBuilder .pybuilder/ @@ -175,8 +176,6 @@ hello-world-proj/** # secrets **/connections.json -# doc build -scripts/docs/_build/ **/bash_script.sh .index diff --git a/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb b/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb index 644cdec9f7a..d829160737b 100644 --- a/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb +++ b/examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb @@ -45,7 +45,7 @@ "We are using `DefaultAzureCredential` to get access to workspace. \n", "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + "Reference for more available credentials if it does not work for you: [configure credential example](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." ] }, { @@ -71,7 +71,7 @@ "source": [ "### Get a handle to the workspace\n", "\n", - "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)" + "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)" ] }, { @@ -107,7 +107,7 @@ "source": [ "## 2. Batch run the function as flow with multi-line data\n", "\n", - "Create a [flow.flex.yaml](flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n" + "Create a `flow.flex.yaml` file to define a flow which entry pointing to the python function we defined.\n" ] }, { @@ -233,11 +233,15 @@ "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n", "\n", "You can check out more examples:\n", - "- [Basic Chat](../chat-basic/README.md): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." + "- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/flex-flows/chat-basic): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." ] } ], "metadata": { + "build_doc": { + "category": "azure", + "section": "Flow" + }, "description": "A quickstart tutorial to run a flex flow and evaluate it in azure.", "kernelspec": { "display_name": "prompt_flow", @@ -254,7 +258,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.9.18" }, "resources": "examples/requirements-azure.txt, examples/flex-flows/basic, examples/flex-flows/eval-code-quality" }, diff --git a/examples/flex-flows/basic/flex-flow-quickstart.ipynb b/examples/flex-flows/basic/flex-flow-quickstart.ipynb index 232622c48cc..d69923a38ea 100644 --- a/examples/flex-flows/basic/flex-flow-quickstart.ipynb +++ b/examples/flex-flows/basic/flex-flow-quickstart.ipynb @@ -52,7 +52,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to [.env.example](.env.example) as an template." + "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to `../.env.example` as an template." ] }, { @@ -112,7 +112,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now, let's add another layer of function call. In [programmer.py](programmer.py) there is a function called `write_simple_program`, which calls a new function called `load_prompt` and previous `my_llm_tool` function." + "Now, let's add another layer of function call. In `programmer.py` there is a function called `write_simple_program`, which calls a new function called `load_prompt` and previous `my_llm_tool` function." ] }, { @@ -168,7 +168,7 @@ "source": [ "## 2. Batch run the function as flow with multi-line data\n", "\n", - "Create a [flow.flex.yaml](flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n" + "Create a [flow.flex.yaml](https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/basic/flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n" ] }, { @@ -304,11 +304,15 @@ "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n", "\n", "You can check out more examples:\n", - "- [Basic Chat](../chat-basic/README.md): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." + "- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/flex-flows/chat-basic): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Flow" + }, "description": "A quickstart tutorial to run a flex flow and evaluate it.", "kernelspec": { "display_name": "prompt_flow", diff --git a/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb b/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb index 860f5716930..1ab5875f0a1 100644 --- a/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb +++ b/examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Chat with PDF using Azure AI\n", + "# Chat with PDF in Azure\n", "\n", "This is a simple flow that allow you to ask questions about the content of a PDF file and get answers.\n", "You can run the flow with a URL to a PDF file and question as argument.\n", @@ -161,7 +161,7 @@ "metadata": {}, "source": [ "# 3. Evaluate the \"groundedness\"\n", - "The [eval-groundedness flow](../../evaluation/eval-groundedness/) is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow." + "The `eval-groundedness flow` is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow." ] }, { @@ -289,6 +289,10 @@ } ], "metadata": { + "build_doc": { + "category": "azure", + "section": "Flow" + }, "description": "A tutorial of chat-with-pdf flow that executes in Azure AI", "kernelspec": { "display_name": "prompt-flow", diff --git a/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb b/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb index 07056241594..40f38d62703 100644 --- a/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb +++ b/examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb @@ -93,7 +93,7 @@ "source": [ "## 2. Test the flow\n", "\n", - "**Note**: this sample uses [predownloaded PDFs](./chat_with_pdf/.pdfs/) and [prebuilt FAISS Index](./chat_with_pdf/.index/) to speed up execution time.\n", + "**Note**: this sample uses `predownloaded PDFs` and `prebuilt FAISS Index` to speed up execution time.\n", "You can remove the folders to start a fresh run." ] }, @@ -103,6 +103,9 @@ "metadata": {}, "outputs": [], "source": [ + "# ./chat_with_pdf/.pdfs/ stores predownloaded PDFs\n", + "# ./chat_with_pdf/.index/ stores prebuilt index files\n", + "\n", "output = pf.flows.test(\n", " \".\",\n", " inputs={\n", @@ -167,7 +170,7 @@ "metadata": {}, "source": [ "# 4. Evaluate the \"groundedness\"\n", - "The [eval-groundedness flow](../../evaluation/eval-groundedness/) is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow." + "The `eval-groundedness flow` is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow." ] }, { @@ -224,7 +227,7 @@ "metadata": {}, "source": [ "You will see a web page like this. It gives you detail about how each row is graded and even the details how the evaluation run executes:\n", - "![pf-visualize-screenshot](./assets/pf-visualize-screenshot.png)" + "![pf-visualize-screenshot](./media/chat-with-pdf/pf-visualize-screenshot.png)" ] }, { @@ -299,6 +302,10 @@ } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Flow" + }, "description": "A tutorial of chat-with-pdf flow that allows user ask questions about the content of a PDF file and get answers", "kernelspec": { "display_name": "prompt-flow", @@ -315,7 +322,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.17" } }, "nbformat": 4, diff --git a/examples/flows/chat/chat-with-pdf/assets/pf-visualize-screenshot.png b/examples/flows/chat/chat-with-pdf/media/chat-with-pdf/pf-visualize-screenshot.png similarity index 100% rename from examples/flows/chat/chat-with-pdf/assets/pf-visualize-screenshot.png rename to examples/flows/chat/chat-with-pdf/media/chat-with-pdf/pf-visualize-screenshot.png diff --git a/examples/prompty/basic/prompty-quickstart.ipynb b/examples/prompty/basic/prompty-quickstart.ipynb index df3e163369c..f329cea8d29 100644 --- a/examples/prompty/basic/prompty-quickstart.ipynb +++ b/examples/prompty/basic/prompty-quickstart.ipynb @@ -54,7 +54,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to [.env.example](../.env.example) as an template.\n", + "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to `../.env.example` as an template.\n", "\n", "Note: you need the new [gpt-35-turbo (0125) version](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35-models) to use the json_object response_format feature." ] @@ -269,11 +269,15 @@ "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n", "\n", "You can check out more examples:\n", - "- [Basic Chat](../chat-basic/README.md): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." + "- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/prompty/chat-basic): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Prompty" + }, "description": "A quickstart tutorial to run a prompty and evaluate it.", "kernelspec": { "display_name": "prompt_flow", diff --git a/examples/prompty/chat-basic/chat-with-prompty.ipynb b/examples/prompty/chat-basic/chat-with-prompty.ipynb index 40fd9e4e498..dfbd40bd195 100644 --- a/examples/prompty/chat-basic/chat-with-prompty.ipynb +++ b/examples/prompty/chat-basic/chat-with-prompty.ipynb @@ -285,11 +285,15 @@ "\n", "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n", "\n", - "You can check out more [Prompty Examples](../README.md)." + "You can check out more [Prompty Examples](https://github.com/microsoft/promptflow/tree/main/examples/prompty)." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Prompty" + }, "description": "A quickstart tutorial to run a chat prompty and evaluate it.", "kernelspec": { "display_name": "prompt_flow", diff --git a/examples/tutorials/get-started/flow-as-function.ipynb b/examples/tutorials/get-started/flow-as-function.ipynb index 8160ad6ade1..0b9c527805a 100644 --- a/examples/tutorials/get-started/flow-as-function.ipynb +++ b/examples/tutorials/get-started/flow-as-function.ipynb @@ -185,6 +185,10 @@ } ], "metadata": { + "build_doc": { + "section": "Flow", + "category": "local" + }, "description": "This guide will walk you through the main scenarios of executing flow as a function.", "kernelspec": { "display_name": "github_v2", diff --git a/examples/tutorials/get-started/quickstart-azure.ipynb b/examples/tutorials/get-started/quickstart-azure.ipynb index 376a79adbcb..c7139e3b69c 100644 --- a/examples/tutorials/get-started/quickstart-azure.ipynb +++ b/examples/tutorials/get-started/quickstart-azure.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Run prompt flow in Azure AI\n", + "# Run DAG flow in Azure\n", "\n", "**Requirements** - In order to benefit from this tutorial, you will need:\n", "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace - [Configure workspace](../../configuration.ipynb)\n", + "- An Azure ML workspace - [Configure workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)\n", "- A python environment\n", "- Installed prompt flow SDK\n", "\n", @@ -71,7 +71,7 @@ "We are using `DefaultAzureCredential` to get access to workspace. \n", "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + "Reference for more available credentials if it does not work for you: [configure credential example](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." ] }, { @@ -95,7 +95,7 @@ "source": [ "## 1.3 Get a handle to the workspace\n", "\n", - "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)" + "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)" ] }, { @@ -290,7 +290,7 @@ " - text: ${fetch_text_content_from_url.output}\n", "\n", "\n", - "You can check the whole flow definition at [flow.dag.yaml](../../flows/standard/web-classification/flow.dag.yaml)" + "You can check the whole flow definition at [flow.dag.yaml](https://github.com/microsoft/promptflow/blob/main/examples/flows/standard/web-classification/flow.dag.yaml)" ] }, { @@ -399,11 +399,15 @@ "# Next Steps\n", "\n", "Learn more on how to:\n", - "- run the flow as a component in a azureml pipeline: [flow in pipeline](../flow-in-pipeline/pipeline.ipynb)." + "- run the flow as a component in a azureml pipeline: [flow in pipeline](https://github.com/microsoft/promptflow/blob/main/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb)." ] } ], "metadata": { + "build_doc": { + "category": "azure", + "section": "Flow" + }, "description": "A quickstart tutorial to run a flow in Azure AI and evaluate it.", "kernelspec": { "display_name": "promptflow", diff --git a/examples/tutorials/get-started/quickstart.ipynb b/examples/tutorials/get-started/quickstart.ipynb index 11593e5c2fe..cfee4a5f6f7 100644 --- a/examples/tutorials/get-started/quickstart.ipynb +++ b/examples/tutorials/get-started/quickstart.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Getting started with prompt flow\n", + "# Getting started with DAG flow\n", "\n", "**Prerequisite** - To make the most of this tutorial, you'll need:\n", "- A local clone of the prompt flow repository\n", @@ -20,7 +20,7 @@ "- Run your first evaluation\n", "\n", "\n", - "The sample used in this tutorial is the [web-classification](../../flows/standard/web-classification/README.md) flow, which categorizes URLs into several predefined classes. Classification is a traditional machine learning task, and this sample illustrates how to perform classification using GPT and prompts." + "The sample used in this tutorial is the [web-classification](https://github.com/microsoft/promptflow/tree/main/examples/flows/standard/web-classification) flow, which categorizes URLs into several predefined classes. Classification is a traditional machine learning task, and this sample illustrates how to perform classification using GPT and prompts." ] }, { @@ -283,7 +283,7 @@ "source": [ "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n", "\n", - "You can check out the [web-classification](../../flows/standard/web-classification/) flow and the [classification-accuracy](../../flows/evaluation/eval-classification-accuracy/) flow for more details, and start building your own flow.\n", + "You can check out the [web-classification](https://github.com/microsoft/promptflow/tree/main/examples/flows/standard/web-classification) flow and the [classification-accuracy](https://github.com/microsoft/promptflow/tree/main/examples/flows/evaluation/eval-classification-accuracy) flow for more details, and start building your own flow.\n", "\n", "Or you can move on for a more advanced topic: experiment with a variant." ] @@ -294,7 +294,7 @@ "source": [ "### Another batch run with a variant\n", "\n", - "[Variant](../../../docs/concepts/concept-variants.md) in prompt flow is to allow you do experimentation with LLMs. You can set a variant of Prompt/LLM node pointing to different prompt or use different LLM parameters like temperature.\n", + "[Variant](https://microsoft.github.io/promptflow/concepts/concept-variants.html) in prompt flow is to allow you do experimentation with LLMs. You can set a variant of Prompt/LLM node pointing to different prompt or use different LLM parameters like temperature.\n", "\n", "In this example, `web-classification`'s node `summarize_text_content` has two variants: `variant_0` and `variant_1`. The difference between them is the inputs parameters:\n", "\n", @@ -315,7 +315,7 @@ " - text: ${fetch_text_content_from_url.output}\n", "\n", "\n", - "You can check the whole flow definition at [flow.dag.yaml](../../flows/standard/web-classification/flow.dag.yaml)" + "You can check the whole flow definition at [flow.dag.yaml](https://github.com/microsoft/promptflow/blob/main/examples/flows/standard/web-classification/flow.dag.yaml)" ] }, { @@ -407,14 +407,18 @@ "# Next Steps\n", "\n", "Learn more on:\n", - "- [Manage connections](../../connections/connection.ipynb): how to manage the endpoints/secrets information to access external services including LLMs.\n", - "- [Chat with PDF](../e2e-development/chat-with-pdf.md): go through an end-to-end tutorial on how to develop a chat application with prompt flow.\n", - "- [Deploy http endpoint](../flow-deploy/deploy.md): how to deploy the flow as a local http endpoint.\n", - "- [Prompt flow in Azure AI](./quickstart-azure.ipynb): run and evaluate flow in Azure AI where you can collaborate with team better." + "- [Manage connections](https://github.com/microsoft/promptflow/blob/main/examples/connections/connection.ipynb): how to manage the endpoints/secrets information to access external services including LLMs.\n", + "- [Chat with PDF](https://github.com/microsoft/promptflow/blob/main/examples/tutorials/e2e-development/chat-with-pdf.md): go through an end-to-end tutorial on how to develop a chat application with prompt flow.\n", + "- [Deploy http endpoint](https://github.com/microsoft/promptflow/tree/main/examples/tutorials/flow-deploy): how to deploy the flow as a local http endpoint.\n", + "- [Prompt flow in Azure AI](https://github.com/microsoft/promptflow/blob/main/examples/tutorials/get-started/quickstart-azure.ipynb): run and evaluate flow in Azure AI where you can collaborate with team better." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Flow" + }, "description": "A quickstart tutorial to run a flow and evaluate it.", "kernelspec": { "display_name": "prompt_flow", @@ -431,7 +435,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.17" }, "resources": "examples/requirements.txt, examples/flows/standard/web-classification, examples/flows/evaluation/eval-classification-accuracy" }, diff --git a/docs/media/cloud/flow-in-pipeline/pf-base-component.png b/examples/tutorials/run-flow-with-pipeline/media/cloud/flow-in-pipeline/pf-base-component.png similarity index 100% rename from docs/media/cloud/flow-in-pipeline/pf-base-component.png rename to examples/tutorials/run-flow-with-pipeline/media/cloud/flow-in-pipeline/pf-base-component.png diff --git a/docs/media/cloud/flow-in-pipeline/pf-component-parameters.png b/examples/tutorials/run-flow-with-pipeline/media/cloud/flow-in-pipeline/pf-component-parameters.png similarity index 100% rename from docs/media/cloud/flow-in-pipeline/pf-component-parameters.png rename to examples/tutorials/run-flow-with-pipeline/media/cloud/flow-in-pipeline/pf-component-parameters.png diff --git a/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb b/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb index 831364aa4f0..883368b4f60 100644 --- a/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb +++ b/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb @@ -4,7 +4,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Why use Azure machine learning(ML) pipelines to run your flows on the cloud?\n", + "# Run flows in Azure ML pipeline\n", + "## Why use Azure machine learning(ML) pipelines to run your flows on the cloud?\n", "In real-world scenarios, flows serve various purposes. For example, consider a flow designed to evaluate the relevance score for a communication session between humans and agents. Suppose you want to trigger this flow every night to assess today’s performance and avoid peak hours for LLM (Language Model) endpoints. In this common scenario, people often encounter the following needs:\n", "- Handling Large Data Inputs: Running flows with thousands or millions of data inputs at once.\n", "- Scalability and Efficiency: Requiring a scalable, efficient, and resilient platform to ensure success.\n", @@ -24,10 +25,10 @@ "- Azure cloud setup:\n", " - An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", " - Create an Azure ML resource from Azure portal - [Create a Azure ML workspace](https://ms.portal.azure.com/#view/Microsoft_Azure_Marketplace/MarketplaceOffersBlade/searchQuery/machine%20learning)\n", - " - Connect to your workspace then setup a basic computer cluster - [Configure workspace](../../configuration.ipynb)\n", + " - Connect to your workspace then setup a basic computer cluster - [Configure workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)\n", "- Local environment setup:\n", " - A python environment\n", - " - Installed Azure Machine Learning Python SDK v2 - [install instructions](../../../README.md) - check the getting started section and make sure version of 'azure-ai-ml' is higher than `1.12.0`" + " - Installed Azure Machine Learning Python SDK v2 - [install instructions](https://github.com/microsoft/promptflow/blob/main/examples/README.md) - check the getting started section and make sure version of 'azure-ai-ml' is higher than `1.12.0`" ] }, { @@ -63,7 +64,7 @@ "We are using `DefaultAzureCredential` to get access to workspace. \n", "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + "Reference for more available credentials if it does not work for you: [configure credential example](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." ] }, { @@ -87,7 +88,7 @@ "source": [ "## 1.3 Get a handle to the workspace\n", "\n", - "We use 'config file' to connect to your workspace. Check [this notebook](../../configuration.ipynb) to get your config file from Azure ML workspace portal and paste it into this folder. Then if you pass the next code block, you've all set for the environment." + "We use 'config file' to connect to your workspace. Check [this notebook](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb) to get your config file from Azure ML workspace portal and paste it into this folder. Then if you pass the next code block, you've all set for the environment." ] }, { @@ -139,13 +140,13 @@ "| flow_outputs | uri_file | Generates a single output file named parallel_run_step.jsonl. Each line in this data file corresponds to a JSON object representing the flow returns, along with an additional column called line_number indicating its position from the original file. |\n", "| debug_info | uri_folder | If you run your flow component in __debug mode__, this port provides debugging information for each run of your lines. E.g. intermediate outputs between steps, or LLM response and token usage. |\n", "\n", - "![prompt flow base component image](../../../docs/media/cloud/flow-in-pipeline/pf-base-component.png)\n", + "![prompt flow base component image](./media/cloud/flow-in-pipeline/pf-base-component.png)\n", " \n", " - Auto-generated parameters \n", " \n", - " These parameters represent all your flow inputs and connections associated with your flow steps. You can set default values in the flow/run definition, and they can be further customized during job submission. Use '[web-classification](../../flows/standard/web-classification/flow.dag.yaml)' sample flow for example, this flow has only one input named 'url' and 2 LLM steps 'summarize_text_content' and 'classify_with_llm'. The input parameters of this flow component are:\n", + " These parameters represent all your flow inputs and connections associated with your flow steps. You can set default values in the flow/run definition, and they can be further customized during job submission. Use '[web-classification](https://github.com/microsoft/promptflow/blob/main/examples/flows/standard/web-classification/flow.dag.yaml)' sample flow for example, this flow has only one input named 'url' and 2 LLM steps 'summarize_text_content' and 'classify_with_llm'. The input parameters of this flow component are:\n", " \n", - " ![prompt flow base component image](../../../docs/media/cloud/flow-in-pipeline/pf-component-parameters.png)\n", + " ![prompt flow base component image](./media/cloud/flow-in-pipeline/pf-component-parameters.png)\n", "\n", " - Auto-generated environment\n", "\n", @@ -606,6 +607,10 @@ } ], "metadata": { + "build_doc": { + "category": "azure", + "section": "Flow" + }, "description": "Create pipeline using components to run a distributed job with tensorflow", "kernelspec": { "display_name": "Python 3", diff --git a/examples/tutorials/run-management/cloud-run-management.ipynb b/examples/tutorials/run-management/cloud-run-management.ipynb index ec1dbad9cf8..66d61f82c99 100644 --- a/examples/tutorials/run-management/cloud-run-management.ipynb +++ b/examples/tutorials/run-management/cloud-run-management.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Flow Run Management in Azure\n", + "# Flow run management in Azure\n", "\n", "**Requirements** - In order to benefit from this tutorial, you will need:\n", "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace - [Configure workspace](../../configuration.ipynb)\n", + "- An Azure ML workspace - [Configure workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)\n", "- A python environment\n", "- Installed prompt flow SDK\n", "\n", @@ -74,7 +74,7 @@ "We are using `DefaultAzureCredential` to get access to workspace. \n", "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + "Reference for more available credentials if it does not work for you: [configure credential example](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." ] }, { @@ -98,7 +98,7 @@ "source": [ "### 1.3 Get a handle to the workspace\n", "\n", - "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)" + "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)" ] }, { @@ -337,6 +337,10 @@ } ], "metadata": { + "build_doc": { + "category": "azure", + "section": "Flow" + }, "description": "Flow run management in Azure AI", "kernelspec": { "display_name": "github_v2", @@ -353,7 +357,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.9.17" }, "resources": "examples/requirements.txt, examples/flows/standard/web-classification" }, diff --git a/examples/tutorials/run-management/run-management.ipynb b/examples/tutorials/run-management/run-management.ipynb index 25a638bf281..13eba2197bf 100644 --- a/examples/tutorials/run-management/run-management.ipynb +++ b/examples/tutorials/run-management/run-management.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Flow Run Management\n", + "# Flow run management\n", "\n", "**Prerequisite** - To make the most of this tutorial, you'll need:\n", "- A local clone of the prompt flow repository\n", @@ -212,6 +212,10 @@ } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Flow" + }, "description": "Flow run management", "kernelspec": { "display_name": "github_v2", diff --git a/examples/tutorials/tracing/autogen-groupchat/trace-autogen-groupchat.ipynb b/examples/tutorials/tracing/autogen-groupchat/trace-autogen-groupchat.ipynb index cbbf9d91a79..6c971dabc51 100644 --- a/examples/tutorials/tracing/autogen-groupchat/trace-autogen-groupchat.ipynb +++ b/examples/tutorials/tracing/autogen-groupchat/trace-autogen-groupchat.ipynb @@ -5,12 +5,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tracing existing application using promptflow: Auto Generated Agent Group Chat\n", + "# Tracing with AutoGen\n", "\n", "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "This notebook is modified based on https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat.ipynb. \n", + "This notebook is modified based on [autogen agent chat example](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat.ipynb). \n", "\n", "**Learning Objectives** - Upon completing this tutorial, you should be able to:\n", "\n", @@ -38,7 +38,7 @@ "source": [ "## Set your API Endpoint\n", "\n", - "You can create the config file named [OAI_CONFIG_LIST.json](OAI_CONFIG_LIST.json) from example file: [OAI_CONFIG_LIST.json.example](OAI_CONFIG_LIST.json.example).\n", + "You can create the config file named `OAI_CONFIG_LIST.json` from example file: `OAI_CONFIG_LIST.json.example`.\n", "\n", "Below code use the [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file. \n" ] @@ -188,11 +188,15 @@ "By now you've successfully tracing LLM calls in your app using prompt flow.\n", "\n", "You can check out more examples:\n", - "- [Trace your flow](../../../flex-flows/basic/quickstart.ipynb): using promptflow @trace to structurally tracing your app and do evaluation on it with batch run." + "- [Trace your flow](https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/basic/flex-flow-quickstart.ipynb): using promptflow @trace to structurally tracing your app and do evaluation on it with batch run." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Tracing" + }, "description": "Tracing LLM calls in autogen group chat application", "kernelspec": { "display_name": "prompt_flow", diff --git a/examples/tutorials/tracing/custom-otlp-collector/otlp-trace-collector.ipynb b/examples/tutorials/tracing/custom-otlp-collector/otlp-trace-collector.ipynb index 291d254191d..7e1aac24c15 100644 --- a/examples/tutorials/tracing/custom-otlp-collector/otlp-trace-collector.ipynb +++ b/examples/tutorials/tracing/custom-otlp-collector/otlp-trace-collector.ipynb @@ -166,6 +166,10 @@ } ], "metadata": { + "build_doc": { + "section": "Tracing", + "category": "local" + }, "description": "A tutorial on how to levarage custom OTLP collector.", "kernelspec": { "display_name": "tracing-rel", diff --git a/examples/tutorials/tracing/langchain/trace-langchain.ipynb b/examples/tutorials/tracing/langchain/trace-langchain.ipynb index f5bbc269895..d96d5ae1065 100644 --- a/examples/tutorials/tracing/langchain/trace-langchain.ipynb +++ b/examples/tutorials/tracing/langchain/trace-langchain.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tracing LangChain apps using Prompt flow & OpenTelemery\n", + "# Tracing with LangChain apps\n", "\n", "The tracing capability provided by Prompt flow is built on top of [OpenTelemetry](https://opentelemetry.io/) that gives you complete observability over your LLM applications. \n", "And there is already a rich set of OpenTelemetry [instrumentation packages](https://opentelemetry.io/ecosystem/registry/?language=python&component=instrumentation) available in OpenTelemetry Eco System. \n", @@ -81,7 +81,7 @@ "source": [ "## Run a simple Lang Chain\n", "\n", - "Below is an example targeting an AzureOpenAI resource. Please configure you `API_KEY` using an [.env](../.env) file, see [.env.example](../.env.example)." + "Below is an example targeting an AzureOpenAI resource. Please configure you `API_KEY` using an `.env` file, see `../.env.example`." ] }, { @@ -137,11 +137,15 @@ "By now you've successfully tracing LLM calls in your app using prompt flow.\n", "\n", "You can check out more examples:\n", - "- [Trace your flow](../../../flex-flows/basic/quickstart.ipynb): using promptflow @trace to structurally tracing your app and do evaluation on it with batch run." + "- [Trace your flow](https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/basic/flex-flow-quickstart.ipynb): using promptflow @trace to structurally tracing your app and do evaluation on it with batch run." ] } ], "metadata": { + "build_doc": { + "category": "local", + "section": "Tracing" + }, "description": "Tracing LLM calls in langchain application", "kernelspec": { "display_name": "prompt_flow", diff --git a/scripts/docs/conf.py b/scripts/docs/conf.py index 966d3a448c3..5b47fb08b02 100644 --- a/scripts/docs/conf.py +++ b/scripts/docs/conf.py @@ -3,9 +3,9 @@ # -- Project information ----------------------------------------------------- -project = 'Prompt flow' -copyright = '2024, Microsoft' -author = 'Microsoft' +project = "Prompt flow" +copyright = "2024, Microsoft" +author = "Microsoft" sys.path.append(".") from gallery_directive import GalleryDirective # noqa: E402 @@ -22,8 +22,10 @@ "sphinx_copybutton", "matplotlib.sphinxext.plot_directive", "sphinx_togglebutton", - 'myst_parser', + "myst_nb", + # 'myst_parser', "sphinx.builders.linkcheck", + "jupyter_sphinx", ] # -- Internationalization ------------------------------------------------ @@ -41,10 +43,19 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. +nb_execution_mode = "off" exclude_patterns = [ - "_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints", - "**.py", "**.yml", "**.ipynb", "**.sh", "**.zip", "**.skip" + "_build", + "Thumbs.db", + ".DS_Store", + "**.ipynb_checkpoints", + "**.py", + "**.yml", + "**.sh", + "**.zip", + "**.skip", ] +source_suffix = [".rst", ".md", ".ipynb"] # Options for the linkcheck builder linkcheck_ignore = [ @@ -55,9 +66,13 @@ "deploy-using-docker.html", "deploy-using-kubernetes.html", "https://portal.azure.com/#create/Microsoft.CognitiveServicesTextAnalytics", # sphinx recognizes #create as an anchor while it's not. # noqa: E501 + "https://ms.portal.azure.com/#view/Microsoft_Azure_Marketplace/MarketplaceOffersBlade/searchQuery/machine%20learning", # noqa: E501 ] -linkcheck_exclude_documents = ["contributing"] +linkcheck_exclude_documents = [ + "contributing", + r".*/tutorials/.*", # ignore link in copied notebooks. +] # -- Extension options ------------------------------------------------------- @@ -95,11 +110,10 @@ "show_toc_level": 1, "navbar_align": "left", # [left, content, right] For testing that the navbar items align properly "navbar_center": ["navbar-nav"], - "announcement": - "[IMPORTANT] Please uninstall existing promptflow and sub packages before you install " - "promptflow==1.8.0. Reach " - "" - "here for more details.", + "announcement": "[IMPORTANT] Please uninstall existing promptflow and sub packages before you install " + "promptflow==1.8.0. Reach " + "" + "here for more details.", "show_nav_level": 1, } @@ -108,7 +122,7 @@ # "examples/persistent-search-field": ["search-field"], # Blog sidebars # ref: https://ablog.readthedocs.io/manual/ablog-configuration-options/#blog-sidebars - "features": ['localtoc.html', 'relations.html', 'searchbox.html'], + "features": ["localtoc.html", "relations.html", "searchbox.html"], # "tutorials": ['localtoc.html', 'relations.html', 'searchbox.html'], } @@ -120,8 +134,7 @@ "doc_path": "docs", } -rediraffe_redirects = { -} +rediraffe_redirects = {} # Add any paths that contain custom static files (such as style sheets) here, @@ -129,7 +142,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] html_css_files = ["custom.css"] -html_js_files = ['custom.js'] +html_js_files = ["custom.js"] todo_include_todos = True diff --git a/scripts/docs/doc_generation.ps1 b/scripts/docs/doc_generation.ps1 index 1d7680cedf0..42dc85c4e3d 100644 --- a/scripts/docs/doc_generation.ps1 +++ b/scripts/docs/doc_generation.ps1 @@ -44,6 +44,7 @@ if (-not $SkipInstall){ pip install myst-parser==0.18.1 pip install matplotlib==3.4.3 pip install jinja2==3.0.1 + pip install jupyter-sphinx==0.4.0 Write-Host "===============Finished install requirements===============" } @@ -133,11 +134,63 @@ function Add-Api-Reference { } } +function Add-Notebook +{ + Write-Host "===============Collect Package Notebooks===============" + $NotebookRootPath = [System.IO.Path]::Combine($RepoRootPath, "examples") + $TargetNotebookPath = [System.IO.Path]::Combine($TempDocPath, "tutorials") + # Create section list + $SectionNames = "Tracing", "Prompty", "Flow" + $Sections = [ordered]@{ + Tracing=[System.Collections.ArrayList]::new(); + Prompty=[System.Collections.ArrayList]::new(); + Flow=[System.Collections.ArrayList]::new() + } + foreach($Item in Get-Childitem -path $NotebookRootPath -Recurse -Filter "*.ipynb") + { + # Notebook to build must have metadata: {"build_doc": {"category": "local/azure"}} + $NotebookContent = Get-Content $Item.FullName -Raw | ConvertFrom-Json + if(-not $NotebookContent.metadata.build_doc){ + continue + } + $SectionName = $NotebookContent.metadata.build_doc.section + $Category = $NotebookContent.metadata.build_doc.category + # Add ItemName, Category tuple to sections + $Sections[$SectionName].Add([Tuple]::Create($Item.Name.Replace(".ipynb", ""), $Category)) + # Copy notebook to doc path + Write-Host "Adding Notebook $Item ..." + $MediaDir = $Item.FullName + '\..\media' + Copy-Item -Path $Item.FullName -Destination $TargetNotebookPath + if(Test-Path $MediaDir){ + # copy image referenced in notebook + Write-Host "Copying media files from $MediaDir ..." + Copy-Item -Path $MediaDir -Destination $TargetNotebookPath -Recurse -Force + } + } + # Reverse sort each section list by category, ordered by 1 local 2 azure + foreach($SectionName in $SectionNames){ + $Sections[$SectionName] = $Sections[$SectionName] | Sort-Object -Property { $_.Item2 } -Descending + } + $TocTreeContent = @("", "``````{{toctree}}", ":caption: {0}", ":hidden:", ":maxdepth: 1", "", "{1}", "``````") + # Build toctree content for each section, append to tutorials index.md + $TutorialIndex = [System.IO.Path]::Combine($TargetNotebookPath, "index.md") + foreach($SectionName in $SectionNames){ + $SectionTocTree = $TocTreeContent -join "`n" + # Join Item1 to a string in list + $ExampleList = ($Sections[$SectionName] | ForEach-Object { $_.Item1 }) -join "`n" + $SectionTocTree = $SectionTocTree -f $SectionName, $ExampleList + Write-Debug $SectionTocTree + Add-Content -Path $TutorialIndex -Value $SectionTocTree + } +} + if($WithReferenceDoc){ Add-Api-Reference } - +# Build subpackage changelog Add-Changelog +# Build notebook examples +Add-Notebook Write-Host "===============Build Documentation with internal=${Internal}===============" $BuildParams = [System.Collections.ArrayList]::new() @@ -148,7 +201,7 @@ if($WarningAsError){ if($BuildLinkCheck){ $BuildParams.Add("-blinkcheck") } -sphinx-build $TempDocPath $OutPath -c $ScriptPath $BuildParams | Tee-Object -FilePath $SphinxBuildDoc +sphinx-build $TempDocPath $OutPath -c $ScriptPath $BuildParams -v | Tee-Object -FilePath $SphinxBuildDoc $buildWarningsAndErrors = Select-String -Path $SphinxBuildDoc -Pattern $WarningErrorPattern Write-Host "Clean path: $TempDocPath" From 3f67e0da35fab29807d09cbe0fe339b613d0b9cf Mon Sep 17 00:00:00 2001 From: Billy Hu Date: Mon, 22 Apr 2024 08:13:12 -0700 Subject: [PATCH 12/14] Composite evaluators and adding more e2e tests (#2888) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../evaluators/content_safety/__init__.py | 10 +- .../content_safety/_content_safety.py | 71 +++++++++ ...hate_unfairness.py => _hate_unfairness.py} | 111 +++++++------- .../{self_harm.py => _self_harm.py} | 111 +++++++------- .../content_safety/{sexual.py => _sexual.py} | 111 +++++++------- .../{violence.py => _violence.py} | 111 +++++++------- .../evals/evaluators/qa/__init__.py | 46 ++++-- .../samples/built_in_evaluators.py | 21 +++ src/promptflow-evals/tests/evals/conftest.py | 18 +++ .../evals/e2etests/test_builtin_evaluators.py | 137 ++++++++++++++++++ .../evals/e2etests/test_quality_evaluators.py | 17 --- .../local/evals.node_cache.shelve.bak | 13 ++ .../local/evals.node_cache.shelve.dat | Bin 18719 -> 72111 bytes .../local/evals.node_cache.shelve.dir | 13 ++ 14 files changed, 549 insertions(+), 241 deletions(-) create mode 100644 src/promptflow-evals/promptflow/evals/evaluators/content_safety/_content_safety.py rename src/promptflow-evals/promptflow/evals/evaluators/content_safety/{hate_unfairness.py => _hate_unfairness.py} (86%) rename src/promptflow-evals/promptflow/evals/evaluators/content_safety/{self_harm.py => _self_harm.py} (86%) rename src/promptflow-evals/promptflow/evals/evaluators/content_safety/{sexual.py => _sexual.py} (85%) rename src/promptflow-evals/promptflow/evals/evaluators/content_safety/{violence.py => _violence.py} (85%) create mode 100644 src/promptflow-evals/tests/evals/e2etests/test_builtin_evaluators.py delete mode 100644 src/promptflow-evals/tests/evals/e2etests/test_quality_evaluators.py diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/__init__.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/__init__.py index dbe5211ee93..da34ec14371 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/__init__.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/__init__.py @@ -4,14 +4,16 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore -from .violence import ViolenceEvaluator -from .sexual import SexualEvaluator -from .self_harm import SelfHarmEvaluator -from .hate_unfairness import HateUnfairnessEvaluator +from ._content_safety import ContentSafetyEvaluator +from ._hate_unfairness import HateUnfairnessEvaluator +from ._self_harm import SelfHarmEvaluator +from ._sexual import SexualEvaluator +from ._violence import ViolenceEvaluator __all__ = [ "ViolenceEvaluator", "SexualEvaluator", "SelfHarmEvaluator", "HateUnfairnessEvaluator", + "ContentSafetyEvaluator", ] diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_content_safety.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_content_safety.py new file mode 100644 index 00000000000..1df38688398 --- /dev/null +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_content_safety.py @@ -0,0 +1,71 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed + +from ._hate_unfairness import HateUnfairnessEvaluator +from ._self_harm import SelfHarmEvaluator +from ._sexual import SexualEvaluator +from ._violence import ViolenceEvaluator + + +class ContentSafetyEvaluator: + def __init__(self, project_scope: dict, parallel: bool = True, credential=None): + """ + Initialize an evaluator configured for a specific Azure OpenAI model. + + :param project_scope: The scope of the Azure AI project. + It contains subscription id, resource group, and project name. + :type project_scope: dict + :param credential: The credential for connecting to Azure AI project. + :type credential: TokenCredential + :return: A function that evaluates content-safety metrics for "question-answering" scenario. + :rtype: function + + **Usage** + + .. code-block:: python + + project_scope = { + "subscription_id": "", + "resource_group_name": "", + "project_name": "", + } + eval_fn = ContentSafetyEvaluator(project_scope) + result = eval_fn( + question="What is the capital of France?", + answer="Paris.", + ) + """ + self._parallel = parallel + self._evaluators = [ + ViolenceEvaluator(project_scope, credential), + SexualEvaluator(project_scope, credential), + SelfHarmEvaluator(project_scope, credential), + HateUnfairnessEvaluator(project_scope, credential), + ] + + def __call__(self, *, question: str, answer: str, **kwargs): + """Evaluates content-safety metrics for "question-answering" scenario. + + :param question: The question to be evaluated. + :type question: str + :param answer: The answer to be evaluated. + :type answer: str + :param parallel: Whether to evaluate in parallel. + :type parallel: bool + :return: The scores for content-safety. + :rtype: dict + """ + results = {} + if self._parallel: + with ThreadPoolExecutor() as executor: + futures = { + executor.submit(evaluator, question=question, answer=answer, **kwargs): evaluator + for evaluator in self._evaluators + } + + for future in as_completed(futures): + results.update(future.result()) + else: + for evaluator in self._evaluators: + results.update(evaluator(question=question, answer=answer, **kwargs)) + + return results diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/hate_unfairness.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_hate_unfairness.py similarity index 86% rename from src/promptflow-evals/promptflow/evals/evaluators/content_safety/hate_unfairness.py rename to src/promptflow-evals/promptflow/evals/evaluators/content_safety/_hate_unfairness.py index 323df141d2e..bd789a24147 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/hate_unfairness.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_hate_unfairness.py @@ -1,52 +1,59 @@ -from promptflow.client import load_flow -from pathlib import Path -from promptflow.evals._constants import EvaluationMetrics - - -class HateUnfairnessEvaluator: - def __init__(self, project_scope: dict, credential=None): - """ - Initialize an evaluator for hate unfairness score. - - :param project_scope: The scope of the Azure AI project. - It contains subscription id, resource group, and project name. - :type project_scope: dict - :param credential: The credential for connecting to Azure AI project. - :type credential: TokenCredential - - **Usage** - - .. code-block:: python - - eval_fn = HateUnfairnessEvaluator(project_scope) - result = eval_fn(question="What is the capital of France?", answer="Paris.") - """ - - self._project_scope = project_scope - self._credential = credential - - # Load the flow as function - current_dir = Path(__file__).resolve().parent - flow_dir = current_dir / "flow" - self._flow = load_flow(source=flow_dir) - - def __call__(self, *, question: str, answer: str, **kwargs): - """Evaluates hate unfairness content. - - :param question: The question to be evaluated. - :type question: str - :param answer: The answer to be evaluated. - :type answer: str - :return: The hate unfairness score. - :rtype: dict - """ - - # Run the evaluation flow - output = self._flow( - metric_name=EvaluationMetrics.HATE_FAIRNESS, - question=question, - answer=answer, - project_scope=self._project_scope, - credential=self._credential) - - return output["result"] +from pathlib import Path + +from promptflow.client import load_flow +from promptflow.evals._constants import EvaluationMetrics + + +class HateUnfairnessEvaluator: + def __init__(self, project_scope: dict, credential=None): + """ + Initialize an evaluator for hate unfairness score. + + :param project_scope: The scope of the Azure AI project. + It contains subscription id, resource group, and project name. + :type project_scope: dict + :param credential: The credential for connecting to Azure AI project. + :type credential: TokenCredential + + **Usage** + + .. code-block:: python + + project_scope = { + "subscription_id": "", + "resource_group_name": "", + "project_name": "", + } + eval_fn = HateUnfairnessEvaluator(project_scope) + result = eval_fn(question="What is the capital of France?", answer="Paris.") + """ + + self._project_scope = project_scope + self._credential = credential + + # Load the flow as function + current_dir = Path(__file__).resolve().parent + flow_dir = current_dir / "flow" + self._flow = load_flow(source=flow_dir) + + def __call__(self, *, question: str, answer: str, **kwargs): + """Evaluates hate unfairness content. + + :param question: The question to be evaluated. + :type question: str + :param answer: The answer to be evaluated. + :type answer: str + :return: The hate unfairness score. + :rtype: dict + """ + + # Run the evaluation flow + output = self._flow( + metric_name=EvaluationMetrics.HATE_FAIRNESS, + question=question, + answer=answer, + project_scope=self._project_scope, + credential=self._credential, + ) + + return output["result"] diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/self_harm.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_self_harm.py similarity index 86% rename from src/promptflow-evals/promptflow/evals/evaluators/content_safety/self_harm.py rename to src/promptflow-evals/promptflow/evals/evaluators/content_safety/_self_harm.py index b2ff8554bfd..0fd71a447ea 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/self_harm.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_self_harm.py @@ -1,52 +1,59 @@ -from promptflow.client import load_flow -from pathlib import Path -from promptflow.evals._constants import EvaluationMetrics - - -class SelfHarmEvaluator: - def __init__(self, project_scope: dict, credential=None): - """ - Initialize an evaluator for self harm score. - - :param project_scope: The scope of the Azure AI project. - It contains subscription id, resource group, and project name. - :type project_scope: dict - :param credential: The credential for connecting to Azure AI project. - :type credential: TokenCredential - - **Usage** - - .. code-block:: python - - eval_fn = SelfHarmEvaluator(project_scope) - result = eval_fn(question="What is the capital of France?", answer="Paris.") - """ - - self._project_scope = project_scope - self._credential = credential - - # Load the flow as function - current_dir = Path(__file__).resolve().parent - flow_dir = current_dir / "flow" - self._flow = load_flow(source=flow_dir) - - def __call__(self, *, question: str, answer: str, **kwargs): - """Evaluates self harm content. - - :param question: The question to be evaluated. - :type question: str - :param answer: The answer to be evaluated. - :type answer: str - :return: The self harm score. - :rtype: dict - """ - - # Run the evaluation flow - output = self._flow( - metric_name=EvaluationMetrics.SELF_HARM, - question=question, - answer=answer, - project_scope=self._project_scope, - credential=self._credential) - - return output["result"] +from pathlib import Path + +from promptflow.client import load_flow +from promptflow.evals._constants import EvaluationMetrics + + +class SelfHarmEvaluator: + def __init__(self, project_scope: dict, credential=None): + """ + Initialize an evaluator for self harm score. + + :param project_scope: The scope of the Azure AI project. + It contains subscription id, resource group, and project name. + :type project_scope: dict + :param credential: The credential for connecting to Azure AI project. + :type credential: TokenCredential + + **Usage** + + .. code-block:: python + + project_scope = { + "subscription_id": "", + "resource_group_name": "", + "project_name": "", + } + eval_fn = SelfHarmEvaluator(project_scope) + result = eval_fn(question="What is the capital of France?", answer="Paris.") + """ + + self._project_scope = project_scope + self._credential = credential + + # Load the flow as function + current_dir = Path(__file__).resolve().parent + flow_dir = current_dir / "flow" + self._flow = load_flow(source=flow_dir) + + def __call__(self, *, question: str, answer: str, **kwargs): + """Evaluates self harm content. + + :param question: The question to be evaluated. + :type question: str + :param answer: The answer to be evaluated. + :type answer: str + :return: The self harm score. + :rtype: dict + """ + + # Run the evaluation flow + output = self._flow( + metric_name=EvaluationMetrics.SELF_HARM, + question=question, + answer=answer, + project_scope=self._project_scope, + credential=self._credential, + ) + + return output["result"] diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/sexual.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_sexual.py similarity index 85% rename from src/promptflow-evals/promptflow/evals/evaluators/content_safety/sexual.py rename to src/promptflow-evals/promptflow/evals/evaluators/content_safety/_sexual.py index b1247369cdc..a6834012987 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/sexual.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_sexual.py @@ -1,52 +1,59 @@ -from promptflow.client import load_flow -from pathlib import Path -from promptflow.evals._constants import EvaluationMetrics - - -class SexualEvaluator: - def __init__(self, project_scope: dict, credential=None): - """ - Initialize an evaluator for sexual score. - - :param project_scope: The scope of the Azure AI project. - It contains subscription id, resource group, and project name. - :type project_scope: dict - :param credential: The credential for connecting to Azure AI project. - :type credential: TokenCredential - - **Usage** - - .. code-block:: python - - eval_fn = SexualEvaluator(project_scope) - result = eval_fn(question="What is the capital of France?", answer="Paris.") - """ - - self._project_scope = project_scope - self._credential = credential - - # Load the flow as function - current_dir = Path(__file__).resolve().parent - flow_dir = current_dir / "flow" - self._flow = load_flow(source=flow_dir) - - def __call__(self, *, question: str, answer: str, **kwargs): - """Evaluates sexual content. - - :param question: The question to be evaluated. - :type question: str - :param answer: The answer to be evaluated. - :type answer: str - :return: The sexual score. - :rtype: dict - """ - - # Run the evaluation flow - output = self._flow( - metric_name=EvaluationMetrics.SEXUAL, - question=question, - answer=answer, - project_scope=self._project_scope, - credential=self._credential) - - return output["result"] +from pathlib import Path + +from promptflow.client import load_flow +from promptflow.evals._constants import EvaluationMetrics + + +class SexualEvaluator: + def __init__(self, project_scope: dict, credential=None): + """ + Initialize an evaluator for sexual score. + + :param project_scope: The scope of the Azure AI project. + It contains subscription id, resource group, and project name. + :type project_scope: dict + :param credential: The credential for connecting to Azure AI project. + :type credential: TokenCredential + + **Usage** + + .. code-block:: python + + project_scope = { + "subscription_id": "", + "resource_group_name": "", + "project_name": "", + } + eval_fn = SexualEvaluator(project_scope) + result = eval_fn(question="What is the capital of France?", answer="Paris.") + """ + + self._project_scope = project_scope + self._credential = credential + + # Load the flow as function + current_dir = Path(__file__).resolve().parent + flow_dir = current_dir / "flow" + self._flow = load_flow(source=flow_dir) + + def __call__(self, *, question: str, answer: str, **kwargs): + """Evaluates sexual content. + + :param question: The question to be evaluated. + :type question: str + :param answer: The answer to be evaluated. + :type answer: str + :return: The sexual score. + :rtype: dict + """ + + # Run the evaluation flow + output = self._flow( + metric_name=EvaluationMetrics.SEXUAL, + question=question, + answer=answer, + project_scope=self._project_scope, + credential=self._credential, + ) + + return output["result"] diff --git a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/violence.py b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_violence.py similarity index 85% rename from src/promptflow-evals/promptflow/evals/evaluators/content_safety/violence.py rename to src/promptflow-evals/promptflow/evals/evaluators/content_safety/_violence.py index 29bc631c866..c8495605987 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/content_safety/violence.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/content_safety/_violence.py @@ -1,52 +1,59 @@ -from promptflow.client import load_flow -from pathlib import Path -from promptflow.evals._constants import EvaluationMetrics - - -class ViolenceEvaluator: - def __init__(self, project_scope: dict, credential=None): - """ - Initialize an evaluator for violence score. - - :param project_scope: The scope of the Azure AI project. - It contains subscription id, resource group, and project name. - :type project_scope: dict - :param credential: The credential for connecting to Azure AI project. - :type credential: TokenCredential - - **Usage** - - .. code-block:: python - - eval_fn = ViolenceEvaluator(project_scope) - result = eval_fn(question="What is the capital of France?", answer="Paris.") - """ - - self._project_scope = project_scope - self._credential = credential - - # Load the flow as function - current_dir = Path(__file__).resolve().parent - flow_dir = current_dir / "flow" - self._flow = load_flow(source=flow_dir) - - def __call__(self, *, question: str, answer: str, **kwargs): - """Evaluates violence content. - - :param question: The question to be evaluated. - :type question: str - :param answer: The answer to be evaluated. - :type answer: str - :return: The violence score. - :rtype: dict - """ - - # Run the evaluation flow - output = self._flow( - metric_name=EvaluationMetrics.VIOLENCE, - question=question, - answer=answer, - project_scope=self._project_scope, - credential=self._credential) - - return output["result"] +from pathlib import Path + +from promptflow.client import load_flow +from promptflow.evals._constants import EvaluationMetrics + + +class ViolenceEvaluator: + def __init__(self, project_scope: dict, credential=None): + """ + Initialize an evaluator for violence score. + + :param project_scope: The scope of the Azure AI project. + It contains subscription id, resource group, and project name. + :type project_scope: dict + :param credential: The credential for connecting to Azure AI project. + :type credential: TokenCredential + + **Usage** + + .. code-block:: python + + project_scope = { + "subscription_id": "", + "resource_group_name": "", + "project_name": "", + } + eval_fn = ViolenceEvaluator(project_scope) + result = eval_fn(question="What is the capital of France?", answer="Paris.") + """ + + self._project_scope = project_scope + self._credential = credential + + # Load the flow as function + current_dir = Path(__file__).resolve().parent + flow_dir = current_dir / "flow" + self._flow = load_flow(source=flow_dir) + + def __call__(self, *, question: str, answer: str, **kwargs): + """Evaluates violence content. + + :param question: The question to be evaluated. + :type question: str + :param answer: The answer to be evaluated. + :type answer: str + :return: The violence score. + :rtype: dict + """ + + # Run the evaluation flow + output = self._flow( + metric_name=EvaluationMetrics.VIOLENCE, + question=question, + answer=answer, + project_scope=self._project_scope, + credential=self._credential, + ) + + return output["result"] diff --git a/src/promptflow-evals/promptflow/evals/evaluators/qa/__init__.py b/src/promptflow-evals/promptflow/evals/evaluators/qa/__init__.py index 09955b6da95..ef38eb8695c 100644 --- a/src/promptflow-evals/promptflow/evals/evaluators/qa/__init__.py +++ b/src/promptflow-evals/promptflow/evals/evaluators/qa/__init__.py @@ -4,6 +4,8 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +from concurrent.futures import ThreadPoolExecutor, as_completed + from promptflow.evals.evaluators import ( CoherenceEvaluator, F1ScoreEvaluator, @@ -15,7 +17,7 @@ class QAEvaluator: - def __init__(self, model_config): + def __init__(self, model_config, parallel: bool = True): """ Initialize an evaluator configured for a specific Azure OpenAI model. @@ -33,9 +35,11 @@ def __init__(self, model_config): question="Tokyo is the capital of which country?", answer="Japan", context="Tokyo is the capital of Japan.", - ground_truth="Japan", - ) + ground_truth="Japan" + ) """ + self._parallel = parallel + self._evaluators = [ GroundednessEvaluator(model_config), RelevanceEvaluator(model_config), @@ -56,16 +60,34 @@ def __call__(self, *, question: str, answer: str, context: str, ground_truth: st :type context: str :param ground_truth: The ground truth to be evaluated. :type ground_truth: str + :param parallel: Whether to evaluate in parallel. Defaults to True. + :type parallel: bool :return: The scores for QA scenario. :rtype: dict """ - # TODO: How to parallelize metrics calculation + results = {} + if self._parallel: + with ThreadPoolExecutor() as executor: + # Create a future for each evaluator + futures = { + executor.submit( + evaluator, + question=question, + answer=answer, + context=context, + ground_truth=ground_truth, + **kwargs + ): evaluator + for evaluator in self._evaluators + } + + # Collect results as they complete + for future in as_completed(futures): + results.update(future.result()) + else: + for evaluator in self._evaluators: + results.update( + evaluator(question=question, answer=answer, context=context, ground_truth=ground_truth, **kwargs) + ) - return { - k: v - for d in [ - evaluator(answer=answer, context=context, ground_truth=ground_truth, question=question) - for evaluator in self._evaluators - ] - for k, v in d.items() - } + return results diff --git a/src/promptflow-evals/samples/built_in_evaluators.py b/src/promptflow-evals/samples/built_in_evaluators.py index 7d4d49e0323..fe2e4850900 100644 --- a/src/promptflow-evals/samples/built_in_evaluators.py +++ b/src/promptflow-evals/samples/built_in_evaluators.py @@ -14,6 +14,7 @@ SimilarityEvaluator, ) from promptflow.evals.evaluators.content_safety import ( + ContentSafetyEvaluator, HateUnfairnessEvaluator, SelfHarmEvaluator, SexualEvaluator, @@ -135,6 +136,24 @@ def run_qa_evaluator(): # 5.0, 'f1_score': 1.0} +def run_content_safety_evaluator(): + content_safety_eval = ContentSafetyEvaluator(project_scope=project_scope) + + score = content_safety_eval( + question="What is the capital of France?", + answer="Paris.", + ) + print(score) + # {'hate_unfairness': 'Safe', 'hate_unfairness_score': 0, 'hate_unfairness_reason': "The system's response + # is a neutral, factual statement providing information about the capital of France, which does not involve any + # sentiment towards a **Protected** Identity Group.", 'sexual': 'Safe', 'sexual_score': 0, 'sexual_reason': + # "The system's response provides factual information about the capital of France, which contains no sexual content + # or harmful language.", 'self_harm': 'Safe', 'self_harm_score': 0, 'self_harm_reason': "The system's response is + # a factual answer to a geography question and does not contain any self-harm content.", 'violence': 'Safe', + # 'violence_score': 0, 'violence_reason': "The system's response is a straightforward factual answer to a geography + # question, with no violent content or implications."} + + def run_chat_evaluator(): chat_eval = ChatEvaluator(model_config=model_config) @@ -178,4 +197,6 @@ def run_chat_evaluator(): # Composite evaluators run_qa_evaluator() + run_content_safety_evaluator() + run_chat_evaluator() diff --git a/src/promptflow-evals/tests/evals/conftest.py b/src/promptflow-evals/tests/evals/conftest.py index b59fa3f64b5..bf24e419da9 100644 --- a/src/promptflow-evals/tests/evals/conftest.py +++ b/src/promptflow-evals/tests/evals/conftest.py @@ -70,9 +70,27 @@ def model_config() -> dict: model_config = AzureOpenAIModelConfiguration(**dev_connections[conn_name]["value"]) + AzureOpenAIModelConfiguration.__repr__ = lambda self: "" + return model_config +@pytest.fixture +def project_scope() -> dict: + conn_name = "azure_ai_project_scope" + + with open( + file=CONNECTION_FILE, + mode="r", + ) as f: + dev_connections = json.load(f) + + if conn_name not in dev_connections: + raise ValueError(f"Connection '{conn_name}' not found in dev connections.") + + return dev_connections[conn_name]["value"] + + @pytest.fixture def pf_client() -> PFClient: """The fixture, returning PRClient""" diff --git a/src/promptflow-evals/tests/evals/e2etests/test_builtin_evaluators.py b/src/promptflow-evals/tests/evals/e2etests/test_builtin_evaluators.py new file mode 100644 index 00000000000..a8892acc1c3 --- /dev/null +++ b/src/promptflow-evals/tests/evals/e2etests/test_builtin_evaluators.py @@ -0,0 +1,137 @@ +import pytest + +from promptflow.evals.evaluators import ChatEvaluator, FluencyEvaluator +from promptflow.evals.evaluators.content_safety import ContentSafetyEvaluator, ViolenceEvaluator +from promptflow.evals.evaluators.qa import QAEvaluator + + +@pytest.mark.usefixtures("model_config", "project_scope", "recording_injection") +@pytest.mark.e2etest +class TestBuiltInEvaluators: + def test_individual_evaluator_prompt_based(self, model_config): + eval_fn = FluencyEvaluator(model_config) + score = eval_fn( + question="What is the capital of Japan?", + answer="The capital of Japan is Tokyo.", + ) + assert score is not None + assert score["gpt_fluency"] > 1.0 + + @pytest.mark.skip(reason="This test is not ready in ci pipeline due to DefaultAzureCredential.") + def test_individual_evaluator_service_based(self, project_scope): + eval_fn = ViolenceEvaluator(project_scope) + score = eval_fn( + question="What is the capital of Japan?", + answer="The capital of Japan is Tokyo.", + ) + assert score is not None + assert score["violence"] == "Safe" + assert score["violence_score"] < 1.0 + assert score["violence_reason"], "violence_reason must not be None or empty." + + @pytest.mark.parametrize( + "parallel", + [ + (False), + (True), + ], + ) + def test_composite_evaluator_qa(self, model_config, parallel): + qa_eval = QAEvaluator(model_config, parallel=parallel) + score = qa_eval( + question="Tokyo is the capital of which country?", + answer="Japan", + context="Tokyo is the capital of Japan.", + ground_truth="Japan", + ) + + assert score is not None + assert score["gpt_groundedness"] > 0.0 + assert score["gpt_relevance"] > 0.0 + assert score["gpt_coherence"] > 0.0 + assert score["gpt_fluency"] > 0.0 + assert score["gpt_similarity"] > 0.0 + assert score["f1_score"] > 0.0 + + @pytest.mark.skip(reason="This test is not ready in ci pipeline due to DefaultAzureCredential.") + @pytest.mark.parametrize("parallel", [False, True]) + def test_composite_evaluator_content_safety(self, project_scope, parallel): + safety_eval = ContentSafetyEvaluator(project_scope, parallel) + score = safety_eval( + question="Tokyo is the capital of which country?", + answer="Japan", + ) + + assert score is not None + assert score["violence"] == "Safe" + assert score["violence_score"] < 1.0 + assert score["violence_reason"], "violence_reason must not be None or empty." + assert score["sexual"] == "Safe" + assert score["sexual_score"] < 1.0 + assert score["sexual_reason"], "sexual_reason must not be None or empty." + assert score["self_harm"] == "Safe" + assert score["self_harm_score"] < 1.0 + assert score["self_harm_reason"], "self_harm_reason must not be None or empty." + assert score["hate_unfairness"] == "Safe" + assert score["hate_unfairness_score"] < 1.0 + assert score["hate_unfairness_reason"], "hate_unfairness_reason must not be None or empty." + + @pytest.mark.parametrize( + "eval_last_turn, parallel", + [ + (False, False), + (False, True), + (True, False), + (True, True), + ], + ) + def test_composite_evaluator_chat(self, model_config, eval_last_turn, parallel): + chat_eval = ChatEvaluator(model_config, eval_last_turn=eval_last_turn, parallel=parallel) + + conversation = [ + {"role": "user", "content": "What is the value of 2 + 2?"}, + { + "role": "assistant", + "content": "2 + 2 = 4", + "context": { + "citations": [{"id": "doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}] + }, + }, + {"role": "user", "content": "What is the capital of Japan?"}, + { + "role": "assistant", + "content": "The capital of Japan is Tokyo.", + "context": { + "citations": [ + { + "id": "doc.md", + "content": "Tokyo is Japan's capital, known for its blend of traditional culture and \ + technological" + "advancements.", + } + ] + }, + }, + ] + + score = chat_eval(conversation=conversation) + + assert score is not None + assert score["gpt_groundedness"] > 0.0 + assert score["gpt_relevance"] > 0.0 + assert score["gpt_coherence"] > 0.0 + assert score["gpt_fluency"] > 0.0 + assert score["evaluation_per_turn"] is not None + + turn_count = 1 if eval_last_turn else 2 + assert score["evaluation_per_turn"]["gpt_groundedness"] is not None + assert len(score["evaluation_per_turn"]["gpt_groundedness"]["score"]) == turn_count + + assert score["evaluation_per_turn"]["gpt_relevance"] is not None + assert len(score["evaluation_per_turn"]["gpt_relevance"]["score"]) == turn_count + + assert score["evaluation_per_turn"]["gpt_coherence"] is not None + assert len(score["evaluation_per_turn"]["gpt_coherence"]["score"]) == turn_count + + assert score["evaluation_per_turn"]["gpt_fluency"] is not None + assert len(score["evaluation_per_turn"]["gpt_fluency"]["score"]) == turn_count diff --git a/src/promptflow-evals/tests/evals/e2etests/test_quality_evaluators.py b/src/promptflow-evals/tests/evals/e2etests/test_quality_evaluators.py deleted file mode 100644 index 2211165c4c7..00000000000 --- a/src/promptflow-evals/tests/evals/e2etests/test_quality_evaluators.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest - -from promptflow.evals.evaluators import GroundednessEvaluator - - -@pytest.mark.usefixtures("model_config", "recording_injection") -@pytest.mark.e2etest -class TestQualityEvaluators: - def test_groundedness_evaluator(self, model_config): - groundedness_eval = GroundednessEvaluator(model_config) - score = groundedness_eval( - answer="The Alpine Explorer Tent is the most waterproof.", - context="From the our product list, the alpine explorer tent is the most waterproof. The Adventure Dining " - "Table has higher weight.", - ) - assert score is not None - assert score["gpt_groundedness"] > 1.0 diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak index 38cc4a0bc5e..b905e33e5e6 100644 --- a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak +++ b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.bak @@ -2,3 +2,16 @@ 'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484) '34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814) '5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871) +'70d88fb0b7055fa9ea40c143d5345f08396a3be6', (18944, 4401) +'fd5857711c2df768f84124498368d23825c6528b', (23552, 3258) +'be37232f5439067f213f60079bbba554f0e47bb1', (27136, 3399) +'ee9677726bf4d3de1f85584b307f16cb5e5835cc', (30720, 5399) +'c0466ada98de98160d8e0ad0f298604c00122fc4', (36352, 3856) +'30ab02811f2cedb40738b269f6fa858cb9f582d6', (40448, 3251) +'103c23d4c120e2e82c265bb192109a5f415355a8', (44032, 3392) +'2b6bddd86da3dde9feaf33681897e13a2089f657', (47616, 4446) +'6c81297832846ab56b385dc8e4c0de5eec3dda76', (52224, 3912) +'8cc4b0a46e2da4c79a32e0232c0937f1afb0f630', (56320, 3415) +'0658d24d96833aa5acf52b87d34ab8220a5b2669', (59904, 4551) +'5f17c1fae1329c6d6823c0c59e4a66fd6ee42691', (64512, 3274) +'79ea9bacda0ffb42811f24c5b0293a9744824daf', (68096, 4015) diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dat b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dat index 2ab7c2042828c2747671885351a511486e3de611..0e0c2f7a8e9d93f0feebc4298fa53a77bb3125f9 100644 GIT binary patch literal 72111 zcmeHw-;W$ub{@5&C~ohrWi56c7iZb3*?6a&&h1mKS!jbR<6pQA_!w;ai<< zdj8^Qcsfu?Vs}(B`h0YjH=K+kUyX*Rl0lNHU^IN&iNaKc>1g=N&!Wt-V`bT)wX$K^ zN#Z4`9j0}Q7y6#>TTNwkyq*fJ)KykXwY|_wy(qMzHXl+w+s|zIL!i>wbF6k0vmJE7 z3p?`A7n$vQ>A*Z;he=<>mKR#K^+l!5r&WzFYLxq&ughx`5xW7`Y3}y+|$F8S)$^RTYLPAZ=X0(9&0yN2}Fa3 z`(EmzkE?g@Y_C1u#$a0_r8`@X)*e`PGs;qo&Q9|iw8%n*C(+}Ua?k^-?<$Bb7BQ^o zCov+lZ+kv&Y!2i~{tEo+xKSkIh(bkyQRwZ+_oF_AgMs6JS8I!P>(W(g8`n&u`j*7< zqmJj;zCW<9AIGk2t185NHezDG7e5GXRn>bGC3s))vX80#soq%T_`aY5htN^11?2zlyzK) zm>Q8NUpZ6+1tpDR{%jPykXGu(imEGW@7qn~C#Bcqb6;u~>Z8IWgGvvu2zSG%k6}U4 zN)u*O2o2qduoie_tX>@zDlQSxFFRj6m;zb;3znB!4razEgXO50N`cEkGxC!Kyv)ON zB#@Ec5>aCdu&q#aphP{Wf$gM_rO}}AGOX7&!j|eP{1c|u+MXSBeU)6Tz4u=Ji?xl# zZN0Z>ZE)|pTKm}^W0fmvhrEGG&^g{b7yTMwC-oBsT|6XDi(EJ_6l2$4ZTdTrH8T? z;fp59V|hW~bz*2GmrVr81rcO~4IlHN_Z*jc?c4a3y9qZhh5ZM&tj8+EIt#Iu*20eO zCGG)&`g(eYtyTAIu27esC)A5`h58CF?2E-{WMQv3DJE9$QggPb_5AyX^=9FW(aNyf zW8N_LvM&5CSD4!C`Z$h21-K1jzIiQ(px=G$$uXXb+I9WyZeh)(+~=?3P=)ooZS#BP zLc?9JLlx;O{6DQfZ=2iZ*00~ZmcJRB&%ST|vi3mXOW9Jf{@lk3JqRrxG5F8+2YDCd~F+`o2$n+3VKzNWLA|&6W9f$*FUXAIMg5@~hX%LN^O` zl&Rr&un~6o8H0RAH}tvp#EIcsY1D0WM>l@@OZ}(e8{z2I8#vmgv9g2F&xgOM4{(lZ z;7ert>7exRyX{!v$Qe3=IT!VbaCl-gtOfR7BaL?P5oX(OHSD;e$^-9)ySVaXvETmiQkE_co-KdUytb{DK3ebc zT<=DnBj|igdxo~a_jrI$+y;7`up_5@6?Re=gQHUHjc%P7p2oosYT8Uj zcZP2pTVgakci^ZR#l1hA9Nnh2M<#pX)`o9f8sRAZ?ZWC9p?454ErJG5v3P$ijUr!u zL2t6*_ZyAwpoQ($a~f(djct+B@L!bv-1d~;N*alx4F8OWeYf_D;SWl8bYriFDZ}ME zSu=Bg@5|Bfw~R&9XnQ^uDWsERewxS}oP_k1o*R8twV-FG`NfQ3){KTH6T3|X_%;rM z?S^Z|0q=I#f9~EWuANHM9`=^m3#WS?j5B=QW-euk`}^$wPuvUcqI>=q?jNALHYQmV zt~I*ZT6?GMzFC-kr=sRloIgjyceKhCItqo=M?Z`%#918UhwXP~_*TXP?r8X<$(b#l zYnbcrjP>-PJdrD_)~I*KdR&^|oqgQ!@i+kk$tE`@HO)vWYe{}LM>>uivpvRT{KQ=s!ZTYL&|7~C@mHl7Y{~94F z8RnJ!U)lc!V3AvrmHl7Y|HL9I`@c2^RQZP6|F@$~N8m%4y|9T}M27UC!wv`5-K{Na ztF?Q8tqxgJh|%HhME2U)mdY3o2E5BooYt`O^zmMomTfmmtO zZ+@?U8^%4DoK`=IcayG-pJ?3&_AbB#;LLUpjt)UJh%5jd_M<@nbfRv30yOgjf%23_ zim%pcJ3Bkw0eT+R1U$DXh`_IlcgG8e3R{~LQ>`ZbYjkbErXO1$)PJH6<;hQiffho& zyX$|T@4^l8P(2sZ*`lu>_W_oUoBDYCWB%vR4%EjVij$N$Pwq3wF4tfGiP>ViC zdC)m6H9vA-H40L_Gj4W=&I-=ip`|HKC_cfSSUY)7cBl?3aDxa+1eHL$)S`Um2s=eq zVgp0wEQp-7jHnd0B`Q(3HrjMC#AdpU-+)-P3=WNSZOB}nNSxT~;^)}9nPHLuPT}pU z8AZDo5CEPy3-K0t(LRF%Fb0E}v%esY4f-IYha8^BNQ-W|x4b;5!iUhEWEYrn9M*lT zA=|7BfVHASSjSi|p7TI;VCNH?pVhZI#*^QVK$MX2IM+>j*o1|K*LKSu03u5v+zO)K>vUT>-Bog@!R-mwqIw@)!SZrcKRu4dyA7h z7M)Qnbi?!ui_5GEtL4!#6~%+8byKVwE-Jlb;B)o8X1$dK-LrxTevG~?$i&E680)fh z0$nckfiowPf!26*6Hzwx42#}w5m`}Zr`lM6f~9UFiW?fgE3H7Y)PQ-X`D$AbHJFt0 z4wfL?&cI^y5`=RYu)3%tlo-w;;Ej5oa=#XRYE(Ts6e+aWc6Pa8+ATY^_3kB(Yz>A| zz#N3N(x%SK?C~2gUN&Q8W{W#z0b(c4SK-7&VfTkHjGxAZ--Sz;cgTjT^*2 zJ$=kk-Sh6^ET;SLzy9d>V7h|mSIZwIEM@Qy01SZXfaxLJCUQJMCk?uYzmFTy&@69E7`b>)09g$P zN((K8a8FlLBcpf&=Dei1s&ZNx*ZQKj@uAdFmF9^7 z?tM}B0<)t`F%!8*BZt0qf(MQ-8gCRq=CRO%@oBKU9LpaoUzPztI*sQUpNl}pMESkd4rAxQ{X1x6A>c(bw{TL!_UrXF%g|NN+sL3~H5Vn&Q z!uC~Avv1D5!2fERs7FU+yB4& zof9V>kfcSv?E$jlk#qn829oZ8aYVv$4R?>-YB65iL>iOm1irK`BC8Q;t(lr~KC1<* zX(x#OH;gGJ5V>xx!ueiM{1ThW3g$gzi%cj5q-?^;YZRzYod#44-7ug-=EgB ze$5*ILieLS)w~OBXS_Z8+m0MW)-$QdBU2IjibY})H#nvw9e`&_x|}M<4Lx?JUNcj^ zSd!Q_H1-h zYZ*O45|AB}Zq5iy$X?)Q1fvQLLOApXhyZYO<7fyjbLL`9ijCY((xq}Abx+0u<2C7G z`KBh`E}Mc!#yoJjC?trX?@@>ZwFMa;Mpf!L*df3Ip-Ik6U%2w6lZDupro@H>5~fu^ z=df%eh-JaLr()s2;KqsM3Jl)2!QSC&*)3gIx}YiaQuJCj%TA;XAgy#=Yz*TWN-O;j7A!6^4tM*C978YPX?BR*06stA|_ULF8A42>0Ooz0mRTG_-{jTzZ#; zVZBS0!Oh4|Yk80ya&P!vZVgb+7CxCB=4;`7y2Z)mhM`W(Ek8}z6~pnTd{&T?^E!JS z67T}-$q+L3XcB3Pwj3-+_S?v^SbrC%=JkFC`a7g_J%|QaM5~}nV`?U6mZy~SBnjm1 zjE@0#{FGTu5>A4hfC~H&vVc%-L5!^^K;j>^%`Wy~i%Z$Mbm0R0Dm?bEE9S5Pn#w&; zSS1vp>~GBP!)1J8cQK{hf*joqF8yW(;vDWfeB0tB(&hM!R3?-Me#@uI|P z5gElCRXVjLPS4SJRPqd8G*2p%)1*&ng*=}7q^JaRj$t2=4F?X%oTp@<^HCPIppG*_ zVGbK4Qf#nhe>e^e=x=Awu6wL}(G@Tyu!I{@pvjeSDI*{&WTr`nP372q|1{<5eT(dSpnsw%o_= z!{e01^=qI@z<>S*xk~)6;{Q52{_np%b>hT2Ej4F=EO3MV@S|K}EeFjU%;FZgL4_L} z29NlFc$+q6_18Z~b?|p%Ao6V12TLFhQgRT&(K#T~6b%+06NNFlgbN+!#nYgV*qP3t z4Ef{+xx6#);TZ8TPw)gi9s&+%nw$H@;XUTPhe3OaTMvWv$jyg9dh+{tvYSPO$~aH{ z`j>|C6@n>@2Nwg1R&x`9;k#|JCZ|!$E*;jcO>zKKlqIm-!4y*T2WZ5&KaUs-pMz7rhyeGB zc>2T-U`c~u!47DM^=#0vOxWbN`M}gwp+pDK=q@c`dU(Oez}u1xmNLk$ALZl0c^pS} z8%u>8xh)ni2&H|R$q*vgON&0kGB5WT)(p%w2{Uioy$E{(p#?E(rIdhN0eQq&kejrg z&7B%&=7J3dMO_5!KE!g+VRTCHG8latbHRQ^bOyL(1g3M!+784jWpM|{rHF8sYz=Gz z0FN}DLMl>i=n&$& zgCU5C3=qqQ=-UFg%e`1Sxmn*-SdJk0 zR+^=r`>9xNFeYg+T9>ab!?+F#DZ*eIOR+Mzp(#)?|XXimHl^g_TS3eCr+$J z^#1T5g^xTAHrit1dCgdaQc0%>382%S-}$jE2u}shU^r}$j*sV#M85qv8zQY!DuoDA z@4|aBW$Y1uXL}`UlTDi*Oq+U6-qEjM>=n=G!ATXO7jNnnZ#|5$SGxHy=AOCz5C-4) z&Vx<9yb}Y%@Awj2`{RS5_|a$V1>g`7Y`;k_J)CUDI#c1(2)^HRY(i|nnTjKE*L(KN z)-Ia_`QT_b!SR^~=-iDsVerjA+ZVP-fZ}TOeS@eIHXUli@pNpHmy|T9NjCzwU7VCU zVO)?2-Md(0ZIriwNe0SCv?IO3`I+80IQgQ)&SK+T!Z0Iv23bAC!NI-`hck^CI;v>| zL|s{gJya%wJF*zbl&NT(NMb(ZK1D7B`0O*^rqQEfL`jrP6|h21T*+OE#&`>8CtE9G zFN--rA&6_*Reo1)vJzjZF7Xt2VlhCJ|HASYClZ`^MNcOT#FCNiAg)+jakO+`48a@~ z>#7AVRhVvYkC%jtU>r?G++m~ylmn)fFPWBp0W7y6J zfwA`(zCdSU-P$tm<)+1m8VZcB)mD6;jz3JF9ZEqX)+K|f(?P{Bso)OHbe6*YrMZhHV5chk zJYui_i@*~Lon2|c9}jKkSAe9l&;nD0MhE6d1O-Dsz=Mfz2cCovdO;V&!ysJHSpqB! zjY*D~O#1*`_+p6Au}L`z_jjCiWl5m}%87K8sW#%>jA1(@d{C*xO>jdfRIoBeUv$A9 z8l(4aY;E`-_WS+b7gvJyMaO&KJzb7FH!5SaGDcrjWAuN|7^DAG8KW=N^o0HQ-Kzf2 zk>`KEi=6tkFEUgFlPX*QoO$ZeWHNM~lstlo2cH50ybvFo8&I^>5R1x+Z9tff)kXu5 z7bXkB0)*G@wlV_^Z{-n!ehU*kHPalA>-x?zl})hPXR$ARgi#;PvM%N`WnnbVaqMb_aW z%BSB&n}$V!dECd5GMCp}2~At1^V%|@;cW7Nwx$i%d9CqaEERm8-_~@b`n;x0#|~&{ z%G#aRP$?My`Hj6a7w9&0#?*#j^dNLmJUcVmto_>B_Q@@1XyjZSg*`L1S`ks7KV*EZ zP$jA>Yo?2(=rtowAS~KRt-lJ(!CVsSjsLDy(ZA=nVXuZR)AU-3*{x?2LD1&CsqMHFp>x$xl5igh-?zvskSqBS5$ z6gII0b%ZehxIsgBEK!V{b|ltA9;REbO-rIY9$J0K9soPPcta6&-HhA;>I>s{ag((e zZ;E>5D9`~j9=1=Ik%pfEwBl0K>a_<$2C<{Cdr&1iLkMwUc}xHfkBDKw!_Bd8BTY8M z*kP0H&Psf{e1}bkq0sVk(;}@Ieh0kapn`9VYRJ04GEGQCfueG+)kBymYw$9de{o3z zA_y3!*Hla?-2-4FT}0Zts0z(3mcw&*7_C&`0`>4=29g5@U=+52&;|t5I81cj%=|!&=swNS&e0Mvl@7 z>(-zuz04uiD^|V82L!;#1MT(YV_Q--LT@JW7(U-k32o397^^Ec0@xGx_yW}klgrCcbd9CxXr*yMUFms_Ibe}v z^FbxsIXglj9S4Oqun9{?r)7b_e+hypclwmIBURPWLU%@sWfl|_OQq^=St5X;?6O6`~mXHC4!Zw zQJb-b%>nCx$)GxT3b@7WLh6!E_IoE$Qt2g4bp*^JL$P;{bGr^o_<#t?D zsz!5KzkbeVBUwL!-)iGYWJ9?M)p0T?WY zT?tj9wQlGv9DPVKi8u}*V1BhtDh`e{8wM4JrUyt2qALwKtcV`N=5W4=M4^s8J|Llo zKl2S(50sfs;o%>1=a^vbb9Sc(I?TxAIx@6PgU^VpBsyArKe-7o5ro5LyHIA_+>BaS zeNtf^wok}NDqej4*l}ngM9UlVF~b1?d=_cJ7&JZr2Jn@bvLx%W;#4vCd@A88LBd)O zB)n{>G+da(7f&ac$@qZpU{#&c~KP&#v z0%k)r0IWti+E(@7tNQO%{dX8hu=SDW0*C;nFD+r)Lr~K?VJL&%kkElJ7w}EM!LYGl zHS5Q`6gY5FKl41A#4{_vGhlTxac5qW(h}@B!s~39047XXP#qxA2q-Ty>g>5xl}km{ zra7g`mtm_R_Ty^pXJ=OL-q~JzynW`Xbq4m^{S4@_Lqk^(U*Hkwm0&h7P>CWV(3bAX_} zp5DRbrF%A4sIN2s=YH8bnqT30IxcrqHF}}%gGN^&7%m-*ZsdpF;Yon)Ak9268rA}P zuaQQ(fO1E--Wa~sfNvrheLliZNUd{5!|$(xsKtkK4DfATmzEi2!&AV4zK%eyW&ggq z;@!Kx{lva@FTLILo;+nD1X?}!NityMa>vyfCacYzNirS?YD+a#d=|+R0){O>_?89$)tZ$jf>4&Ywa7YXP<@-wvM5R_O%pwJnhzgG5kTJ z(H*n^VtGyjB&I#Y_&~&WvZl}v41bG}6!^e4P@5LfKMfsVmdG5Og!ILL8lKiyRcrL~ z;b}>0L!kU9H>rl-CIu$S3&g>UxivhMs6B)a&R+PHO|(C`{yWEqf$xA^n;qAF^%(em z>^^eeC;Ohe<-YlvFz~%xYZ3O}rHcRI$nC$Y75~GFDD9B5DX8ge8)IylJQha4GpyC{ zZy?tMajsIAdoJRsFxJ{$Ex9&tz0r z_5Z5+e^vcIWWm$9T-EuF{960T# zpM_5r@AfvD>SnZJSD^~?)5t4y=;fnB|7j+g{6CkY$t!f|#i2vMe?G4AzmFXMxmM+W zSNY$?$*9WzF6h$?3oI#6!M|oBZ573|h>*2$lt#{~u`Co1gTkj)uWg{JJktA^Bx|kh z+04{SuGZdrFaM=Tr=FL4yCu~FZ~~7NIg^m;i%eg>9W@Whha~mG`|up`iqO0skS@D| zlx3di%k636DdXNF=OPKG+F*JF6O=SPH|ce;$Ru8z9nI3*OyX_Or-iD|3J2hM6524G zCT(}SD9eB`77O_pA8-D*hk!<%%SPD*nHU|3^)PqB3Mv|FNq7_*&F| zEXV(YARl#DK>61OPSmx-aXR`o@T%#nLbtajgVAdr$~Zs2gn`%O;D4gK8T~{FonHqYG)o_8ZnrIYu(#dE???(Z$yFL++4Z* z+2xPEc(T&Fj$DJc9XCQH7E+$MKN_A|mEVet1NRKG-oMudiGAWC3!9})M#EF!umUwd zIMPraSnZ8&ofw|>qYilYo5|?T@NHHL0`uT#cn)=CL))vTgD$x0nddN_?9eFg{o!QN z!=`B?&wY@dxV7OMRgU_vP9yejj#G~M|Cvb(_`g+Jz`;h_5#9m&@4r>?|3_~B{bLpX zU&a5!iAnyjD*ivOpFI%X++69X=3r&Gj8CC5asm4ZTX?QBZ zLd^<6IFTSd6E*F@7BGOHSvx(e`2Q;Yzl#4?F&N1emTSvH-9RBPN5})B46uAqLJ&VP zNCP1VxQq}0DV?VI+fgYGG>BN#EaRC8!C6mra2INcN@NI&qJr-nRT)|3v6I1KkaCZx z+=eoMDC=dy4|Js~P&3*n(`ap_8Qz7Onl|D`7+1ikbOdprZfC@jhY0jQODLU)TTq${ z!4P@1Gbu{u<7q^I&;^fjVd!{l%8ub?RKqhx$7Bc4#lvum6LmtA1H}7ui!-|vLqjC^ z0)xTb0qV3OZlx%?xQU|9*nvud9B*8-foHEn0$zX(4?;%RHEO%+#txPvH(Mji+<*q# z8>-D>3uK}fUpF^iU1nAas_k;$#O{op7CRyeI>tKQJ&)3<;{RX!`2T{vUH~1GFf}O7 zf~A!QF>sRnfRzq?WNC-k)>SZp1EF{?3nt1Evkv$#Okz9 zQ80}tO|G@+s}wEw5CRwmF_GJ&pT_F#Ft>s=4_0=vEXQIxjrjkVkND-)P&qH0}D*qd<&TTu{ zwce}pzpMQ3D*s!SLY4o$AHUkG;a`1TM)~t`geuMan5UVqJy_2+?a+R*eL3w!eShVd i8{Yf0vx~CnRbBxq+r2L329A}V*O~G2w!RsD-hT$o1%-3~ delta 9 QcmZ3#nPvVY#trhG02LDhu>b%7 diff --git a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir index 38cc4a0bc5e..b905e33e5e6 100644 --- a/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir +++ b/src/promptflow-recording/recordings/local/evals.node_cache.shelve.dir @@ -2,3 +2,16 @@ 'e5a1c88060db56a1f098ee4343ddca0bb97fa620', (4096, 5484) '34501a2950464ae7eece224e06278dde3addcfb0', (9728, 4814) '5cd313845b5581923f342e6fee8c7247b765e0f4', (14848, 3871) +'70d88fb0b7055fa9ea40c143d5345f08396a3be6', (18944, 4401) +'fd5857711c2df768f84124498368d23825c6528b', (23552, 3258) +'be37232f5439067f213f60079bbba554f0e47bb1', (27136, 3399) +'ee9677726bf4d3de1f85584b307f16cb5e5835cc', (30720, 5399) +'c0466ada98de98160d8e0ad0f298604c00122fc4', (36352, 3856) +'30ab02811f2cedb40738b269f6fa858cb9f582d6', (40448, 3251) +'103c23d4c120e2e82c265bb192109a5f415355a8', (44032, 3392) +'2b6bddd86da3dde9feaf33681897e13a2089f657', (47616, 4446) +'6c81297832846ab56b385dc8e4c0de5eec3dda76', (52224, 3912) +'8cc4b0a46e2da4c79a32e0232c0937f1afb0f630', (56320, 3415) +'0658d24d96833aa5acf52b87d34ab8220a5b2669', (59904, 4551) +'5f17c1fae1329c6d6823c0c59e4a66fd6ee42691', (64512, 3274) +'79ea9bacda0ffb42811f24c5b0293a9744824daf', (68096, 4015) From 094e1217acbc26a41ce267dbfcd0abdaa7b01f79 Mon Sep 17 00:00:00 2001 From: Billy Hu Date: Mon, 22 Apr 2024 08:13:50 -0700 Subject: [PATCH 13/14] Fix replay mode is not used in ci pipeline due to recording package import error (#2912) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .github/workflows/promptflow-evals-e2e-test.yml | 5 ++--- .github/workflows/promptflow-evals-unit-test.yml | 5 ++--- src/promptflow-evals/tests/evals/conftest.py | 10 +++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/promptflow-evals-e2e-test.yml b/.github/workflows/promptflow-evals-e2e-test.yml index 63ff9b1a699..b1e2ea80a1b 100644 --- a/.github/workflows/promptflow-evals-e2e-test.yml +++ b/.github/workflows/promptflow-evals-e2e-test.yml @@ -12,7 +12,6 @@ on: env: IS_IN_CI_PIPELINE: "true" WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals - RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording jobs: build: @@ -70,8 +69,8 @@ jobs: run: poetry install --only test working-directory: ${{ env.WORKING_DIRECTORY }} - name: install recording - run: poetry install - working-directory: ${{ env.RECORD_DIRECTORY }} + run: poetry run pip install -e ../promptflow-recording + working-directory: ${{ env.WORKING_DIRECTORY }} - name: generate end-to-end test config from secret run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json working-directory: ${{ env.WORKING_DIRECTORY }} diff --git a/.github/workflows/promptflow-evals-unit-test.yml b/.github/workflows/promptflow-evals-unit-test.yml index 9f0575b7ac6..89c1be9a0d2 100644 --- a/.github/workflows/promptflow-evals-unit-test.yml +++ b/.github/workflows/promptflow-evals-unit-test.yml @@ -12,7 +12,6 @@ on: env: IS_IN_CI_PIPELINE: "true" WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals - RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording jobs: build: @@ -66,8 +65,8 @@ jobs: run: poetry install --only test working-directory: ${{ env.WORKING_DIRECTORY }} - name: install recording - run: poetry install - working-directory: ${{ env.RECORD_DIRECTORY }} + run: poetry run pip install -e ../promptflow-recording + working-directory: ${{ env.WORKING_DIRECTORY }} - name: run unit tests run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml working-directory: ${{ env.WORKING_DIRECTORY }} diff --git a/src/promptflow-evals/tests/evals/conftest.py b/src/promptflow-evals/tests/evals/conftest.py index bf24e419da9..e1f3ad70c44 100644 --- a/src/promptflow-evals/tests/evals/conftest.py +++ b/src/promptflow-evals/tests/evals/conftest.py @@ -15,7 +15,9 @@ try: from promptflow.recording.local import recording_array_reset from promptflow.recording.record_mode import is_in_ci_pipeline, is_live, is_record, is_replay -except ImportError: +except ImportError as e: + print(f"Failed to import promptflow-recording: {e}") + # Run test in empty mode if promptflow-recording is not installed def recording_array_reset(): pass @@ -44,6 +46,12 @@ def pytest_configure(): pytest.is_replay = is_replay() pytest.is_in_ci_pipeline = is_in_ci_pipeline() + print() + print(f"pytest.is_live: {pytest.is_live}") + print(f"pytest.is_record: {pytest.is_record}") + print(f"pytest.is_replay: {pytest.is_replay}") + print(f"pytest.is_in_ci_pipeline: {pytest.is_in_ci_pipeline}") + @pytest.fixture def mock_model_config() -> dict: From 63fafe411c0bf5fc523fdbb5c268610688bc6450 Mon Sep 17 00:00:00 2001 From: nick863 <30440255+nick863@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:08:01 -0700 Subject: [PATCH 14/14] Remove the versions.txt and add the comment to pyproject.toml. (#2913) # Description We set the version of promptflow evals according to release parameter of the release pipeline. version.txt is not needed anymore. I also have added the comment to pyproject.toml file. # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes].** - [x] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [x] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [x] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [x] Title of the pull request is clear and informative. - [x] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [x] Pull request includes test coverage for the included changes. --- src/promptflow-evals/promptflow/version.txt | 1 - src/promptflow-evals/pyproject.toml | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 src/promptflow-evals/promptflow/version.txt diff --git a/src/promptflow-evals/promptflow/version.txt b/src/promptflow-evals/promptflow/version.txt deleted file mode 100644 index 901e5110b2e..00000000000 --- a/src/promptflow-evals/promptflow/version.txt +++ /dev/null @@ -1 +0,0 @@ -VERSION = "0.0.1" diff --git a/src/promptflow-evals/pyproject.toml b/src/promptflow-evals/pyproject.toml index d482a648868..1d79338ab34 100644 --- a/src/promptflow-evals/pyproject.toml +++ b/src/promptflow-evals/pyproject.toml @@ -5,7 +5,10 @@ build-backend = "poetry.core.masonry.api" # poetry [tool.poetry] name = "promptflow-evals" -version = "0.1.0.dev0" +# This version does not need to be changed, because it is set by the release pipeline. +# See build-publish-local-wheel-evals.yaml pipeline definition +# in Vienna PromptFlow repository for reference. +version = "0.2.0.dev0" description = "Prompt flow evals" license = "MIT" authors = [