From f9a7a851f758db52fcc8660ef00d3ce53ab2328d Mon Sep 17 00:00:00 2001 From: chenyang Date: Mon, 6 May 2024 11:26:14 +0800 Subject: [PATCH 1/4] fix config override tracing.destination (#3057) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. 1. azureml full ![image](https://github.com/microsoft/promptflow/assets/23182548/33ab9606-478c-4782-909f-bb36311bdc09) ![image](https://github.com/microsoft/promptflow/assets/23182548/2dbebc6c-1b18-40ed-9e19-968e8fdbd8eb) 2. azureml ![image](https://github.com/microsoft/promptflow/assets/23182548/44ffee1b-0cb7-4adf-ade5-d5b1d36189f4) # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [x] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [x] Pull request includes test coverage for the included changes. --- .../_sdk/_orchestrator/run_submitter.py | 9 ++++----- .../promptflow/_sdk/_pf_client.py | 6 +++--- .../promptflow/_sdk/_tracing.py | 19 +++++++++++++------ .../promptflow/_sdk/entities/_run.py | 8 +++----- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py b/src/promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py index 982489a71222..5df99665b18c 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py +++ b/src/promptflow-devkit/promptflow/_sdk/_orchestrator/run_submitter.py @@ -23,7 +23,6 @@ from promptflow.tracing._operation_context import OperationContext from promptflow.tracing._start_trace import is_collection_writeable, start_trace -from .._configuration import Configuration from .._load_functions import load_flow from ..entities._flows import FlexFlow from .utils import SubmitterHelper, variant_overwrite_context @@ -36,7 +35,7 @@ class RunSubmitter: def __init__(self, client): self._client = client - self._config = Configuration(overrides=self._client._config) + self._config = self._client._config self.run_operations = self._client.runs def submit(self, run: Run, stream=False, **kwargs): @@ -96,13 +95,13 @@ def _run_bulk(self, run: Run, stream=False, **kwargs): # pass with internal parameter `_collection` start_trace( attributes=attributes, - run=run.name, + run=run, _collection=collection_for_run, path=flow_path, ) else: logger.debug("trace collection is protected, will honor existing collection.") - start_trace(attributes=attributes, run=run.name, path=flow_path) + start_trace(attributes=attributes, run=run, path=flow_path) self._validate_inputs(run=run) @@ -276,7 +275,7 @@ def _upload_run_to_cloud(cls, run: Run): from promptflow._sdk._tracing import _get_ws_triad_from_pf_config from promptflow.azure._cli._utils import _get_azure_pf_client - ws_triad = _get_ws_triad_from_pf_config(path=run._get_flow_dir().resolve()) + ws_triad = _get_ws_triad_from_pf_config(path=run._get_flow_dir().resolve(), config=run._config) pf = _get_azure_pf_client( subscription_id=ws_triad.subscription_id, resource_group=ws_triad.resource_group_name, diff --git a/src/promptflow-devkit/promptflow/_sdk/_pf_client.py b/src/promptflow-devkit/promptflow/_sdk/_pf_client.py index bb5c63f5af48..8da79af330f7 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_pf_client.py +++ b/src/promptflow-devkit/promptflow/_sdk/_pf_client.py @@ -44,7 +44,7 @@ def __init__(self, **kwargs): # when this is set, telemetry from this client will use this user agent and ignore the one from OperationContext self._user_agent_override = kwargs.pop(USER_AGENT_OVERRIDE_KEY, None) self._connection_provider = kwargs.pop("connection_provider", None) - self._config = kwargs.get("config", None) or {} + self._config = Configuration(overrides=kwargs.get("config", None) or {}) # The credential is used as an option to override # DefaultAzureCredential when using workspace connection provider self._credential = kwargs.get("credential", None) @@ -212,7 +212,7 @@ def _run( connections=connections, environment_variables=environment_variables, properties=properties, - config=Configuration(overrides=self._config), + config=self._config, init=init, dynamic_callable=dynamic_callable, ) @@ -372,7 +372,7 @@ def tools(self) -> ToolOperations: def _ensure_connection_provider(self) -> str: if not self._connection_provider: # Get a copy with config override instead of the config instance - self._connection_provider = Configuration(overrides=self._config).get_connection_provider() + self._connection_provider = self._config.get_connection_provider() logger.debug("PFClient connection provider: %s, setting to env.", self._connection_provider) from promptflow.core._connection_provider._connection_provider import ConnectionProvider diff --git a/src/promptflow-devkit/promptflow/_sdk/_tracing.py b/src/promptflow-devkit/promptflow/_sdk/_tracing.py index a6fa980fdb1a..0e21ceec7254 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_tracing.py +++ b/src/promptflow-devkit/promptflow/_sdk/_tracing.py @@ -56,6 +56,7 @@ extract_workspace_triad_from_trace_provider, ) from promptflow._sdk._utilities.tracing_utils import get_workspace_kind, parse_kv_from_pb_attribute, parse_protobuf_span +from promptflow._sdk.entities import Run from promptflow._utils.logger_utils import get_cli_sdk_logger from promptflow._utils.thread_utils import ThreadWithContextVars from promptflow.tracing._integrations._openai_injector import inject_openai_api @@ -195,14 +196,15 @@ def _invoke_pf_svc() -> str: return port -def _get_ws_triad_from_pf_config(path: typing.Optional[Path]) -> typing.Optional[AzureMLWorkspaceTriad]: +def _get_ws_triad_from_pf_config(path: typing.Optional[Path], config=None) -> typing.Optional[AzureMLWorkspaceTriad]: from promptflow._sdk._configuration import Configuration - config = Configuration.get_instance().get_trace_destination(path=path) - _logger.info("resolved tracing.trace.destination: %s", config) - if not TraceDestinationConfig.need_to_export_to_azure(config): + config = config or Configuration.get_instance() + trace_destination = config.get_trace_destination(path=path) + _logger.info("resolved tracing.trace.destination: %s", trace_destination) + if not TraceDestinationConfig.need_to_export_to_azure(trace_destination): return None - return extract_workspace_triad_from_trace_provider(config) + return extract_workspace_triad_from_trace_provider(trace_destination) # priority: run > experiment > collection @@ -368,6 +370,11 @@ def start_trace_with_devkit(collection: str, **kwargs: typing.Any) -> None: _logger.debug("kwargs: %s", kwargs) attrs = kwargs.get("attributes", None) run = kwargs.get("run", None) + if isinstance(run, Run): + run_config = run._config + run = run.name + else: + run_config = None path = kwargs.get("path", None) # honor and set attributes if user has specified @@ -395,7 +402,7 @@ def start_trace_with_devkit(collection: str, **kwargs: typing.Any) -> None: # local to cloud feature _logger.debug("start_trace_with_devkit.path(from kwargs): %s", path) - ws_triad = _get_ws_triad_from_pf_config(path=path) + ws_triad = _get_ws_triad_from_pf_config(path=path, config=run_config) is_azure_ext_installed = _is_azure_ext_installed() if ws_triad is not None and not is_azure_ext_installed: warning_msg = ( diff --git a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py index 2bf42c299344..8196634f1e72 100644 --- a/src/promptflow-devkit/promptflow/_sdk/entities/_run.py +++ b/src/promptflow-devkit/promptflow/_sdk/entities/_run.py @@ -172,15 +172,14 @@ def __init__( # default run name: flow directory name + timestamp self.name = name or self._generate_run_name() experiment_name = kwargs.get("experiment_name", None) + self._config: Configuration = kwargs.get("config", Configuration.get_instance()) if self._run_source == RunInfoSources.LOCAL and not self._use_remote_flow: self.flow = Path(str(flow)).resolve().absolute() flow_dir = self._get_flow_dir() # sanitize flow_dir to avoid invalid experiment name self._experiment_name = _sanitize_python_variable_name(flow_dir.name) self._lineage_id = get_flow_lineage_id(flow_dir=flow_dir) - self._output_path = Path( - kwargs.get("output_path", self._generate_output_path(config=kwargs.get("config", None))) - ) + self._output_path = Path(kwargs.get("output_path", self._generate_output_path(config=self._config))) if is_prompty_flow(self.flow): self._flow_name = Path(self.flow).stem else: @@ -786,8 +785,7 @@ def _validate_for_run_create_operation(self): if not self.run and not self.data: raise UserErrorException("at least one of data or run must be provided") - def _generate_output_path(self, config: Optional[Configuration]) -> Path: - config = config or Configuration.get_instance() + def _generate_output_path(self, config: Configuration) -> Path: path = config.get_run_output_path() if path is None: path = HOME_PROMPT_FLOW_DIR / ".runs" From f8c7738199e358c34ad2021836ee2cab21b0a778 Mon Sep 17 00:00:00 2001 From: Ge Gao <49388944+dorisjoy@users.noreply.github.com> Date: Mon, 6 May 2024 11:44:28 +0800 Subject: [PATCH 2/4] Add environment into flow.dag.yaml for eval-chat-math flow (#3082) Add environment into flow.dag.yaml for eval-chat-math flow: run link in portal: https://ml.azure.com/prompts/bulkrun/eval_chat_math_variant_0_20240430_143934_923359/details?wsid=/subscriptions/96aede12-2f73-41cb-b983-6d11a904839b/resourcegroups/promptflow/providers/Microsoft.MachineLearningServices/workspaces/promptflow-eastus-dev&tid=72f988bf-86f1-41af-91ab-2d7cd011db47 Co-authored-by: Ge Gao --- examples/flows/evaluation/eval-chat-math/flow.dag.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/flows/evaluation/eval-chat-math/flow.dag.yaml b/examples/flows/evaluation/eval-chat-math/flow.dag.yaml index d7e9a591ae3f..ff302d88b49b 100644 --- a/examples/flows/evaluation/eval-chat-math/flow.dag.yaml +++ b/examples/flows/evaluation/eval-chat-math/flow.dag.yaml @@ -1,3 +1,4 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json inputs: groundtruth: type: string @@ -31,4 +32,5 @@ nodes: aggregation: true use_variants: false node_variants: {} -$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json +environment: + python_requirements_txt: requirements.txt From c8f0e8c6b412b04660d7bd73004357ee1840bb53 Mon Sep 17 00:00:00 2001 From: Honglin Date: Mon, 6 May 2024 12:51:49 +0800 Subject: [PATCH 3/4] [CI] Update link check white list (#3111) # Description Please add an informative description that covers that changes made by the pull request and link all relevant issues. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- scripts/docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/docs/conf.py b/scripts/docs/conf.py index cb1be38a6957..80b8e2a48651 100644 --- a/scripts/docs/conf.py +++ b/scripts/docs/conf.py @@ -59,6 +59,8 @@ # Options for the linkcheck builder linkcheck_ignore = [ + # openai related sites blocks the IP of the CI server. + r"https://openai\.com/", r"https://platform\.openai\.com/", r"https://help\.openai\.com/", # These are used in card links, for example 'xx.html', .md can't be resolved. From 1607cc71fe8ce603d20723758ffcc3e2045281b9 Mon Sep 17 00:00:00 2001 From: riddle xu Date: Mon, 6 May 2024 14:02:02 +0800 Subject: [PATCH 4/4] [Internal] Make runtime cloud trace also async (#3081) # Description Currently, the cloud trace in runtime is in sync mode. This seems unnecessary. The local and runtime should use the same async pattern. # All Promptflow Contribution checklist: - [ ] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [ ] Title of the pull request is clear and informative. - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. Co-authored-by: Yangtong Xu --- .../promptflow/_sdk/_tracing.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/promptflow-devkit/promptflow/_sdk/_tracing.py b/src/promptflow-devkit/promptflow/_sdk/_tracing.py index 0e21ceec7254..2a1a0b69f37f 100644 --- a/src/promptflow-devkit/promptflow/_sdk/_tracing.py +++ b/src/promptflow-devkit/promptflow/_sdk/_tracing.py @@ -609,17 +609,11 @@ def process_otlp_trace_request( else: all_spans.append(span) - if cloud_trace_only: - # If we only trace to cloud, we should make sure the data writing is success before return. - _try_write_trace_to_cosmosdb( - all_spans, get_created_by_info_with_cache, logger, get_credential, is_cloud_trace=True - ) - else: - # Create a new thread to write trace to cosmosdb to avoid blocking the main thread - ThreadWithContextVars( - target=_try_write_trace_to_cosmosdb, - args=(all_spans, get_created_by_info_with_cache, logger, get_credential, False), - ).start() + # Create a new thread to write trace to cosmosdb to avoid blocking the main thread + ThreadWithContextVars( + target=_try_write_trace_to_cosmosdb, + args=(all_spans, get_created_by_info_with_cache, logger, get_credential, cloud_trace_only), + ).start() return