promptflow-release-testing-matrix #136
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: promptflow-release-testing-matrix | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| inputs: | |
| # can leave empty when trigger manually | |
| # GitHub Actions API for trigger does not return workflow run id | |
| # there we reference below Stack Overflow solution: | |
| # https://stackoverflow.com/a/69500478 | |
| # which adds an identifier in workflow run jobs and can be used for filter | |
| id: | |
| description: Identifier for the workflow run | |
| required: false | |
| type: string | |
| env: | |
| testWorkingDirectory: src/promptflow | |
| PYTHONPATH: ${{ github.workspace }}/src/promptflow | |
| IS_IN_CI_PIPELINE: "true" | |
| jobs: | |
| id: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: workflow run id - ${{ inputs.id }} | |
| run: | | |
| echo "workflow run id: ${{ inputs.id }}" | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Python Setup - 3.9 | |
| uses: "./.github/actions/step_create_python_environment" | |
| with: | |
| pythonVersion: 3.9 | |
| - name: Build wheel | |
| uses: "./.github/actions/step_sdk_setup" | |
| with: | |
| setupType: promptflow_with_extra | |
| scriptPath: ${{ env.testWorkingDirectory }} | |
| - name: Upload Wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel | |
| path: | | |
| ${{ github.workspace }}/src/promptflow/dist/*.whl | |
| ${{ github.workspace }}/src/promptflow-tools/dist/*.whl | |
| promptflow_sdk_cli_tests: | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }} | |
| needs: build | |
| env: | |
| PROMPT_FLOW_TEST_MODE: "live" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Display and Set Environment Variables | |
| run: | |
| env | sort >> $GITHUB_OUTPUT | |
| shell: bash -el {0} | |
| - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} | |
| uses: "./.github/actions/step_create_python_environment" | |
| with: | |
| pythonVersion: ${{ matrix.pythonVersion }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheel | |
| path: artifacts | |
| - name: Azure Login | |
| uses: azure/login@v1 | |
| if: env.PROMPT_FLOW_TEST_MODE == 'live' | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Generate Configs | |
| uses: "./.github/actions/step_generate_configs" | |
| if: env.PROMPT_FLOW_TEST_MODE == 'live' | |
| with: | |
| targetFolder: ${{ env.testWorkingDirectory }} | |
| - name: generate test resources placeholder | |
| if: env.PROMPT_FLOW_TEST_MODE != 'live' | |
| shell: pwsh | |
| working-directory: ${{ env.testWorkingDirectory }} | |
| run: | | |
| cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json | |
| - name: Install pf | |
| shell: pwsh | |
| working-directory: artifacts | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt | |
| gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}} | |
| gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} | |
| pip freeze | |
| - name: Run SDK CLI Test | |
| shell: pwsh | |
| working-directory: ${{ env.testWorkingDirectory }} | |
| run: | | |
| python "../../scripts/building/run_coverage_tests.py" ` | |
| -p promptflow ` | |
| -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test ` | |
| -l eastus ` | |
| -m "unittest or e2etest" ` | |
| -o "${{ github.workspace }}/test-results-sdk-cli.xml" ` | |
| --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py | |
| - name: Install pf executable | |
| shell: pwsh | |
| working-directory: artifacts | |
| run: | | |
| Set-PSDebug -Trace 1 | |
| gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executable]"}} | |
| pip freeze | |
| - name: Run SDK CLI Executable Test | |
| shell: pwsh | |
| working-directory: ${{ env.testWorkingDirectory }} | |
| run: | | |
| python "../../scripts/building/run_coverage_tests.py" ` | |
| -p promptflow ` | |
| -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py ` | |
| -l eastus ` | |
| -m "unittest or e2etest" ` | |
| -o "${{ github.workspace }}/test-results-sdk-cli-executable.xml" | |
| - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: promptflow_sdk_cli_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| path: ${{ github.workspace }}/*.xml | |
| # promptflow_sdk_cli_azure_tests: | |
| # needs: build | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # os: [ubuntu-latest, windows-latest, macos-latest] | |
| # pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| # env: | |
| # PROMPT_FLOW_TEST_MODE: "live" | |
| # runs-on: ${{ matrix.os }} | |
| # steps: | |
| # - name: checkout | |
| # uses: actions/checkout@v4 | |
| # - name: Display and Set Environment Variables | |
| # run: | |
| # env | sort >> $GITHUB_OUTPUT | |
| # shell: bash -el {0} | |
| # - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} | |
| # uses: "./.github/actions/step_create_python_environment" | |
| # with: | |
| # pythonVersion: ${{ matrix.pythonVersion }} | |
| # - name: Download Artifacts | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: wheel | |
| # path: artifacts | |
| # - name: Azure Login | |
| # uses: azure/login@v1 | |
| # if: env.PROMPT_FLOW_TEST_MODE == 'live' | |
| # with: | |
| # creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # - name: Generate Configs | |
| # uses: "./.github/actions/step_generate_configs" | |
| # if: env.PROMPT_FLOW_TEST_MODE == 'live' | |
| # with: | |
| # targetFolder: ${{ env.testWorkingDirectory }} | |
| # - name: Install pf azure | |
| # shell: pwsh | |
| # working-directory: artifacts | |
| # run: | | |
| # pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt | |
| # gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} | |
| # gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}} | |
| # pip freeze | |
| # - name: Run SDK CLI Azure Test | |
| # shell: pwsh | |
| # working-directory: ${{ env.testWorkingDirectory }} | |
| # run: | | |
| # python "../../scripts/building/run_coverage_tests.py" ` | |
| # -p promptflow ` | |
| # -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ` | |
| # -l eastus ` | |
| # -m "unittest or e2etest" ` | |
| # -o "${{ github.workspace }}/test-results-sdk-cli-azure.xml" | |
| # - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| # if: ${{ always() }} | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: promptflow_sdk_cli_azure_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| # path: ${{ github.workspace }}/*.xml | |
| # promptflow_executor_tests: | |
| # if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }} | |
| # needs: build | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # os: [ubuntu-latest, windows-latest, macos-latest] | |
| # pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| # runs-on: ${{ matrix.os }} | |
| # steps: | |
| # - name: checkout | |
| # uses: actions/checkout@v4 | |
| # - name: Display and Set Environment Variables | |
| # run: | |
| # env | sort >> $GITHUB_OUTPUT | |
| # shell: bash -el {0} | |
| # - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} | |
| # uses: "./.github/actions/step_create_python_environment" | |
| # with: | |
| # pythonVersion: ${{ matrix.pythonVersion }} | |
| # - name: Download Artifacts | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: wheel | |
| # path: artifacts | |
| # - name: Azure Login | |
| # uses: azure/login@v1 | |
| # with: | |
| # creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # - name: Generate Configs | |
| # uses: "./.github/actions/step_generate_configs" | |
| # with: | |
| # targetFolder: ${{ env.testWorkingDirectory }} | |
| # - name: Install pf | |
| # shell: pwsh | |
| # working-directory: artifacts | |
| # run: | | |
| # pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt | |
| # gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} | |
| # gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} | |
| # pip freeze | |
| # - name: Run Executor Test | |
| # shell: pwsh | |
| # working-directory: ${{ github.workspace }} | |
| # run: | | |
| # pip install langchain | |
| # pip install numexpr | |
| # python scripts/building/run_coverage_tests.py ` | |
| # -p ${{ github.workspace }}/src/promptflow/promptflow ` | |
| # -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests ` | |
| # -l eastus ` | |
| # -m "all" ` | |
| # -o "${{ github.workspace }}/test-results-executor.xml" | |
| # - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| # if: ${{ always() }} | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| # path: ${{ github.workspace }}/*.xml | |
| publish-test-results: | |
| name: "Publish Tests Results" | |
| needs: [ promptflow_sdk_cli_tests] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| contents: read | |
| issues: read | |
| if: always() | |
| steps: | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@v2 | |
| with: | |
| files: "artifacts/**/test-*.xml" |