OAI v2 examples (#2710)

* oaiv2 sdk example * modified workflows and notebooks * removed all cell outputs * removed old files * changed cron param * reverted cron param * black format * Added cli examples for new finetune pipeline component (#2711) * Added cli examples for new finetune pipeline component * Added screenshots for cli examples * added and renamed cli workflows * directory fixes * Vvatsalya/fix cli oai v2 workflow (#2717) * setting location as ncus * set in setup-cli step * new init and setup script for oai v2 * correcting syntax for init sh * fix init oai v2 script * fix * fix 1 * fix 2 * change training dataset name in cli oai v2 example * sdk workflow region change * use oai as suffix in ws name * added oai v1 and v2 in readme for cli and sdk * adding install reqs for sdk --------- Co-authored-by: Vishal Vatsalya <[email protected]> Co-authored-by: Vishal Vatsalya <[email protected]> Co-authored-by: Ayush Mishra <[email protected]>
Azure · Oct 10, 2023 · a796d3a · a796d3a
1 parent 321bb44
commit a796d3a
Show file tree

Hide file tree

Showing 30 changed files with 1,526 additions and 370 deletions.
diff --git a/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml b/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml
@@ -3,7 +3,7 @@
 # Any manual changes to this file may cause incorrect behavior.
 # Any manual changes will be overwritten if the code is regenerated.
 
-name: cli-foundation-models-azure_openai-openai_completions_finetune
+name: cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune
 # This file is created by readme.py.
 # Please do not edit directly.
 on:
@@ -14,8 +14,8 @@ on:
     branches:
       - main
     paths:
-      - cli/foundation-models/azure_openai/openai_completions_finetune/**
-      - .github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune.yml
+      - cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune/**
+      - .github/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml
       - infra/bootstrapping/**
       - cli/run-pipeline-jobs.sh
       - cli/setup.sh
@@ -52,5 +52,5 @@ jobs:
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           bash -x ../dataset-create.sh
-          bash -x ../../../run-job.sh openai_completions_finetune_spec.yaml
-      working-directory: cli/foundation-models/azure_openai/openai_completions_finetune
+          bash -x ../../../../run-job.sh openai_completions_finetune_spec.yaml
+      working-directory: cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune
diff --git a/.github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml b/.github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml
@@ -0,0 +1,56 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune
+# This file is created by readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 2/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune_pipeline/**
+      - .github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml
+      - infra/bootstrapping/**
+      - cli/run-pipeline-jobs.sh
+      - cli/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: check out repo
+        uses: actions/checkout@v2
+      - name: azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{secrets.AZUREML_CREDENTIALS}}
+      - name: bootstrap resources
+        run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap_oai_v2.sh
+        working-directory: infra/bootstrapping
+        continue-on-error: false
+      - name: setup-cli
+        run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash setup_oai_v2.sh
+        working-directory: cli
+        continue-on-error: true
+      - name: run job
+        run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          bash -x ../dataset-create.sh
+          bash -x ../../../../run-job.sh openai_completions_finetune_pipeline_spec.yaml
+        working-directory: cli/foundation-models/azure_openai/oai-v2/openai_completions_finetune_pipeline
diff --git a/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml b/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml
@@ -1,75 +1,75 @@
-# This code is autogenerated.
-# Code is generated by running custom script: python3 readme.py
-# Any manual changes to this file may cause incorrect behavior.
-# Any manual changes will be overwritten if the code is regenerated.
-
-name: sdk-foundation-models-azure_openai-openai_completions_finetune
-# This file is created by sdk/python/readme.py.
-# Please do not edit directly.
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "23 2/12 * * *"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - sdk/python/foundation-models/azure_openai/**
-      - .github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune.yml
-      - sdk/python/dev-requirements.txt
-      - infra/bootstrapping/**
-      - sdk/python/setup.sh
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: check out repo
-      uses: actions/checkout@v2
-    - name: setup python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.8"
-    - name: pip install notebook reqs
-      run: pip install -r sdk/python/dev-requirements.txt
-    - name: azure login
-      uses: azure/login@v1
-      with:
-        creds: ${{secrets.AZUREML_CREDENTIALS}}
-    - name: bootstrap resources
-      run: |
-          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
-          bash bootstrap.sh
-      working-directory: infra/bootstrapping
-      continue-on-error: false
-    - name: setup SDK
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash setup.sh
-      working-directory: sdk/python
-      continue-on-error: true
-    - name: setup-cli
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash setup.sh
-      working-directory: cli
-      continue-on-error: true
-    - name: run foundation-models/azure_openai/openai_completions_finetune.ipynb
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
-          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_completions_finetune.ipynb";
-          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
-          papermill -k python openai_completions_finetune.ipynb openai_completions_finetune.output.ipynb
-      working-directory: sdk/python/foundation-models/azure_openai
-    - name: upload notebook's working folder as an artifact
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v2
-      with:
-        name: openai_completions_finetune
-        path: sdk/python/foundation-models/azure_openai
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 2/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/foundation-models/azure_openai/oai-v1/**
+      - .github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml
+      - sdk/python/dev-requirements.txt
+      - infra/bootstrapping/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run foundation-models/azure_openai/oai-v1/openai_completions_finetune.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_completions_finetune.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python openai_completions_finetune.ipynb openai_completions_finetune.output.ipynb
+      working-directory: sdk/python/foundation-models/azure_openai/oai-v1
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: openai_completions_finetune
+        path: sdk/python/foundation-models/azure_openai/oai-v1
diff --git a/...hub/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml b/...hub/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml
@@ -0,0 +1,75 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "5 11/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/foundation-models/azure_openai/oai-v2/**
+      - .github/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml
+      - sdk/python/dev-requirements.txt
+      - infra/bootstrapping/**
+      - sdk/python/setup_oai_v2.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap_oai_v2.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash setup_oai_v2.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash setup_oai_v2.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run foundation-models/azure_openai/oai-v2/openai_chat_finetune_pipeline.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_chat_finetune_pipeline.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python openai_chat_finetune_pipeline.ipynb openai_chat_finetune_pipeline.output.ipynb
+      working-directory: sdk/python/foundation-models/azure_openai/oai-v2
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: openai_chat_finetune_pipeline
+        path: sdk/python/foundation-models/azure_openai/oai-v2
diff --git a/cli/README.md b/cli/README.md
@@ -212,7 +212,8 @@ path|status|description
 [jobs/parallel/1a_oj_sales_prediction/pipeline.yml](jobs/parallel/1a_oj_sales_prediction/pipeline.yml)|[![jobs/parallel/1a_oj_sales_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline.yml)|The hello world pipeline job with partition by key
 [jobs/parallel/2a_iris_batch_prediction/pipeline.yml](jobs/parallel/2a_iris_batch_prediction/pipeline.yml)|[![jobs/parallel/2a_iris_batch_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline.yml)|The hello world pipeline job with inline parallel job
 [jobs/parallel/3a_mnist_batch_identification/pipeline.yml](jobs/parallel/3a_mnist_batch_identification/pipeline.yml)|[![jobs/parallel/3a_mnist_batch_identification/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline.yml)|The hello world pipeline job with inline parallel job
-[cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml](cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml)|[![cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml](.github\workflows\cli-foundation-models-azure_openai-openai_completions_finetune/badge.svg?branch=main)](.github\workflows\cli-foundation-models-azure_openai-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job
+[cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml](cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml)|[![cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune\openai_completions_finetune_spec.yaml](https://github.com/Azure/azureml-examples/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job
+[cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml](cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml)|[![cli\foundation-models\azure_openai\oai-v2\openai_completions_finetune_pipeline\openai_completions_finetune_pipeline_spec.yaml](https://github.com/Azure/azureml-examples/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job
 
 
 **Endpoints** ([endpoints](endpoints))

diff --git a/...tion-models/azure_openai/data/train.jsonl → ...dels/azure_openai/oai-v1/data/train.jsonl b/...tion-models/azure_openai/data/train.jsonl → ...dels/azure_openai/oai-v1/data/train.jsonl
diff --git a/...ion-models/azure_openai/dataset-create.sh → ...els/azure_openai/oai-v1/dataset-create.sh b/...ion-models/azure_openai/dataset-create.sh → ...els/azure_openai/oai-v1/dataset-create.sh
diff --git a/...n-models/azure_openai/dataset-create.yaml → ...s/azure_openai/oai-v1/dataset-create.yaml b/...n-models/azure_openai/dataset-create.yaml → ...s/azure_openai/oai-v1/dataset-create.yaml
diff --git a/...ns_finetune/images/pipeline_completed.png → ...ns_finetune/images/pipeline_completed.png b/...ns_finetune/images/pipeline_completed.png → ...ns_finetune/images/pipeline_completed.png
diff --git a/...ions_finetune/images/pipeline_running.png → ...ions_finetune/images/pipeline_running.png b/...ions_finetune/images/pipeline_running.png → ...ions_finetune/images/pipeline_running.png
diff --git a/...ions_finetune/images/registered_model.png → ...ions_finetune/images/registered_model.png b/...ions_finetune/images/registered_model.png → ...ions_finetune/images/registered_model.png
diff --git a/...s_finetune/openai_completions_finetune.md → ...s_finetune/openai_completions_finetune.md b/...s_finetune/openai_completions_finetune.md → ...s_finetune/openai_completions_finetune.md
@@ -2,7 +2,7 @@
 
 1. Login to az cli using `az login`
 
-2. Navigate to `cli\foundation-models\azure_openai\openai_completions_finetune`
+2. Navigate to `cli\foundation-models\azure_openai\oai-v1\openai_completions_finetune`
 
 3. Run `az ml job create --subscription <SUBSCRIPTION_ID> --resource-group <RESOURCE_GROUP_NAME> --workspace-name <WORKSPACE_NAME> --file "openai_completions_finetune_spec.yaml"`
 

diff --git a/...une/openai_completions_finetune_spec.yaml → ...une/openai_completions_finetune_spec.yaml b/...une/openai_completions_finetune_spec.yaml → ...une/openai_completions_finetune_spec.yaml