Merge pull request #271 from Azure-Samples/users/daviwu/ignite2024

Run evaluation from `microsoft/genai-evals` and merge evaluation output

.github/workflows/evaluate.yml

@@ -8,6 +8,11 @@ on:
     branches:
       - main
       - ignite2024
+  pull_request:
+    # Run when pull requests are opened or updated
+    branches:
+      - main
+      - ignite2024
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
@@ -35,6 +40,9 @@ jobs:
       AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
       BING_SEARCH_ENDPOINT: ${{ vars.BING_SEARCH_ENDPOINT }}
       BING_SEARCH_KEY: ${{ secrets.BING_SEARCH_KEY }}
+      EVAL_CONFIG_FILE_PATH: ${{ github.workspace }}/evaluate-config.json
+      GENAI_EVALS_INPUT_FILE_PATH: ${{ github.workspace }}/src/api/evaluate/eval_results.jsonl
+      GENAI_EVALS_OUTPUT_FILE_PATH: ${{ github.workspace }}/src/api/evaluate/genai_evals_data.jsonl
 
     steps:
       - name: checkout repo content
@@ -82,14 +90,42 @@ jobs:
         with:
           name: eval_image_result
           path: ./src/api/evaluate/image_eval_results.jsonl
+
+      - name: Convert to microsoft/genai-evals Data Format
+        run: |
+          python -m src.api.evaluate.genai_evals_convert ${{ env.GENAI_EVALS_INPUT_FILE_PATH }} ${{ env.GENAI_EVALS_OUTPUT_FILE_PATH }}
 
-      - name: GitHub Summary Step
+      - name: Prepare AI evaluation configuration file
+        run: |
+          cat > ${{ env.EVAL_CONFIG_FILE_PATH }}<<EOF
+          {
+            "data": "${{ env.GENAI_EVALS_OUTPUT_FILE_PATH }}",
+            "evaluators": {
+              "coherence": "CoherenceEvaluator",
+              "fluency": "FluencyEvaluator"
+            },
+            "ai_model_configuration": {
+              "type": "azure_openai",
+              "azure_endpoint": "${{ vars.AZURE_OPENAI_ENDPOINT }}",
+              "azure_deployment": "${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}",
+              "api_version": "${{ vars.AZURE_OPENAI_API_VERSION }}"
+            }
+          }
+          EOF
+
+      - name: Text Evaluation Summary
+        id: run-ai-evaluation
+        uses: microsoft/genai-evals@users/daviwu/ignite2024
+        with:          
+          evaluate-configuration: ${{ env.EVAL_CONFIG_FILE_PATH }}
+          show-summary: true
+          show-raw-output: true
+
+      - name: Image Evaluation Summary
         if: ${{ success() }}
         working-directory: ./src/api
         run: |
           echo "" >> $GITHUB_STEP_SUMMARY
 
-          echo "📊 Evaluation Results" >> $GITHUB_STEP_SUMMARY
-          cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY
+          echo "📊 Image Evaluation Results" >> $GITHUB_STEP_SUMMARY
           cat evaluate/image_eval_results.md >> $GITHUB_STEP_SUMMARY
-        

src/api/evaluate/genai_evals_convert.py

@@ -0,0 +1,38 @@
+import json
+import sys
+import uuid
+
+if len(sys.argv) != 3:
+    print("Usage: python genai_evals_convert.py input_file output_file")
+    sys.exit(1)
+else:
+    # TODO: remove this print
+    print("Converting to genai-evals data format...\n", sys.argv[1], "\n", sys.argv[2])
+
+# Get input_file from the first parameters
+input_file = sys.argv[1]
+output_file = sys.argv[2]
+
+with open(input_file, 'r') as f:
+    content = f.read()  # Reads the entire file into a string
+eval_results = json.loads(content)
+
+with open(output_file, 'w') as f:
+    for index, row in enumerate(eval_results["rows"]):
+        description = {
+            "context": {
+                "system-prompt": f"Test {index + 1}",
+            },
+        }
+
+        new_row = {
+            "id": uuid.uuid4().hex,
+            "description": json.dumps(description),
+            "query": row["inputs.query"],
+            "context": row["inputs.context"],
+            "response": row["inputs.response"],
+            "ground_truth": "",
+        }
+        f.write(json.dumps(new_row) + "\n")
+
+print("Converted to genai-evals data format")