Chat-Bot_Results_Evaluation_Test-Data_Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56e3d3e4-8e36-459b-b15b-fb754a86246d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-25 11:58:39.316169: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "2023-09-25 11:58:40.929420: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1960] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import ast\n",
    "import time\n",
    "import spacy\n",
    "import openai\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "from sentence_transformers import util\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from vertexai.language_models import TextEmbeddingModel\n",
    "from vertexai.language_models import TextGenerationModel\n",
    "from vertexai.preview.language_models import *#ChatModel, InputOutputTextPair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3ceab0d0-848c-4364-9c18-36a45bab4d15",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = \"\" # Replace with OPENAI API Key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eeb4f82f-da30-4f3a-b2e8-13bfe4a5fbce",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"\" # Replace with GCP Credentials JSON File Path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28a0bad6-5435-4532-a303-2e650649c763",
   "metadata": {},
   "source": [
    "# Generate Testing Variations from Question/Answer CSV using OPENAI API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "af532f40-6653-4aff-bb14-2b57495d1271",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_variations_openai(prompt):\n",
    "    full_prompt = f\"Generate a 'single' unique variation for the question below and return another single question without changing the semantics and the meaning, only the wording should be different: \\\"{prompt}\\\"\"\n",
    "    \n",
    "    response = openai.Completion.create(\n",
    "    model=\"text-davinci-003\",\n",
    "    prompt=full_prompt,\n",
    "    temperature=0.1\n",
    "    )\n",
    "\n",
    "    output = list()\n",
    "    for k in response['choices']:\n",
    "        output.append(k['text'].strip())\n",
    "    return ''.join(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "5f4f6870-9e7e-423a-a8b4-cdbac5c2bd34",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_variations_vertex(prompt):\n",
    "    model = TextGenerationModel.from_pretrained(\"text-bison@001\")\n",
    "    \n",
    "    full_prompt = f\"Generate a 'single' unique variation for the question below and return another single question without changing the semantics and the meaning, only the wording should be different: \\\"{prompt}\\\"\"\n",
    "    \n",
    "    response = model.predict(prompt=full_prompt, temperature=0.4)\n",
    "\n",
    "    return (response.text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0e6d0e1-9f3f-41cf-ab36-2b6eeb8ce6f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "24d5a533-3b47-43d2-836a-a81f8779dda2",
   "metadata": {},
   "source": [
    "### Saving Variations locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe2918e2-2a48-4b52-913f-fcb84af4267b",
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 1\n",
    "response_list =[]\n",
    "answer_list=[]\n",
    "\n",
    "for i in range(0, len(df), 6):\n",
    "    # print(f\"{i}:  {df['Question'][i]}\")\n",
    "    # response = generate_variations_vertex(df['Question'][i])\n",
    "    response = generate_variations_openai(df['Question'][i])\n",
    "    \n",
    "    print(f\"{i}: {response}\")\n",
    "\n",
    "    response_list.append(response)\n",
    "    answer_list.append(df['Answer'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "0e994cab-90ed-4f25-a4f6-21de49edd613",
   "metadata": {},
   "outputs": [],
   "source": [
    "# answer_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b7c495e0-8fb7-482e-8366-e137c8c63d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_df = pd.DataFrame({'Question': response_list, 'Answer': answer_list})\n",
    "testing_df.head()\n",
    "testing_df.to_csv('Test_Questions.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d799c1f-57e6-42ab-9c46-0dd094817ecf",
   "metadata": {},
   "source": [
    "## Evaluating the Chat Bot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a882fa4a-00c6-49e6-9076-cead8dbda2eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c30a422b-9c55-4b94-a8c6-be7742f452db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    doc = nlp(text)\n",
    "    return \" \".join([token.lemma_ for token in doc if not token.is_punct])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99d94ec1-dad3-4ecb-90a4-f1e77e4389d8",
   "metadata": {},
   "source": [
    "### Answering the user question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3b237691-2bdb-4de9-84a3-4f9c1626049f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_most_similar_question(user_question, question_embeddings_df):\n",
    "    user_question = preprocess_text(user_question)\n",
    "    model = TextEmbeddingModel.from_pretrained(\"textembedding-gecko@001\")\n",
    "    user_question_embedding = model.get_embeddings([user_question])\n",
    "    for embedding in user_question_embedding:\n",
    "        user_question_embedding = embedding.values\n",
    "    \n",
    "    question_embeddings = question_embeddings_df['Embedding']\n",
    "    sim_score = []\n",
    "    for i in question_embeddings:\n",
    "        sim_score.append(util.pytorch_cos_sim(user_question_embedding, i).item())\n",
    "    \n",
    "    threshhold = 0.80\n",
    "    \n",
    "    max_sim_score = max(sim_score)\n",
    "\n",
    "    # print(max_sim_score)\n",
    "    \n",
    "    if max_sim_score >= threshhold:\n",
    "        sim_arg = sim_score.index(max_sim_score)\n",
    "        return embedding_df['Answer'][sim_arg], max_sim_score\n",
    "    else:\n",
    "        var = \"Sorry, Couldn't find an answer\"\n",
    "        return var, max_sim_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "980f17c7-f0c0-4fbb-8cd1-24da695f9c45",
   "metadata": {},
   "source": [
    "### Applying text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75f26ad0-e290-46b0-b367-19f96ece3b1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Path/To/question-answers.csv\")\n",
    "embedding_df['Embedding'] = embedding_df['Embedding'].apply(ast.literal_eval)\n",
    "embedding_df['Question'] = embedding_df['Question'].str.replace(r'[^\\x00-\\x7F]+', '', regex=True)\n",
    "embedding_df['Answer'] = embedding_df['Answer'].str.replace(r'[^\\x00-\\x7F]+', '', regex=True)\n",
    "embedding_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "564dee9a-deb0-4f85-87c0-1a1a5d3346ab",
   "metadata": {},
   "source": [
    "### Evaluating each question and storing the results in a txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e51baa74-1191-415c-8565-e086644b00fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pdfdf\n",
    "\n",
    "def evaluate_chatbot(test_dataframe, embedding_df, output_file_path):\n",
    "    correct_answers = 0\n",
    "    unable_to_decide = 0\n",
    "    total_questions = len(test_dataframe)\n",
    "\n",
    "    with open(output_file_path, 'w', encoding='utf-8') as output_file:\n",
    "        for index, row in test_dataframe.iterrows():\n",
    "            test_question = row['Question']\n",
    "            ground_truth_answer = row['Answer']\n",
    "    \n",
    "            # print(test_question)\n",
    "    \n",
    "            response, score = get_most_similar_question(test_question, embedding_df)\n",
    "    \n",
    "            if response == ground_truth_answer:\n",
    "                correct_answers += 1\n",
    "                result = \"Correct\"\n",
    "            elif response == \"Sorry, Couldn't find an answer\":\n",
    "                result = \"Unable to Decide\"\n",
    "            else:\n",
    "                result = \"Wrong\"\n",
    "                # print(\"\\nWrong Answer\")\n",
    "            \n",
    "            print(f\"\\n\\nQuestion: {test_question}\\n\")\n",
    "            print(f\"Chatbot's Response: {response}\\n\")\n",
    "            print(f\"Ground Truth Answer: {ground_truth_answer}\\n\")\n",
    "            print(f\"Score: {score}\\n\")\n",
    "            print(f\"Result: {result}\\n\\n\")\n",
    "\n",
    "            output_file.write(f\"\\n\\nQuestion: {test_question}\\n\")\n",
    "            output_file.write(f\"Chatbot's Response: {response}\\n\")\n",
    "            output_file.write(f\"Ground Truth Answer: {ground_truth_answer}\\n\")\n",
    "            output_file.write(f\"Score: {score}\\n\")\n",
    "            output_file.write(f\"Result: {result}\\n\\n\")\n",
    "            \n",
    "        accuracy = (correct_answers / total_questions) * 100\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afa60bcf-2a55-4704-9672-3b6d5f8f13c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_chatbot(testing_df, embedding_df, \"evaluation_results.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b669b196-9417-4402-a47c-e17ca3015449",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "566acd28-cb14-4643-9497-4f13178bc2ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeb0e0a7-0ce3-4f54-ade9-314f6aae8a93",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4684f1a-9371-41d5-9fbc-fa5be06e1504",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}