Comparison Between Fine-tuned and Default GPT-3 Turbo for Text Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c335b6ff",
   "metadata": {},
   "source": [
    "## Detailed Article Explaination\n",
    "\n",
    "The detailed code explanation for this article is available at the following link:\n",
    "\n",
    "https://www.daniweb.com/programming/computer-science/tutorials/542050/comparing-fine-tuned-and-default-gpt-3-5-turbo-for-text-classification\n",
    "\n",
    "For my other articles for Daniweb.com, please see this link:\n",
    "\n",
    "https://www.daniweb.com/members/1235222/usmanmalik57"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fc0903d-754d-43ac-ab59-d23772e1f509",
   "metadata": {},
   "source": [
    "## Importing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0cea0fe-5f4a-4af0-9694-9fa1fb7145de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from openai import OpenAI\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df098269-3b1f-413f-9f0e-9be1204965d7",
   "metadata": {},
   "source": [
    "## Importing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "430b7160-e0d9-4c8e-a3db-74867c667501",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(r\"/home/mani/Datasets/Tweets.csv\")\n",
    "print(dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f3bff62-895e-45fb-8625-ca5a10074d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(dataset, n):\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are NaN\n",
    "    dataset = dataset.dropna(subset=['airline_sentiment', 'text'])\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are empty strings\n",
    "    dataset = dataset[(dataset['airline_sentiment'].str.strip() != '') & (dataset['text'].str.strip() != '')]\n",
    "\n",
    "    # Filter the DataFrame for each sentiment\n",
    "    neutral_df = dataset[dataset['airline_sentiment'] == 'neutral']\n",
    "    positive_df = dataset[dataset['airline_sentiment'] == 'positive']\n",
    "    negative_df = dataset[dataset['airline_sentiment'] == 'negative']\n",
    "\n",
    "    # Select records from Nth index\n",
    "    neutral_sample = neutral_df[n: n +34]\n",
    "    positive_sample = positive_df[n: n +33]\n",
    "    negative_sample = negative_df[n: n +33]\n",
    "\n",
    "    # Concatenate the samples into one DataFrame\n",
    "    dataset = pd.concat([neutral_sample, positive_sample, negative_sample])\n",
    "\n",
    "    # Reset index if needed\n",
    "    dataset.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    dataset = dataset[[\"text\", \"airline_sentiment\"]]\n",
    "\n",
    "    return dataset\n",
    "\n",
    "filtered_data = preprocess_data(dataset, 0)\n",
    "# print value counts\n",
    "print(filtered_data[\"airline_sentiment\"].value_counts())\n",
    "\n",
    "filtered_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9aa08149-0b97-4b79-8ef5-f659b5f25cfd",
   "metadata": {},
   "source": [
    "## Converting Data to JSON Format for Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "208a11a8-4187-4d47-b9ae-f4c0dc58049c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# JSON file path\n",
    "json_file_path = '/home/mani/Datasets/airline_sentiments.json'\n",
    "\n",
    "# Function to create the JSON structure for each row\n",
    "def create_json_structure(row):\n",
    "    return {\n",
    "        \"messages\": [\n",
    "            {\"role\": \"system\", \"content\": \"You are a Twitter sentiment analysis expert who can predict sentiment expressed in the tweets about an airline. You select sentiment value from positive, negative, or neutral.\"},\n",
    "            {\"role\": \"user\", \"content\": row['text']},\n",
    "            {\"role\": \"assistant\", \"content\": row['airline_sentiment']}\n",
    "        ]\n",
    "    }\n",
    "\n",
    "# Convert DataFrame to JSON structures\n",
    "json_structures = filtered_data.apply(create_json_structure, axis=1).tolist()\n",
    "\n",
    "# Write JSON structures to file, each on a new line\n",
    "with open(json_file_path, 'w') as f:\n",
    "    for json_structure in json_structures:\n",
    "        f.write(json.dumps(json_structure) + '\\n')\n",
    "\n",
    "print(f\"Data has been written to {json_file_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e47cb35-e695-46a8-b121-839b862b9af7",
   "metadata": {},
   "source": [
    "## Fine-Tuning GPT-3 Turbo for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65cbb12a-9690-4b1d-b04c-ad55797955c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"YOUR_OPEN_AI_API_KEY\"\n",
    "\n",
    "client = OpenAI(api_key = api_key)\n",
    "\n",
    "training_file = client.files.create(\n",
    "  file=open(json_file_path, \"rb\"),\n",
    "  purpose=\"fine-tune\"\n",
    ")\n",
    "\n",
    "print(training_file.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3afe8213-38ab-4db3-a95d-c8d43e969bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tuning_job = client.fine_tuning.jobs.create(\n",
    "  training_file=training_file.id, \n",
    "  model=\"gpt-3.5-turbo\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33b67d01-81ec-4b65-9271-2ce86e0371a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "List up to 10 events from a fine-tuning job\n",
    "print(client.fine_tuning.jobs.list_events(fine_tuning_job_id = fine_tuning_job.id, \n",
    "                                    limit=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2619e8e-8c8e-4bd8-9278-74eecdd096be",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_model_id = client.fine_tuning.jobs.retrieve(fine_tuning_job.id).fine_tuned_model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3851c1a6-c663-41b2-bf09-7b0cf59c20f2",
   "metadata": {},
   "source": [
    "## Text Classification with Fine-tuned GPT 3.5 Turbo Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4603f3f9-411b-41bb-9c7d-b550df2dc823",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_sentiment_ft_gpt(tweet):\n",
    "\n",
    "    content = \"\"\"What is the sentiment expressed in the following tweet about an airline?\n",
    "    Select sentiment value from positive, negative, or neutral. Return only the sentiment value in small letters.\n",
    "    tweet: {}\"\"\".format(tweet)\n",
    "\n",
    "    sentiment = client.chat.completions.create(\n",
    "      model= ft_model_id,\n",
    "      temperature = 0,\n",
    "      max_tokens = 10,\n",
    "      messages=[\n",
    "            {\"role\": \"user\", \"content\": content}\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return sentiment.choices[0].message.content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71b8041b-47d8-4550-b6cd-47274937c22a",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = preprocess_data(dataset, 100)\n",
    "\n",
    "all_sentiments = []\n",
    "\n",
    "tweets_list = test_data[\"text\"].tolist()\n",
    "\n",
    "i = 0\n",
    "exceptions = 0\n",
    "while i < len(tweets_list):\n",
    "\n",
    "    try:\n",
    "        tweet = tweets_list[i]\n",
    "        sentiment_value = find_sentiment_ft_gpt(tweet)\n",
    "        all_sentiments.append(sentiment_value)\n",
    "        i = i + 1\n",
    "        print(i, sentiment_value)\n",
    "\n",
    "    except Except as e:\n",
    "        print(\"===================\")\n",
    "        print(\"Exception occured\", e)\n",
    "        exception = exception + 1\n",
    "\n",
    "print(\"Total exception count:\", exceptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a66f233a-3440-4dfb-b6c4-818d34ba958b",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = accuracy_score(all_sentiments, test_data[\"airline_sentiment\"])\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c497dc3-85a1-4c35-abc0-a5f17b393be1",
   "metadata": {},
   "source": [
    "## Text Classification with Default GPT 3.5 Turbo Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6657d26-c63d-466d-bc4c-770ea2b28def",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_sentiment_gpt(tweet):\n",
    "\n",
    "    content = \"\"\"What is the sentiment expressed in the following tweet about an airline?\n",
    "    Select sentiment value from positive, negative, or neutral. Return only the sentiment value in small letters.\n",
    "    tweet: {}\"\"\".format(tweet)\n",
    "\n",
    "    sentiment = client.chat.completions.create(\n",
    "      model= \"gpt-3.5-turbo\",\n",
    "      temperature = 0,\n",
    "      max_tokens = 10,\n",
    "      messages=[\n",
    "            {\"role\": \"user\", \"content\": content}\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return sentiment.choices[0].message.content\n",
    "\n",
    "all_sentiments = []\n",
    "\n",
    "i = 0\n",
    "exceptions = 0\n",
    "while i < len(tweets_list):\n",
    "\n",
    "    try:\n",
    "        tweet = tweets_list[i]\n",
    "        sentiment_value = find_sentiment_gpt(tweet)\n",
    "        all_sentiments.append(sentiment_value)\n",
    "        i = i + 1\n",
    "        print(i, sentiment_value)\n",
    "\n",
    "    except Except as e:\n",
    "        print(\"===================\")\n",
    "        print(\"Exception occured\", e)\n",
    "        exception = exception + 1\n",
    "\n",
    "print(\"Total exception count:\", exceptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9086cb00-9d6e-4804-af40-ab43de6ae0de",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = accuracy_score(all_sentiments, test_data[\"airline_sentiment\"])\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "180b3ee4-9f3f-4b1c-81ad-43208e05ce35",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}