How to Fine-tune OpenAI GPT-4o - The Wait is Finally Over.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d8fec017",
   "metadata": {},
   "source": [
    "# Detailed article explaination\n",
    "The detailed code explanation for this article is available at the following link:\n",
    "\n",
    "https://www.daniweb.com/programming/computer-science/tutorials/542333/how-to-fine-tune-the-openai-gpt-4o-model-the-wait-is-finally-over\n",
    "\n",
    "For my other articles for Daniweb.com, please see this link:\n",
    "\n",
    "https://www.daniweb.com/members/1235222/usmanmalik57"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e69d7c13",
   "metadata": {},
   "source": [
    "## Installing and Importing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d40bc839",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in c:\\users\\usman\\anaconda3\\lib\\site-packages (1.40.6)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (0.25.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (0.5.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (2.4.2)\n",
      "Requirement already satisfied: sniffio in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (1.2.0)\n",
      "Requirement already satisfied: tqdm>4 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (4.66.2)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.11 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openai) (4.12.2)\n",
      "Requirement already satisfied: idna>=2.8 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from anyio<5,>=3.5.0->openai) (3.4)\n",
      "Requirement already satisfied: certifi in c:\\users\\usman\\anaconda3\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (2023.7.22)\n",
      "Requirement already satisfied: httpcore<0.19.0,>=0.18.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (0.18.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (0.5.0)\n",
      "Requirement already satisfied: pydantic-core==2.10.1 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (2.10.1)\n",
      "Requirement already satisfied: colorama in c:\\users\\usman\\anaconda3\\lib\\site-packages (from tqdm>4->openai) (0.4.6)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from httpcore<0.19.0,>=0.18.0->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: rouge-score in c:\\users\\usman\\anaconda3\\lib\\site-packages (0.1.2)\n",
      "Requirement already satisfied: absl-py in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (2.0.0)\n",
      "Requirement already satisfied: nltk in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (3.8.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (1.24.3)\n",
      "Requirement already satisfied: six>=1.14.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: click in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (8.0.4)\n",
      "Requirement already satisfied: joblib in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (1.2.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (2022.7.9)\n",
      "Requirement already satisfied: tqdm in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (4.66.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\usman\\anaconda3\\lib\\site-packages (from click->nltk->rouge-score) (0.4.6)\n",
      "Requirement already satisfied: openpyxl in c:\\users\\usman\\anaconda3\\lib\\site-packages (3.1.5)\n",
      "Requirement already satisfied: et-xmlfile in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openpyxl) (1.1.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\usman\\anaconda3\\lib\\site-packages (2.2.1)\n",
      "Requirement already satisfied: openpyxl in c:\\users\\usman\\anaconda3\\lib\\site-packages (3.1.5)\n",
      "Requirement already satisfied: numpy<2,>=1.23.2 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pandas) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pandas) (2022.7)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from pandas) (2023.3)\n",
      "Requirement already satisfied: et-xmlfile in c:\\users\\usman\\anaconda3\\lib\\site-packages (from openpyxl) (1.1.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install openai\n",
    "!pip install rouge-score\n",
    "!pip install --upgrade openpyxl\n",
    "!pip install pandas openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a08ab911",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "from rouge_score import rouge_scorer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "475c2c96",
   "metadata": {},
   "source": [
    "## Fine-tuning GPT-4o for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a84b7a07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "\n",
       "               tweet_created tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800            NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800            NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800      Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800            NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800            NaN  Pacific Time (US & Canada)  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dataset = pd.read_csv(r\"D:\\Datasets\\Tweets.csv\")\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "3600f07b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def preprocess_data(dataset, n, records):\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are NaN\n",
    "    dataset = dataset.dropna(subset=['airline_sentiment', 'text'])\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are empty strings\n",
    "    dataset = dataset[(dataset['airline_sentiment'].str.strip() != '') & (dataset['text'].str.strip() != '')]\n",
    "\n",
    "    # Filter the DataFrame for each sentiment\n",
    "    neutral_df = dataset[dataset['airline_sentiment'] == 'neutral']\n",
    "    positive_df = dataset[dataset['airline_sentiment'] == 'positive']\n",
    "    negative_df = dataset[dataset['airline_sentiment'] == 'negative']\n",
    "\n",
    "    # Select records from Nth index\n",
    "    neutral_sample = neutral_df[n: n +records]\n",
    "    positive_sample = positive_df[n: n +records]\n",
    "    negative_sample = negative_df[n: n +records]\n",
    "\n",
    "    # Concatenate the samples into one DataFrame\n",
    "    dataset = pd.concat([neutral_sample, positive_sample, negative_sample])\n",
    "\n",
    "    # Reset index if needed\n",
    "    dataset.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    dataset = dataset[[\"text\", \"airline_sentiment\"]]\n",
    "\n",
    "    return dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "608881ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data value counts:\n",
      " airline_sentiment\n",
      "neutral     200\n",
      "positive    200\n",
      "negative    200\n",
      "Name: count, dtype: int64\n",
      "===========================\n",
      "Test data value counts:\n",
      " airline_sentiment\n",
      "neutral     33\n",
      "positive    33\n",
      "negative    33\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "training_data = preprocess_data(dataset, 0, 200)\n",
    "print(\"Training data value counts:\\n\", training_data[\"airline_sentiment\"].value_counts())\n",
    "print(\"===========================\")\n",
    "test_data = preprocess_data(dataset, 600, 33)\n",
    "print(\"Test data value counts:\\n\", test_data[\"airline_sentiment\"].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "e11f1cc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data has been written to D:\\Datasets\\airline_sentiments.json\n"
     ]
    }
   ],
   "source": [
    "# JSON file path\n",
    "json_file_path = r\"D:\\Datasets\\airline_sentiments.json\"\n",
    "\n",
    "# Function to create the JSON structure for each row\n",
    "def create_json_structure(row):\n",
    "    return {\n",
    "        \"messages\": [\n",
    "            {\"role\": \"system\", \"content\": \"You are a Twitter sentiment analysis expert who can predict sentiment expressed in the tweets about an airline. You select sentiment value from positive, negative, or neutral.\"},\n",
    "            {\"role\": \"user\", \"content\": row['text']},\n",
    "            {\"role\": \"assistant\", \"content\": row['airline_sentiment']}\n",
    "        ]\n",
    "    }\n",
    "\n",
    "# Convert DataFrame to JSON structures\n",
    "json_structures = training_data.apply(create_json_structure, axis=1).tolist()\n",
    "\n",
    "# Write JSON structures to file, each on a new line\n",
    "with open(json_file_path, 'w') as f:\n",
    "    for json_structure in json_structures:\n",
    "        f.write(json.dumps(json_structure) + '\\n')\n",
    "\n",
    "print(f\"Data has been written to {json_file_path}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "20c005ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI(\n",
    "    # This is the default and can be omitted\n",
    "    api_key = os.environ.get('OPENAI_API_KEY'),\n",
    ")\n",
    "\n",
    "\n",
    "training_file = client.files.create(\n",
    "  file=open(json_file_path, \"rb\"),\n",
    "  purpose=\"fine-tune\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9ef62b5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tuning_job_gpt4o = client.fine_tuning.jobs.create(\n",
    "  training_file=training_file.id, \n",
    "  model=\"gpt-4o-2024-08-06\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4dbbb31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List up to 10 events from a fine-tuning job\n",
    "print(client.fine_tuning.jobs.list_events(fine_tuning_job_id = fine_tuning_job_gpt4o.id,\n",
    "                                    limit=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "4b0189db",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_model_id = client.fine_tuning.jobs.retrieve(fine_tuning_job_gpt4o.id).fine_tuned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "cd59cc51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 neutral\n",
      "2 neutral\n",
      "3 neutral\n",
      "4 positive\n",
      "5 neutral\n",
      "6 neutral\n",
      "7 neutral\n",
      "8 neutral\n",
      "9 positive\n",
      "10 positive\n",
      "11 neutral\n",
      "12 positive\n",
      "13 neutral\n",
      "14 neutral\n",
      "15 neutral\n",
      "16 neutral\n",
      "17 neutral\n",
      "18 neutral\n",
      "19 neutral\n",
      "20 positive\n",
      "21 neutral\n",
      "22 neutral\n",
      "23 neutral\n",
      "24 neutral\n",
      "25 neutral\n",
      "26 neutral\n",
      "27 neutral\n",
      "28 neutral\n",
      "29 neutral\n",
      "30 neutral\n",
      "31 neutral\n",
      "32 neutral\n",
      "33 neutral\n",
      "34 positive\n",
      "35 positive\n",
      "36 positive\n",
      "37 positive\n",
      "38 positive\n",
      "39 positive\n",
      "40 neutral\n",
      "41 positive\n",
      "42 positive\n",
      "43 positive\n",
      "44 positive\n",
      "45 positive\n",
      "46 positive\n",
      "47 positive\n",
      "48 positive\n",
      "49 positive\n",
      "50 positive\n",
      "51 positive\n",
      "52 positive\n",
      "53 positive\n",
      "54 positive\n",
      "55 positive\n",
      "56 positive\n",
      "57 positive\n",
      "58 positive\n",
      "59 positive\n",
      "60 positive\n",
      "61 positive\n",
      "62 positive\n",
      "63 positive\n",
      "64 positive\n",
      "65 positive\n",
      "66 positive\n",
      "67 neutral\n",
      "68 negative\n",
      "69 negative\n",
      "70 negative\n",
      "71 negative\n",
      "72 negative\n",
      "73 negative\n",
      "74 negative\n",
      "75 negative\n",
      "76 negative\n",
      "77 negative\n",
      "78 negative\n",
      "79 negative\n",
      "80 negative\n",
      "81 negative\n",
      "82 negative\n",
      "83 negative\n",
      "84 negative\n",
      "85 negative\n",
      "86 negative\n",
      "87 negative\n",
      "88 negative\n",
      "89 negative\n",
      "90 negative\n",
      "91 negative\n",
      "92 negative\n",
      "93 negative\n",
      "94 negative\n",
      "95 negative\n",
      "96 negative\n",
      "97 negative\n",
      "98 negative\n",
      "99 negative\n",
      "Accuracy: 0.9292929292929293\n"
     ]
    }
   ],
   "source": [
    "def find_sentiment(client, model, dataset):\n",
    "    tweets_list = dataset[\"text\"].tolist()\n",
    "\n",
    "    all_sentiments = []\n",
    "\n",
    "\n",
    "    i = 0\n",
    "\n",
    "\n",
    "    while i < len(tweets_list):\n",
    "\n",
    "        try:\n",
    "            tweet = tweets_list[i]\n",
    "            content = \"\"\"What is the sentiment expressed in the following tweet about an airline?\n",
    "            Select sentiment value from positive, negative, or neutral. Return only the sentiment value in small letters.\n",
    "            tweet: {}\"\"\".format(tweet)\n",
    "\n",
    "            response = client.chat.completions.create(\n",
    "                model=model,\n",
    "                temperature=0,\n",
    "                max_tokens=10,\n",
    "                messages=[\n",
    "                    {\"role\": \"user\", \"content\": content}\n",
    "                ]\n",
    "            )\n",
    "\n",
    "            sentiment_value = response.choices[0].message.content\n",
    "\n",
    "            all_sentiments.append(sentiment_value)\n",
    "            i += 1\n",
    "            print(i, sentiment_value)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(\"===================\")\n",
    "            print(\"Exception occurred:\", e)\n",
    "\n",
    "    accuracy = accuracy_score(all_sentiments, dataset[\"airline_sentiment\"])\n",
    "    print(f\"Accuracy: {accuracy}\")\n",
    "    \n",
    "find_sentiment(client,ft_model_id, test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25a05b1f",
   "metadata": {},
   "source": [
    "## Fine-tuning GPT-4o for Text Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0f4249f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length of summaries: 1168.78 characters\n",
      "(1000, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>human_summary</th>\n",
       "      <th>publication</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>theme</th>\n",
       "      <th>content</th>\n",
       "      <th>summary_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>259</td>\n",
       "      <td>18240</td>\n",
       "      <td>His decision to withdraw comes the same week a...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Susanne Craig</td>\n",
       "      <td>2017-02-06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>politics</td>\n",
       "      <td>Vincent Viola, a billionaire Wall Street trade...</td>\n",
       "      <td>945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>259</td>\n",
       "      <td>18015</td>\n",
       "      <td>President Trump spoke by telephone with the ac...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Michael D. Shear and Maggie Haberman</td>\n",
       "      <td>2017-01-27</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>politics</td>\n",
       "      <td>WASHINGTON  —   President Trump spoke by telep...</td>\n",
       "      <td>1392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>259</td>\n",
       "      <td>18307</td>\n",
       "      <td>Among the major brands that used their commerc...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Sapna Maheshwari</td>\n",
       "      <td>2017-02-06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>sports</td>\n",
       "      <td>While many Super Bowl advertisers chose to be ...</td>\n",
       "      <td>896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>0</td>\n",
       "      <td>17493</td>\n",
       "      <td>A year later, the reason for the strike remain...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Joseph Goldstein</td>\n",
       "      <td>2017-01-10</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lifestyle</td>\n",
       "      <td>On an overcast Friday morning last January, lo...</td>\n",
       "      <td>1412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>259</td>\n",
       "      <td>18461</td>\n",
       "      <td>Signing balotelli was not just a way to garner...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Rory Smith</td>\n",
       "      <td>2017-02-10</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>sports</td>\n",
       "      <td>NICE, France  —     Rivère accepts the complim...</td>\n",
       "      <td>834</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0     id                                      human_summary  \\\n",
       "800         259  18240  His decision to withdraw comes the same week a...   \n",
       "622         259  18015  President Trump spoke by telephone with the ac...   \n",
       "858         259  18307  Among the major brands that used their commerc...   \n",
       "172           0  17493  A year later, the reason for the strike remain...   \n",
       "996         259  18461  Signing balotelli was not just a way to garner...   \n",
       "\n",
       "        publication                                author        date    year  \\\n",
       "800  New York Times                         Susanne Craig  2017-02-06  2017.0   \n",
       "622  New York Times  Michael D. Shear and Maggie Haberman  2017-01-27  2017.0   \n",
       "858  New York Times                      Sapna Maheshwari  2017-02-06  2017.0   \n",
       "172  New York Times                      Joseph Goldstein  2017-01-10  2017.0   \n",
       "996  New York Times                            Rory Smith  2017-02-10  2017.0   \n",
       "\n",
       "    month      theme                                            content  \\\n",
       "800   2.0   politics  Vincent Viola, a billionaire Wall Street trade...   \n",
       "622   1.0   politics  WASHINGTON  —   President Trump spoke by telep...   \n",
       "858   2.0     sports  While many Super Bowl advertisers chose to be ...   \n",
       "172   1.0  lifestyle  On an overcast Friday morning last January, lo...   \n",
       "996   2.0     sports  NICE, France  —     Rivère accepts the complim...   \n",
       "\n",
       "     summary_length  \n",
       "800             945  \n",
       "622            1392  \n",
       "858             896  \n",
       "172            1412  \n",
       "996             834  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pd.read_excel(r\"D:\\Datasets\\dataset.xlsx\")\n",
    "dataset = dataset.sample(frac=1)\n",
    "dataset['summary_length'] = dataset['human_summary'].apply(len)\n",
    "average_length = dataset['summary_length'].mean()\n",
    "print(f\"Average length of summaries: {average_length:.2f} characters\")\n",
    "print(dataset.shape)\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f8820304",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data has been written to D:\\Datasets\\news_summaries.json\n"
     ]
    }
   ],
   "source": [
    "selected_data = dataset.iloc[101:201]\n",
    "\n",
    "# Function to create the JSON structure for each row\n",
    "def create_json_structure(row):\n",
    "    return {\n",
    "        \"messages\": [\n",
    "            {\"role\": \"system\", \"content\": \"You are analyzing news articles. Use the provided content to generate a concise summary.\"},\n",
    "            {\"role\": \"user\", \"content\": row['content']},\n",
    "            {\"role\": \"assistant\", \"content\": row['human_summary']}\n",
    "        ]\n",
    "    }\n",
    "\n",
    "# Convert selected DataFrame rows to JSON structures\n",
    "json_structures = selected_data.apply(create_json_structure, axis=1).tolist()\n",
    "\n",
    "# JSON file path\n",
    "json_file_path = r\"D:\\Datasets\\news_summaries.json\"\n",
    "\n",
    "# Write JSON structures to file, each on a new line\n",
    "with open(json_file_path, 'w') as f:\n",
    "    for json_structure in json_structures:\n",
    "        f.write(json.dumps(json_structure) + '\\n')\n",
    "\n",
    "print(f\"Data has been written to {json_file_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0372eb51",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_file = client.files.create(\n",
    "  file=open(json_file_path, \"rb\"),\n",
    "  purpose=\"fine-tune\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1944649e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tuning_job_gpt4o_ts = client.fine_tuning.jobs.create(\n",
    "  training_file=training_file.id, \n",
    "  model=\"gpt-4o-2024-08-06\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42842428",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(client.fine_tuning.jobs.list_events(fine_tuning_job_id = fine_tuning_job_gpt4o_ts.id,\n",
    "                                    limit=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2df87989",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_model_id = client.fine_tuning.jobs.retrieve(fine_tuning_job_gpt4o_ts.id).fine_tuned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "366a5b79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate ROUGE scores\n",
    "def calculate_rouge(reference, candidate):\n",
    "    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n",
    "    scores = scorer.score(reference, candidate)\n",
    "    return {key: value.fmeasure for key, value in scores.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "05e86c36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summarizing article 1.\n",
      "Summarizing article 2.\n",
      "Summarizing article 3.\n",
      "Summarizing article 4.\n",
      "Summarizing article 5.\n",
      "Summarizing article 6.\n",
      "Summarizing article 7.\n",
      "Summarizing article 8.\n",
      "Summarizing article 9.\n",
      "Summarizing article 10.\n",
      "Summarizing article 11.\n",
      "Summarizing article 12.\n",
      "Summarizing article 13.\n",
      "Summarizing article 14.\n",
      "Summarizing article 15.\n",
      "Summarizing article 16.\n",
      "Summarizing article 17.\n",
      "Summarizing article 18.\n",
      "Summarizing article 19.\n",
      "Summarizing article 20.\n",
      "CPU times: total: 625 ms\n",
      "Wall time: 1min 32s\n"
     ]
    }
   ],
   "source": [
    "\n",
    "%%time\n",
    "\n",
    "results = []\n",
    "\n",
    "i = 0\n",
    "\n",
    "for _, row in dataset[:20].iterrows():\n",
    "    article = row['content']\n",
    "    human_summary = row['human_summary']\n",
    "\n",
    "    i = i + 1\n",
    "    print(f\"Summarizing article {i}.\")\n",
    "\n",
    "    prompt = f\"Summarize the following article in 1150 characters. The summary should look like human created:\\n\\n{article}\\n\\nSummary:\"\n",
    "\n",
    "    response = client.chat.completions.create(\n",
    "        model= ft_model_id,\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "        max_tokens=1150,\n",
    "        temperature=0.7\n",
    "    )\n",
    "    generated_summary = response.choices[0].message.content\n",
    "    rouge_scores = calculate_rouge(human_summary, generated_summary)\n",
    "\n",
    "    results.append({\n",
    "    'article_id': row.id,\n",
    "    'generated_summary': generated_summary,\n",
    "    'rouge1': rouge_scores['rouge1'],\n",
    "    'rouge2': rouge_scores['rouge2'],\n",
    "    'rougeL': rouge_scores['rougeL']\n",
    "    })\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a36524ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rouge1    0.579758\n",
      "rouge2    0.417515\n",
      "rougeL    0.431266\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "\n",
    "results_df = pd.DataFrame(results)\n",
    "mean_values = results_df[[\"rouge1\", \"rouge2\", \"rougeL\"]].mean()\n",
    "print(mean_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27b7e84d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}