Qwen vs Llama - Who is winning the Open Source LLM Race.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fefd7f9c",
   "metadata": {},
   "source": [
    "## Detailed Article Explaination\n",
    "\n",
    "The detailed code explanation for this article is available at the following link:\n",
    "\n",
    "https://www.daniweb.com/programming/computer-science/tutorials/542539/qwen-vs-llama-who-is-winning-the-open-source-llm-race\n",
    "\n",
    "For my other articles for Daniweb.com, please see this link:\n",
    "\n",
    "https://www.daniweb.com/members/1235222/usmanmalik57"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7df48a7b",
   "metadata": {},
   "source": [
    "## Installing and Importing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36cb0c68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: huggingface_hub==0.24.7 in c:\\users\\usman\\anaconda3\\lib\\site-packages (0.24.7)\n",
      "Requirement already satisfied: filelock in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (3.9.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (2024.3.1)\n",
      "Requirement already satisfied: packaging>=20.9 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (6.0.1)\n",
      "Requirement already satisfied: requests in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (4.66.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from huggingface_hub==0.24.7) (4.12.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\usman\\anaconda3\\lib\\site-packages (from tqdm>=4.42.1->huggingface_hub==0.24.7) (0.4.6)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from requests->huggingface_hub==0.24.7) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from requests->huggingface_hub==0.24.7) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from requests->huggingface_hub==0.24.7) (2.2.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from requests->huggingface_hub==0.24.7) (2023.7.22)\n",
      "Requirement already satisfied: rouge-score in c:\\users\\usman\\anaconda3\\lib\\site-packages (0.1.2)\n",
      "Requirement already satisfied: absl-py in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (2.0.0)\n",
      "Requirement already satisfied: nltk in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (3.8.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (1.24.3)\n",
      "Requirement already satisfied: six>=1.14.0 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: click in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (8.0.4)\n",
      "Requirement already satisfied: joblib in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (1.2.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (2022.7.9)\n",
      "Requirement already satisfied: tqdm in c:\\users\\usman\\anaconda3\\lib\\site-packages (from nltk->rouge-score) (4.66.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\usman\\anaconda3\\lib\\site-packages (from click->nltk->rouge-score) (0.4.6)\n",
      "^C\n"
     ]
    }
   ],
   "source": [
    "!pip install huggingface_hub==0.24.7\n",
    "!pip install rouge-score\n",
    "!pip install --upgrade openpyxl\n",
    "!pip install pandas openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "99ccc819",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\usman\\anaconda3\\Lib\\site-packages\\pandas\\core\\arrays\\masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import InferenceClient\n",
    "import os\n",
    "import pandas as pd\n",
    "from rouge_score import rouge_scorer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e400e94",
   "metadata": {},
   "source": [
    "## Calling Qwen 2.5 and Llama 3.1 Using Hugging Face Inference API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6868995",
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_token = os.environ.get('HF_TOKEN') \n",
    "\n",
    "#qwen 2.5 endpoint\n",
    "#https://huggingface.co/Qwen/Qwen2.5-72B-Instruct\n",
    "qwen_model_client = InferenceClient(\n",
    "    \"Qwen/Qwen2.5-72B-Instruct\",\n",
    "    token=hf_token\n",
    ")\n",
    "\n",
    "#Llama 3.1 endpoint\n",
    "#https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct\n",
    "llama_model_client = InferenceClient(\n",
    "    \"meta-llama/Llama-3.1-70B-Instruct\",\n",
    "    token=hf_token\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1806ceb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction(model, system_role, user_query):\n",
    "    \n",
    "    response = model.chat_completion(\n",
    "    messages=[{\"role\": \"system\", \"content\": system_role},\n",
    "        {\"role\": \"user\", \"content\": user_query}],\n",
    "    max_tokens=10,\n",
    "    )\n",
    "         \n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1c172cb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'positive'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "system_role = \"Assign positive, negative, or neutral sentiment to the movie review. Return only a single word in your response\"\n",
    "user_query = \"I like this movie a lot\"\n",
    "make_prediction(qwen_model_client,\n",
    "               system_role,\n",
    "               user_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5e326ae7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Negative'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "system_role = \"Assign positive, negative, or neutral sentiment to the movie review. Return only a single word in your response\"\n",
    "user_query = \"I hate this movie a lot\"\n",
    "make_prediction(llama_model_client,\n",
    "               system_role,\n",
    "               user_query)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22475d33",
   "metadata": {},
   "source": [
    "## Qwen 2.5-72b vs Llama 3.1-70b For Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0a823d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "\n",
       "               tweet_created tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800            NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800            NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800      Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800            NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800            NaN  Pacific Time (US & Canada)  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Dataset download link\n",
    "## https://www.kaggle.com/datasets/crowdflower/twitter-airline-sentiment?select=Tweets.csv\n",
    "\n",
    "dataset = pd.read_csv(r\"D:\\Datasets\\Tweets.csv\")\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "eedad4bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "airline_sentiment\n",
      "neutral     34\n",
      "positive    33\n",
      "negative    33\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Remove rows where 'airline_sentiment' or 'text' are NaN\n",
    "dataset = dataset.dropna(subset=['airline_sentiment', 'text'])\n",
    "\n",
    "# Remove rows where 'airline_sentiment' or 'text' are empty strings\n",
    "dataset = dataset[(dataset['airline_sentiment'].str.strip() != '') & (dataset['text'].str.strip() != '')]\n",
    "\n",
    "# Filter the DataFrame for each sentiment\n",
    "neutral_df = dataset[dataset['airline_sentiment'] == 'neutral']\n",
    "positive_df = dataset[dataset['airline_sentiment'] == 'positive']\n",
    "negative_df = dataset[dataset['airline_sentiment'] == 'negative']\n",
    "\n",
    "# Randomly sample records from each sentiment\n",
    "neutral_sample = neutral_df.sample(n=34)\n",
    "positive_sample = positive_df.sample(n=33)\n",
    "negative_sample = negative_df.sample(n=33)\n",
    "\n",
    "# Concatenate the samples into one DataFrame\n",
    "dataset = pd.concat([neutral_sample, positive_sample, negative_sample])\n",
    "\n",
    "# Reset index if needed\n",
    "dataset.reset_index(drop=True, inplace=True)\n",
    "\n",
    "# print value counts\n",
    "print(dataset[\"airline_sentiment\"].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "620c8309",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_sentiment(model, system_role, user_query):\n",
    "    \n",
    "    response = model.chat_completion(\n",
    "    messages=[{\"role\": \"system\", \"content\": system_role},\n",
    "        {\"role\": \"user\", \"content\": user_query}],\n",
    "    max_tokens=10,\n",
    "    )\n",
    "         \n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6a3128eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tweet 1 with model qwen2.5-72b\n",
      "1 qwen2.5-72b neutral\n",
      "Processing tweet 1 with model llama3.1-70b\n",
      "1 llama3.1-70b neutral\n",
      "Processing tweet 2 with model qwen2.5-72b\n",
      "2 qwen2.5-72b neutral\n",
      "Processing tweet 2 with model llama3.1-70b\n",
      "2 llama3.1-70b neutral\n",
      "Processing tweet 3 with model qwen2.5-72b\n",
      "3 qwen2.5-72b neutral\n",
      "Processing tweet 3 with model llama3.1-70b\n",
      "3 llama3.1-70b neutral\n",
      "Processing tweet 4 with model qwen2.5-72b\n",
      "4 qwen2.5-72b neutral\n",
      "Processing tweet 4 with model llama3.1-70b\n",
      "4 llama3.1-70b neutral\n",
      "Processing tweet 5 with model qwen2.5-72b\n",
      "5 qwen2.5-72b neutral\n",
      "Processing tweet 5 with model llama3.1-70b\n",
      "5 llama3.1-70b neutral\n",
      "Processing tweet 6 with model qwen2.5-72b\n",
      "6 qwen2.5-72b negative\n",
      "Processing tweet 6 with model llama3.1-70b\n",
      "6 llama3.1-70b neutral\n",
      "Processing tweet 7 with model qwen2.5-72b\n",
      "7 qwen2.5-72b positive\n",
      "Processing tweet 7 with model llama3.1-70b\n",
      "7 llama3.1-70b neutral\n",
      "Processing tweet 8 with model qwen2.5-72b\n",
      "8 qwen2.5-72b neutral\n",
      "Processing tweet 8 with model llama3.1-70b\n",
      "8 llama3.1-70b neutral\n",
      "Processing tweet 9 with model qwen2.5-72b\n",
      "9 qwen2.5-72b neutral\n",
      "Processing tweet 9 with model llama3.1-70b\n",
      "9 llama3.1-70b neutral\n",
      "Processing tweet 10 with model qwen2.5-72b\n",
      "10 qwen2.5-72b negative\n",
      "Processing tweet 10 with model llama3.1-70b\n",
      "10 llama3.1-70b negative\n",
      "Processing tweet 11 with model qwen2.5-72b\n",
      "11 qwen2.5-72b neutral\n",
      "Processing tweet 11 with model llama3.1-70b\n",
      "11 llama3.1-70b neutral\n",
      "Processing tweet 12 with model qwen2.5-72b\n",
      "12 qwen2.5-72b neutral\n",
      "Processing tweet 12 with model llama3.1-70b\n",
      "12 llama3.1-70b neutral\n",
      "Processing tweet 13 with model qwen2.5-72b\n",
      "13 qwen2.5-72b neutral\n",
      "Processing tweet 13 with model llama3.1-70b\n",
      "13 llama3.1-70b neutral\n",
      "Processing tweet 14 with model qwen2.5-72b\n",
      "14 qwen2.5-72b negative\n",
      "Processing tweet 14 with model llama3.1-70b\n",
      "14 llama3.1-70b neutral\n",
      "Processing tweet 15 with model qwen2.5-72b\n",
      "15 qwen2.5-72b positive\n",
      "Processing tweet 15 with model llama3.1-70b\n",
      "15 llama3.1-70b positive\n",
      "Processing tweet 16 with model qwen2.5-72b\n",
      "16 qwen2.5-72b negative\n",
      "Processing tweet 16 with model llama3.1-70b\n",
      "16 llama3.1-70b negative\n",
      "Processing tweet 17 with model qwen2.5-72b\n",
      "17 qwen2.5-72b neutral\n",
      "Processing tweet 17 with model llama3.1-70b\n",
      "17 llama3.1-70b neutral\n",
      "Processing tweet 18 with model qwen2.5-72b\n",
      "18 qwen2.5-72b positive\n",
      "Processing tweet 18 with model llama3.1-70b\n",
      "18 llama3.1-70b neutral\n",
      "Processing tweet 19 with model qwen2.5-72b\n",
      "19 qwen2.5-72b neutral\n",
      "Processing tweet 19 with model llama3.1-70b\n",
      "19 llama3.1-70b neutral\n",
      "Processing tweet 20 with model qwen2.5-72b\n",
      "20 qwen2.5-72b negative\n",
      "Processing tweet 20 with model llama3.1-70b\n",
      "20 llama3.1-70b negative\n",
      "Processing tweet 21 with model qwen2.5-72b\n",
      "21 qwen2.5-72b neutral\n",
      "Processing tweet 21 with model llama3.1-70b\n",
      "21 llama3.1-70b neutral\n",
      "Processing tweet 22 with model qwen2.5-72b\n",
      "22 qwen2.5-72b positive\n",
      "Processing tweet 22 with model llama3.1-70b\n",
      "22 llama3.1-70b neutral\n",
      "Processing tweet 23 with model qwen2.5-72b\n",
      "23 qwen2.5-72b negative\n",
      "Processing tweet 23 with model llama3.1-70b\n",
      "23 llama3.1-70b negative\n",
      "Processing tweet 24 with model qwen2.5-72b\n",
      "24 qwen2.5-72b neutral\n",
      "Processing tweet 24 with model llama3.1-70b\n",
      "24 llama3.1-70b neutral\n",
      "Processing tweet 25 with model qwen2.5-72b\n",
      "25 qwen2.5-72b neutral\n",
      "Processing tweet 25 with model llama3.1-70b\n",
      "25 llama3.1-70b neutral\n",
      "Processing tweet 26 with model qwen2.5-72b\n",
      "26 qwen2.5-72b neutral\n",
      "Processing tweet 26 with model llama3.1-70b\n",
      "26 llama3.1-70b neutral\n",
      "Processing tweet 27 with model qwen2.5-72b\n",
      "27 qwen2.5-72b neutral\n",
      "Processing tweet 27 with model llama3.1-70b\n",
      "27 llama3.1-70b neutral\n",
      "Processing tweet 28 with model qwen2.5-72b\n",
      "28 qwen2.5-72b neutral\n",
      "Processing tweet 28 with model llama3.1-70b\n",
      "28 llama3.1-70b neutral\n",
      "Processing tweet 29 with model qwen2.5-72b\n",
      "29 qwen2.5-72b neutral\n",
      "Processing tweet 29 with model llama3.1-70b\n",
      "29 llama3.1-70b neutral\n",
      "Processing tweet 30 with model qwen2.5-72b\n",
      "30 qwen2.5-72b neutral\n",
      "Processing tweet 30 with model llama3.1-70b\n",
      "30 llama3.1-70b neutral\n",
      "Processing tweet 31 with model qwen2.5-72b\n",
      "31 qwen2.5-72b neutral\n",
      "Processing tweet 31 with model llama3.1-70b\n",
      "31 llama3.1-70b neutral\n",
      "Processing tweet 32 with model qwen2.5-72b\n",
      "32 qwen2.5-72b negative\n",
      "Processing tweet 32 with model llama3.1-70b\n",
      "32 llama3.1-70b negative\n",
      "Processing tweet 33 with model qwen2.5-72b\n",
      "33 qwen2.5-72b neutral\n",
      "Processing tweet 33 with model llama3.1-70b\n",
      "33 llama3.1-70b neutral\n",
      "Processing tweet 34 with model qwen2.5-72b\n",
      "34 qwen2.5-72b positive\n",
      "Processing tweet 34 with model llama3.1-70b\n",
      "34 llama3.1-70b positive\n",
      "Processing tweet 35 with model qwen2.5-72b\n",
      "35 qwen2.5-72b positive\n",
      "Processing tweet 35 with model llama3.1-70b\n",
      "35 llama3.1-70b neutral\n",
      "Processing tweet 36 with model qwen2.5-72b\n",
      "36 qwen2.5-72b positive\n",
      "Processing tweet 36 with model llama3.1-70b\n",
      "36 llama3.1-70b positive\n",
      "Processing tweet 37 with model qwen2.5-72b\n",
      "37 qwen2.5-72b positive\n",
      "Processing tweet 37 with model llama3.1-70b\n",
      "37 llama3.1-70b positive\n",
      "Processing tweet 38 with model qwen2.5-72b\n",
      "38 qwen2.5-72b neutral\n",
      "Processing tweet 38 with model llama3.1-70b\n",
      "38 llama3.1-70b neutral\n",
      "Processing tweet 39 with model qwen2.5-72b\n",
      "39 qwen2.5-72b positive\n",
      "Processing tweet 39 with model llama3.1-70b\n",
      "39 llama3.1-70b positive\n",
      "Processing tweet 40 with model qwen2.5-72b\n",
      "40 qwen2.5-72b positive\n",
      "Processing tweet 40 with model llama3.1-70b\n",
      "40 llama3.1-70b positive\n",
      "Processing tweet 41 with model qwen2.5-72b\n",
      "41 qwen2.5-72b positive\n",
      "Processing tweet 41 with model llama3.1-70b\n",
      "41 llama3.1-70b positive\n",
      "Processing tweet 42 with model qwen2.5-72b\n",
      "42 qwen2.5-72b positive\n",
      "Processing tweet 42 with model llama3.1-70b\n",
      "42 llama3.1-70b positive\n",
      "Processing tweet 43 with model qwen2.5-72b\n",
      "43 qwen2.5-72b positive\n",
      "Processing tweet 43 with model llama3.1-70b\n",
      "43 llama3.1-70b positive\n",
      "Processing tweet 44 with model qwen2.5-72b\n",
      "44 qwen2.5-72b negative\n",
      "Processing tweet 44 with model llama3.1-70b\n",
      "44 llama3.1-70b negative\n",
      "Processing tweet 45 with model qwen2.5-72b\n",
      "45 qwen2.5-72b positive\n",
      "Processing tweet 45 with model llama3.1-70b\n",
      "45 llama3.1-70b neutral\n",
      "Processing tweet 46 with model qwen2.5-72b\n",
      "46 qwen2.5-72b positive\n",
      "Processing tweet 46 with model llama3.1-70b\n",
      "46 llama3.1-70b negative\n",
      "Processing tweet 47 with model qwen2.5-72b\n",
      "47 qwen2.5-72b neutral\n",
      "Processing tweet 47 with model llama3.1-70b\n",
      "47 llama3.1-70b neutral\n",
      "Processing tweet 48 with model qwen2.5-72b\n",
      "48 qwen2.5-72b positive\n",
      "Processing tweet 48 with model llama3.1-70b\n",
      "48 llama3.1-70b positive\n",
      "Processing tweet 49 with model qwen2.5-72b\n",
      "49 qwen2.5-72b positive\n",
      "Processing tweet 49 with model llama3.1-70b\n",
      "49 llama3.1-70b positive\n",
      "Processing tweet 50 with model qwen2.5-72b\n",
      "50 qwen2.5-72b positive\n",
      "Processing tweet 50 with model llama3.1-70b\n",
      "50 llama3.1-70b positive\n",
      "Processing tweet 51 with model qwen2.5-72b\n",
      "51 qwen2.5-72b positive\n",
      "Processing tweet 51 with model llama3.1-70b\n",
      "51 llama3.1-70b positive\n",
      "Processing tweet 52 with model qwen2.5-72b\n",
      "52 qwen2.5-72b positive\n",
      "Processing tweet 52 with model llama3.1-70b\n",
      "52 llama3.1-70b positive\n",
      "Processing tweet 53 with model qwen2.5-72b\n",
      "53 qwen2.5-72b positive\n",
      "Processing tweet 53 with model llama3.1-70b\n",
      "53 llama3.1-70b positive\n",
      "Processing tweet 54 with model qwen2.5-72b\n",
      "54 qwen2.5-72b positive\n",
      "Processing tweet 54 with model llama3.1-70b\n",
      "54 llama3.1-70b positive\n",
      "Processing tweet 55 with model qwen2.5-72b\n",
      "55 qwen2.5-72b neutral\n",
      "Processing tweet 55 with model llama3.1-70b\n",
      "55 llama3.1-70b positive\n",
      "Processing tweet 56 with model qwen2.5-72b\n",
      "56 qwen2.5-72b neutral\n",
      "Processing tweet 56 with model llama3.1-70b\n",
      "56 llama3.1-70b neutral\n",
      "Processing tweet 57 with model qwen2.5-72b\n",
      "57 qwen2.5-72b positive\n",
      "Processing tweet 57 with model llama3.1-70b\n",
      "57 llama3.1-70b positive\n",
      "Processing tweet 58 with model qwen2.5-72b\n",
      "58 qwen2.5-72b positive\n",
      "Processing tweet 58 with model llama3.1-70b\n",
      "58 llama3.1-70b positive\n",
      "Processing tweet 59 with model qwen2.5-72b\n",
      "59 qwen2.5-72b positive\n",
      "Processing tweet 59 with model llama3.1-70b\n",
      "59 llama3.1-70b neutral\n",
      "Processing tweet 60 with model qwen2.5-72b\n",
      "60 qwen2.5-72b positive\n",
      "Processing tweet 60 with model llama3.1-70b\n",
      "60 llama3.1-70b positive\n",
      "Processing tweet 61 with model qwen2.5-72b\n",
      "61 qwen2.5-72b positive\n",
      "Processing tweet 61 with model llama3.1-70b\n",
      "61 llama3.1-70b positive\n",
      "Processing tweet 62 with model qwen2.5-72b\n",
      "62 qwen2.5-72b positive\n",
      "Processing tweet 62 with model llama3.1-70b\n",
      "62 llama3.1-70b neutral\n",
      "Processing tweet 63 with model qwen2.5-72b\n",
      "63 qwen2.5-72b positive\n",
      "Processing tweet 63 with model llama3.1-70b\n",
      "63 llama3.1-70b positive\n",
      "Processing tweet 64 with model qwen2.5-72b\n",
      "64 qwen2.5-72b positive\n",
      "Processing tweet 64 with model llama3.1-70b\n",
      "64 llama3.1-70b neutral\n",
      "Processing tweet 65 with model qwen2.5-72b\n",
      "65 qwen2.5-72b positive\n",
      "Processing tweet 65 with model llama3.1-70b\n",
      "65 llama3.1-70b positive\n",
      "Processing tweet 66 with model qwen2.5-72b\n",
      "66 qwen2.5-72b positive\n",
      "Processing tweet 66 with model llama3.1-70b\n",
      "66 llama3.1-70b positive\n",
      "Processing tweet 67 with model qwen2.5-72b\n",
      "67 qwen2.5-72b positive\n",
      "Processing tweet 67 with model llama3.1-70b\n",
      "67 llama3.1-70b positive\n",
      "Processing tweet 68 with model qwen2.5-72b\n",
      "68 qwen2.5-72b negative\n",
      "Processing tweet 68 with model llama3.1-70b\n",
      "68 llama3.1-70b negative\n",
      "Processing tweet 69 with model qwen2.5-72b\n",
      "69 qwen2.5-72b negative\n",
      "Processing tweet 69 with model llama3.1-70b\n",
      "69 llama3.1-70b positive\n",
      "Processing tweet 70 with model qwen2.5-72b\n",
      "70 qwen2.5-72b negative\n",
      "Processing tweet 70 with model llama3.1-70b\n",
      "70 llama3.1-70b negative\n",
      "Processing tweet 71 with model qwen2.5-72b\n",
      "71 qwen2.5-72b negative\n",
      "Processing tweet 71 with model llama3.1-70b\n",
      "71 llama3.1-70b negative\n",
      "Processing tweet 72 with model qwen2.5-72b\n",
      "72 qwen2.5-72b negative\n",
      "Processing tweet 72 with model llama3.1-70b\n",
      "72 llama3.1-70b negative\n",
      "Processing tweet 73 with model qwen2.5-72b\n",
      "73 qwen2.5-72b negative\n",
      "Processing tweet 73 with model llama3.1-70b\n",
      "73 llama3.1-70b negative\n",
      "Processing tweet 74 with model qwen2.5-72b\n",
      "74 qwen2.5-72b negative\n",
      "Processing tweet 74 with model llama3.1-70b\n",
      "74 llama3.1-70b negative\n",
      "Processing tweet 75 with model qwen2.5-72b\n",
      "75 qwen2.5-72b neutral\n",
      "Processing tweet 75 with model llama3.1-70b\n",
      "75 llama3.1-70b neutral\n",
      "Processing tweet 76 with model qwen2.5-72b\n",
      "76 qwen2.5-72b negative\n",
      "Processing tweet 76 with model llama3.1-70b\n",
      "76 llama3.1-70b negative\n",
      "Processing tweet 77 with model qwen2.5-72b\n",
      "77 qwen2.5-72b negative\n",
      "Processing tweet 77 with model llama3.1-70b\n",
      "77 llama3.1-70b negative\n",
      "Processing tweet 78 with model qwen2.5-72b\n",
      "78 qwen2.5-72b negative\n",
      "Processing tweet 78 with model llama3.1-70b\n",
      "78 llama3.1-70b negative\n",
      "Processing tweet 79 with model qwen2.5-72b\n",
      "79 qwen2.5-72b negative\n",
      "Processing tweet 79 with model llama3.1-70b\n",
      "79 llama3.1-70b negative\n",
      "Processing tweet 80 with model qwen2.5-72b\n",
      "80 qwen2.5-72b negative\n",
      "Processing tweet 80 with model llama3.1-70b\n",
      "80 llama3.1-70b negative\n",
      "Processing tweet 81 with model qwen2.5-72b\n",
      "81 qwen2.5-72b negative\n",
      "Processing tweet 81 with model llama3.1-70b\n",
      "81 llama3.1-70b negative\n",
      "Processing tweet 82 with model qwen2.5-72b\n",
      "82 qwen2.5-72b negative\n",
      "Processing tweet 82 with model llama3.1-70b\n",
      "82 llama3.1-70b negative\n",
      "Processing tweet 83 with model qwen2.5-72b\n",
      "83 qwen2.5-72b negative\n",
      "Processing tweet 83 with model llama3.1-70b\n",
      "83 llama3.1-70b negative\n",
      "Processing tweet 84 with model qwen2.5-72b\n",
      "84 qwen2.5-72b negative\n",
      "Processing tweet 84 with model llama3.1-70b\n",
      "84 llama3.1-70b negative\n",
      "Processing tweet 85 with model qwen2.5-72b\n",
      "85 qwen2.5-72b negative\n",
      "Processing tweet 85 with model llama3.1-70b\n",
      "85 llama3.1-70b negative\n",
      "Processing tweet 86 with model qwen2.5-72b\n",
      "86 qwen2.5-72b negative\n",
      "Processing tweet 86 with model llama3.1-70b\n",
      "86 llama3.1-70b neutral\n",
      "Processing tweet 87 with model qwen2.5-72b\n",
      "87 qwen2.5-72b neutral\n",
      "Processing tweet 87 with model llama3.1-70b\n",
      "87 llama3.1-70b neutral\n",
      "Processing tweet 88 with model qwen2.5-72b\n",
      "88 qwen2.5-72b negative\n",
      "Processing tweet 88 with model llama3.1-70b\n",
      "88 llama3.1-70b negative\n",
      "Processing tweet 89 with model qwen2.5-72b\n",
      "89 qwen2.5-72b negative\n",
      "Processing tweet 89 with model llama3.1-70b\n",
      "89 llama3.1-70b negative\n",
      "Processing tweet 90 with model qwen2.5-72b\n",
      "90 qwen2.5-72b negative\n",
      "Processing tweet 90 with model llama3.1-70b\n",
      "90 llama3.1-70b negative\n",
      "Processing tweet 91 with model qwen2.5-72b\n",
      "91 qwen2.5-72b neutral\n",
      "Processing tweet 91 with model llama3.1-70b\n",
      "91 llama3.1-70b neutral\n",
      "Processing tweet 92 with model qwen2.5-72b\n",
      "92 qwen2.5-72b negative\n",
      "Processing tweet 92 with model llama3.1-70b\n",
      "92 llama3.1-70b negative\n",
      "Processing tweet 93 with model qwen2.5-72b\n",
      "93 qwen2.5-72b negative\n",
      "Processing tweet 93 with model llama3.1-70b\n",
      "93 llama3.1-70b neutral\n",
      "Processing tweet 94 with model qwen2.5-72b\n",
      "94 qwen2.5-72b negative\n",
      "Processing tweet 94 with model llama3.1-70b\n",
      "94 llama3.1-70b negative\n",
      "Processing tweet 95 with model qwen2.5-72b\n",
      "95 qwen2.5-72b negative\n",
      "Processing tweet 95 with model llama3.1-70b\n",
      "95 llama3.1-70b negative\n",
      "Processing tweet 96 with model qwen2.5-72b\n",
      "96 qwen2.5-72b negative\n",
      "Processing tweet 96 with model llama3.1-70b\n",
      "96 llama3.1-70b negative\n",
      "Processing tweet 97 with model qwen2.5-72b\n",
      "97 qwen2.5-72b negative\n",
      "Processing tweet 97 with model llama3.1-70b\n",
      "97 llama3.1-70b negative\n",
      "Processing tweet 98 with model qwen2.5-72b\n",
      "98 qwen2.5-72b negative\n",
      "Processing tweet 98 with model llama3.1-70b\n",
      "98 llama3.1-70b negative\n",
      "Processing tweet 99 with model qwen2.5-72b\n",
      "99 qwen2.5-72b negative\n",
      "Processing tweet 99 with model llama3.1-70b\n",
      "99 llama3.1-70b negative\n",
      "Processing tweet 100 with model qwen2.5-72b\n",
      "100 qwen2.5-72b negative\n",
      "Processing tweet 100 with model llama3.1-70b\n",
      "100 llama3.1-70b negative\n",
      "Total exception count: 0\n"
     ]
    }
   ],
   "source": [
    "models = {\n",
    "    \"qwen2.5-72b\": qwen_model_client,\n",
    "    \"llama3.1-70b\": llama_model_client\n",
    "}\n",
    "\n",
    "tweets_list = dataset[\"text\"].tolist()\n",
    "all_sentiments = []\n",
    "exceptions = 0\n",
    "\n",
    "for i, tweet in enumerate(tweets_list, 1):\n",
    "    for model_name, model_client in models.items():\n",
    "        try:\n",
    "            print(f\"Processing tweet {i} with model {model_name}\")\n",
    "\n",
    "            system_role = \"You are an expert in annotating tweets with positive, negative, and neutral emotions\"\n",
    "\n",
    "            user_query = (\n",
    "                f\"What is the sentiment expressed in the following tweet about an airline? \"\n",
    "                f\"Select sentiment value from positive, negative, or neutral. \"\n",
    "                f\"Return only the sentiment value in small letters.\\n\\n\"\n",
    "                f\"tweet: {tweet}\"\n",
    "            )\n",
    "\n",
    "            sentiment_value = predict_sentiment(model_client, system_role, user_query)\n",
    "            all_sentiments.append({\n",
    "                'tweet_id': i,\n",
    "                'model': model_name,\n",
    "                'sentiment': sentiment_value\n",
    "            })\n",
    "            print(i, model_name, sentiment_value)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(\"===================\")\n",
    "            print(\"Exception occurred with model:\", model_name, \"| Tweet:\", i, \"| Error:\", e)\n",
    "            exceptions += 1\n",
    "\n",
    "print(\"Total exception count:\", exceptions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "854f37cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for qwen2.5-72b: 0.8\n",
      "Accuracy for llama3.1-70b: 0.77\n"
     ]
    }
   ],
   "source": [
    "results_df = pd.DataFrame(all_sentiments)\n",
    "for model_name in models.keys():\n",
    "    model_results = results_df[results_df['model'] == model_name]\n",
    "    accuracy = accuracy_score(model_results['sentiment'], dataset[\"airline_sentiment\"].iloc[:len(model_results)])\n",
    "    print(f\"Accuracy for {model_name}: {accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f050b56d",
   "metadata": {},
   "source": [
    "## Qwen 2.5-72b vs Llama 3.1-70b For Text Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f8d5d9b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>human_summary</th>\n",
       "      <th>publication</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>theme</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>259</td>\n",
       "      <td>18334</td>\n",
       "      <td>the new york times • in bel air the most expen...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Mike McPhate</td>\n",
       "      <td>2017-02-06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>science</td>\n",
       "      <td>Good morning. (Want to get California Today by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0</td>\n",
       "      <td>17333</td>\n",
       "      <td>Donald J. Trump on Tuesday named as his chief ...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Binyamin Appelbaum</td>\n",
       "      <td>2017-01-04</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>politics</td>\n",
       "      <td>WASHINGTON  —     Donald J. Trump on Tuesday n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>259</td>\n",
       "      <td>18209</td>\n",
       "      <td>Both the coal and rules were made final in the...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Hiroko Tabuchi</td>\n",
       "      <td>2017-02-03</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>politics</td>\n",
       "      <td>Republicans on Thursday took one of their firs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>259</td>\n",
       "      <td>18246</td>\n",
       "      <td>Due to some of the provocations out of north k...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Michael R. Gordon and Motoko Rich</td>\n",
       "      <td>2017-02-04</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>politics</td>\n",
       "      <td>TOKYO  —   Defense Secretary Jim Mattis assure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>259</td>\n",
       "      <td>18311</td>\n",
       "      <td>The rest of the money goes toward the fledglin...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Clair MacDougall</td>\n",
       "      <td>2017-02-06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>crime</td>\n",
       "      <td>MONROVIA, Liberia  —   Emmanuel Dongo, who spe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0     id                                      human_summary  \\\n",
       "883         259  18334  the new york times • in bel air the most expen...   \n",
       "38            0  17333  Donald J. Trump on Tuesday named as his chief ...   \n",
       "773         259  18209  Both the coal and rules were made final in the...   \n",
       "806         259  18246  Due to some of the provocations out of north k...   \n",
       "862         259  18311  The rest of the money goes toward the fledglin...   \n",
       "\n",
       "        publication                             author        date    year  \\\n",
       "883  New York Times                       Mike McPhate  2017-02-06  2017.0   \n",
       "38   New York Times                 Binyamin Appelbaum  2017-01-04  2017.0   \n",
       "773  New York Times                     Hiroko Tabuchi  2017-02-03  2017.0   \n",
       "806  New York Times  Michael R. Gordon and Motoko Rich  2017-02-04  2017.0   \n",
       "862  New York Times                   Clair MacDougall  2017-02-06  2017.0   \n",
       "\n",
       "    month     theme                                            content  \n",
       "883   2.0   science  Good morning. (Want to get California Today by...  \n",
       "38    1.0  politics  WASHINGTON  —     Donald J. Trump on Tuesday n...  \n",
       "773   2.0  politics  Republicans on Thursday took one of their firs...  \n",
       "806   2.0  politics  TOKYO  —   Defense Secretary Jim Mattis assure...  \n",
       "862   2.0     crime  MONROVIA, Liberia  —   Emmanuel Dongo, who spe...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Kaggle dataset download link\n",
    "# https://github.com/reddzzz/DataScience_FP/blob/main/dataset.xlsx\n",
    "\n",
    "dataset = pd.read_excel(r\"D:\\Datasets\\dataset.xlsx\")\n",
    "dataset = dataset.sample(frac=1)\n",
    "print(dataset.shape)\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cd9a0744",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length of summaries: 1168.78 characters\n"
     ]
    }
   ],
   "source": [
    "dataset['summary_length'] = dataset['human_summary'].apply(len)\n",
    "average_length = dataset['summary_length'].mean()\n",
    "print(f\"Average length of summaries: {average_length:.2f} characters\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dfb7b303",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_summary(model, system_role, user_query):\n",
    "    \n",
    "    response = model.chat_completion(\n",
    "    messages=[{\"role\": \"system\", \"content\": system_role},\n",
    "        {\"role\": \"user\", \"content\": user_query}],\n",
    "    max_tokens=1200,\n",
    "    )\n",
    "         \n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a1c8813f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate ROUGE scores\n",
    "def calculate_rouge(reference, candidate):\n",
    "    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n",
    "    scores = scorer.score(reference, candidate)\n",
    "    return {key: value.fmeasure for key, value in scores.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cad482ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summarizing article 1 with model qwen2.5-72b\n",
      "Summarizing article 1 with model llama3.1-70b\n",
      "Summarizing article 2 with model qwen2.5-72b\n",
      "Summarizing article 2 with model llama3.1-70b\n",
      "Summarizing article 3 with model qwen2.5-72b\n",
      "Summarizing article 3 with model llama3.1-70b\n",
      "Summarizing article 4 with model qwen2.5-72b\n",
      "Summarizing article 4 with model llama3.1-70b\n",
      "Summarizing article 5 with model qwen2.5-72b\n",
      "Summarizing article 5 with model llama3.1-70b\n",
      "Summarizing article 6 with model qwen2.5-72b\n",
      "Summarizing article 6 with model llama3.1-70b\n",
      "Summarizing article 7 with model qwen2.5-72b\n",
      "Summarizing article 7 with model llama3.1-70b\n",
      "Summarizing article 8 with model qwen2.5-72b\n",
      "Summarizing article 8 with model llama3.1-70b\n",
      "Summarizing article 9 with model qwen2.5-72b\n",
      "Summarizing article 9 with model llama3.1-70b\n",
      "Summarizing article 10 with model qwen2.5-72b\n",
      "Summarizing article 10 with model llama3.1-70b\n",
      "Summarizing article 11 with model qwen2.5-72b\n",
      "Summarizing article 11 with model llama3.1-70b\n",
      "Summarizing article 12 with model qwen2.5-72b\n",
      "Summarizing article 12 with model llama3.1-70b\n",
      "Summarizing article 13 with model qwen2.5-72b\n",
      "Summarizing article 13 with model llama3.1-70b\n",
      "Summarizing article 14 with model qwen2.5-72b\n",
      "Summarizing article 14 with model llama3.1-70b\n",
      "Summarizing article 15 with model qwen2.5-72b\n",
      "Summarizing article 15 with model llama3.1-70b\n",
      "Summarizing article 16 with model qwen2.5-72b\n",
      "Summarizing article 16 with model llama3.1-70b\n",
      "Summarizing article 17 with model qwen2.5-72b\n",
      "Summarizing article 17 with model llama3.1-70b\n",
      "Summarizing article 18 with model qwen2.5-72b\n",
      "Summarizing article 18 with model llama3.1-70b\n",
      "Summarizing article 19 with model qwen2.5-72b\n",
      "Summarizing article 19 with model llama3.1-70b\n",
      "Summarizing article 20 with model qwen2.5-72b\n",
      "Summarizing article 20 with model llama3.1-70b\n"
     ]
    }
   ],
   "source": [
    "models = {\"qwen2.5-72b\": qwen_model_client,\n",
    "          \"llama3.1-70b\": llama_model_client}\n",
    "\n",
    "results = []\n",
    "\n",
    "i = 0\n",
    "for _, row in dataset[:20].iterrows():\n",
    "    article = row['content']\n",
    "    human_summary = row['human_summary']\n",
    "    \n",
    "    i = i + 1\n",
    "    \n",
    "    for model_name, model_client in models.items():\n",
    "        \n",
    "        print(f\"Summarizing article {i} with model {model_name}\")\n",
    "        system_role = \"You are an expert in creating summaries from text\"\n",
    "        user_query = f\"Summarize the following article in 1150 characters. The summary should look like human created:\\n\\n{article}\\n\\nSummary:\"\n",
    "        \n",
    "        generated_summary = generate_summary(model_client, system_role, user_query)\n",
    "        rouge_scores = calculate_rouge(human_summary, generated_summary)\n",
    "        \n",
    "        results.append({\n",
    "            'model': model_name,\n",
    "            'article_id': row.id,\n",
    "            'generated_summary': generated_summary,\n",
    "            'rouge1': rouge_scores['rouge1'],\n",
    "            'rouge2': rouge_scores['rouge2'],\n",
    "            'rougeL': rouge_scores['rougeL']\n",
    "        })\n",
    "\n",
    "# Create a DataFrame with results\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b293f058",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average ROUGE scores by model:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>qwen2.5-72b</th>\n",
       "      <td>0.377589</td>\n",
       "      <td>0.096248</td>\n",
       "      <td>0.186228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>llama3.1-70b</th>\n",
       "      <td>0.337821</td>\n",
       "      <td>0.082739</td>\n",
       "      <td>0.174995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                rouge1    rouge2    rougeL\n",
       "model                                     \n",
       "qwen2.5-72b   0.377589  0.096248  0.186228\n",
       "llama3.1-70b  0.337821  0.082739  0.174995"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_scores = results_df.groupby('model')[['rouge1', 'rouge2', 'rougeL']].mean()\n",
    "average_scores_sorted = average_scores.sort_values(by='rouge1', ascending=False)\n",
    "print(\"Average ROUGE scores by model:\")\n",
    "average_scores_sorted.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ee0b34b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}