Comparison of OpenAI GPT-4o vs Meta Llama 3 for Zero Shot Text Classifiation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Detailed Article Explaination\n",
    "\n",
    "The detailed code explanation for this article is available at the following link:\n",
    "\n",
    "https://www.daniweb.com/programming/computer-science/tutorials/542001/openai-gpt-4o-vs-meta-llama-3-for-zero-shot-text-classifiation\n",
    "\n",
    "For my other articles for Daniweb.com, please see this link:\n",
    "\n",
    "https://www.daniweb.com/members/1235222/usmanmalik57"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing and Installing Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (1.30.1)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (4.3.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (0.27.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (2.7.1)\n",
      "Requirement already satisfied: sniffio in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from openai) (4.11.0)\n",
      "Requirement already satisfied: idna>=2.8 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Requirement already satisfied: certifi in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.2 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (2.18.2)\n",
      "Requirement already satisfied: groq in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (0.5.0)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (4.3.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (0.27.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (2.7.1)\n",
      "Requirement already satisfied: sniffio in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (1.3.1)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from groq) (4.11.0)\n",
      "Requirement already satisfied: idna>=2.8 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from anyio<5,>=3.5.0->groq) (3.7)\n",
      "Requirement already satisfied: certifi in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->groq) (2024.2.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->groq) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->groq) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->groq) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.2 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->groq) (2.18.2)\n",
      "Requirement already satisfied: pandas in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (2.2.2)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pandas) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Collecting scikit-learn\n",
      "  Downloading scikit_learn-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /home/mani/BraimIA/rd-rag-langchain/ragenv/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n",
      "Collecting scipy>=1.6.0 (from scikit-learn)\n",
      "  Downloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting joblib>=1.2.0 (from scikit-learn)\n",
      "  Downloading joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n",
      "Collecting threadpoolctl>=3.1.0 (from scikit-learn)\n",
      "  Downloading threadpoolctl-3.5.0-py3-none-any.whl.metadata (13 kB)\n",
      "Downloading scikit_learn-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.3 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.3/13.3 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hDownloading joblib-1.4.2-py3-none-any.whl (301 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.8/301.8 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.6 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.6/38.6 MB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading threadpoolctl-3.5.0-py3-none-any.whl (18 kB)\n",
      "Installing collected packages: threadpoolctl, scipy, joblib, scikit-learn\n",
      "Successfully installed joblib-1.4.2 scikit-learn-1.5.0 scipy-1.13.1 threadpoolctl-3.5.0\n"
     ]
    }
   ],
   "source": [
    "!pip install openai\n",
    "!pip install groq\n",
    "!pip install pandas\n",
    "!pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from openai import OpenAI\n",
    "from groq import Groq\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing and Preprocessing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14640, 15)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "\n",
       "               tweet_created tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800            NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800            NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800      Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800            NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800            NaN  Pacific Time (US & Canada)  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pd.read_csv(r\"/home/mani/Datasets/Tweets.csv\")\n",
    "print(dataset.shape)\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "airline_sentiment\n",
      "neutral     34\n",
      "positive    33\n",
      "negative    33\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "def preprocess_data(dataset):\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are NaN\n",
    "    dataset = dataset.dropna(subset=['airline_sentiment', 'text'])\n",
    "\n",
    "    # Remove rows where 'airline_sentiment' or 'text' are empty strings\n",
    "    dataset = dataset[(dataset['airline_sentiment'].str.strip() != '') & (dataset['text'].str.strip() != '')]\n",
    "\n",
    "    # Filter the DataFrame for each sentiment\n",
    "    neutral_df = dataset[dataset['airline_sentiment'] == 'neutral']\n",
    "    positive_df = dataset[dataset['airline_sentiment'] == 'positive']\n",
    "    negative_df = dataset[dataset['airline_sentiment'] == 'negative']\n",
    "\n",
    "    # Randomly sample records from each sentiment\n",
    "    neutral_sample = neutral_df.sample(n=34)\n",
    "    positive_sample = positive_df.sample(n=33)\n",
    "    negative_sample = negative_df.sample(n=33)\n",
    "\n",
    "    # Concatenate the samples into one DataFrame\n",
    "    dataset = pd.concat([neutral_sample, positive_sample, negative_sample])\n",
    "\n",
    "    # Reset index if needed\n",
    "    dataset.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    return dataset\n",
    "\n",
    "\n",
    "dataset = preprocess_data(dataset)\n",
    "# print value counts\n",
    "print(dataset[\"airline_sentiment\"].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zero Shot Text Classification with GPT-4o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI(\n",
    "    api_key = os.environ.get('OPENAI_API_KEY'),\n",
    ")\n",
    "\n",
    "\n",
    "def find_sentiment_gpt(tweet):\n",
    "\n",
    "    content = \"\"\"What is the sentiment expressed in the following tweet about an airline?\n",
    "    Select sentiment value from positive, negative, or neutral. Return only the sentiment value in small letters.\n",
    "    tweet: {}\"\"\".format(tweet)\n",
    "\n",
    "    sentiment = client.chat.completions.create(\n",
    "      model= \"gpt-4o\",\n",
    "      temperature = 0,\n",
    "      max_tokens = 10,\n",
    "      messages=[\n",
    "            {\"role\": \"user\", \"content\": content}\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return sentiment.choices[0].message.content\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 negative\n",
      "2 neutral\n",
      "3 negative\n",
      "4 neutral\n",
      "5 neutral\n",
      "6 neutral\n",
      "7 neutral\n",
      "8 negative\n",
      "9 neutral\n",
      "10 negative\n",
      "11 neutral\n",
      "12 neutral\n",
      "13 neutral\n",
      "14 negative\n",
      "15 neutral\n",
      "16 neutral\n",
      "17 neutral\n",
      "18 neutral\n",
      "19 neutral\n",
      "20 neutral\n",
      "21 negative\n",
      "22 neutral\n",
      "23 neutral\n",
      "24 neutral\n",
      "25 neutral\n",
      "26 neutral\n",
      "27 positive\n",
      "28 neutral\n",
      "29 negative\n",
      "30 neutral\n",
      "31 neutral\n",
      "32 negative\n",
      "33 neutral\n",
      "34 neutral\n",
      "35 positive\n",
      "36 neutral\n",
      "37 neutral\n",
      "38 positive\n",
      "39 positive\n",
      "40 positive\n",
      "41 positive\n",
      "42 positive\n",
      "43 positive\n",
      "44 neutral\n",
      "45 neutral\n",
      "46 positive\n",
      "47 positive\n",
      "48 positive\n",
      "49 positive\n",
      "50 neutral\n",
      "51 positive\n",
      "52 neutral\n",
      "53 positive\n",
      "54 positive\n",
      "55 neutral\n",
      "56 positive\n",
      "57 positive\n",
      "58 positive\n",
      "59 positive\n",
      "60 neutral\n",
      "61 positive\n",
      "62 neutral\n",
      "63 negative\n",
      "64 positive\n",
      "65 positive\n",
      "66 positive\n",
      "67 positive\n",
      "68 negative\n",
      "69 negative\n",
      "70 negative\n",
      "71 negative\n",
      "72 negative\n",
      "73 negative\n",
      "74 negative\n",
      "75 negative\n",
      "76 negative\n",
      "77 negative\n",
      "78 negative\n",
      "79 negative\n",
      "80 negative\n",
      "81 negative\n",
      "82 negative\n",
      "83 negative\n",
      "84 neutral\n",
      "85 negative\n",
      "86 negative\n",
      "87 negative\n",
      "88 negative\n",
      "89 negative\n",
      "90 negative\n",
      "91 negative\n",
      "92 negative\n",
      "93 neutral\n",
      "94 neutral\n",
      "95 negative\n",
      "96 negative\n",
      "97 negative\n",
      "98 negative\n",
      "99 negative\n",
      "100 negative\n",
      "Total exception count: 0\n",
      "CPU times: user 658 ms, sys: 43.4 ms, total: 701 ms\n",
      "Wall time: 57.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "all_sentiments = []\n",
    "\n",
    "tweets_list = dataset[\"text\"].tolist()\n",
    "\n",
    "i = 0\n",
    "exceptions = 0\n",
    "while i < len(tweets_list):\n",
    "\n",
    "    try:\n",
    "        tweet = tweets_list[i]\n",
    "        sentiment_value = find_sentiment_gpt(tweet)\n",
    "        all_sentiments.append(sentiment_value)\n",
    "        i = i + 1\n",
    "        print(i, sentiment_value)\n",
    "\n",
    "    except Except as e:\n",
    "        print(\"===================\")\n",
    "        print(\"Exception occured\", e)\n",
    "        exception = exception + 1\n",
    "\n",
    "print(\"Total exception count:\", exceptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.78\n"
     ]
    }
   ],
   "source": [
    "accuracy = accuracy_score(all_sentiments, dataset[\"airline_sentiment\"])\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zero Shot Text Classification with Groq Llama-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Groq(\n",
    "    api_key=os.environ.get(\"GROQ_API_KEY\"),\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "def find_sentiment_llama3(tweet):\n",
    "\n",
    "    content = \"\"\"What is the sentiment expressed in the following tweet about an airline?\n",
    "    Select sentiment value from positive, negative, or neutral. Return only the sentiment value in small letters.\n",
    "    tweet: {}\"\"\".format(tweet)\n",
    "\n",
    "    sentiment = client.chat.completions.create(\n",
    "      model=\"llama3-70b-8192\",\n",
    "      temperature = 0,\n",
    "      max_tokens = 10,\n",
    "      messages=[\n",
    "            {\"role\": \"user\", \"content\": content}\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return sentiment.choices[0].message.content\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 negative\n",
      "2 neutral\n",
      "3 neutral\n",
      "4 neutral\n",
      "5 neutral\n",
      "6 neutral\n",
      "7 neutral\n",
      "8 neutral\n",
      "9 negative\n",
      "10 positive\n",
      "11 negative\n",
      "12 neutral\n",
      "13 negative\n",
      "14 neutral\n",
      "15 negative\n",
      "16 neutral\n",
      "17 negative\n",
      "18 negative\n",
      "19 negative\n",
      "20 neutral\n",
      "21 positive\n",
      "22 neutral\n",
      "23 negative\n",
      "24 negative\n",
      "25 neutral\n",
      "26 negative\n",
      "27 neutral\n",
      "28 negative\n",
      "29 positive\n",
      "30 neutral\n",
      "31 neutral\n",
      "32 positive\n",
      "33 neutral\n",
      "34 negative\n",
      "35 positive\n",
      "36 positive\n",
      "37 positive\n",
      "38 positive\n",
      "39 positive\n",
      "40 negative\n",
      "41 positive\n",
      "42 positive\n",
      "43 positive\n",
      "44 positive\n",
      "45 negative\n",
      "46 positive\n",
      "47 positive\n",
      "48 positive\n",
      "49 positive\n",
      "50 positive\n",
      "51 positive\n",
      "52 positive\n",
      "53 positive\n",
      "54 positive\n",
      "55 positive\n",
      "56 positive\n",
      "57 neutral\n",
      "58 positive\n",
      "59 positive\n",
      "60 positive\n",
      "61 positive\n",
      "62 positive\n",
      "63 positive\n",
      "64 positive\n",
      "65 positive\n",
      "66 positive\n",
      "67 positive\n",
      "68 negative\n",
      "69 negative\n",
      "70 negative\n",
      "71 negative\n",
      "72 negative\n",
      "73 negative\n",
      "74 negative\n",
      "75 negative\n",
      "76 negative\n",
      "77 negative\n",
      "78 negative\n",
      "79 negative\n",
      "80 negative\n",
      "81 negative\n",
      "82 negative\n",
      "83 negative\n",
      "84 negative\n",
      "85 negative\n",
      "86 negative\n",
      "87 negative\n",
      "88 negative\n",
      "89 positive\n",
      "90 negative\n",
      "91 negative\n",
      "92 negative\n",
      "93 negative\n",
      "94 negative\n",
      "95 negative\n",
      "96 positive\n",
      "97 negative\n",
      "98 negative\n",
      "99 negative\n",
      "100 negative\n",
      "Total exception count: 0\n",
      "CPU times: user 899 ms, sys: 44.4 ms, total: 944 ms\n",
      "Wall time: 4min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "all_sentiments = []\n",
    "\n",
    "tweets_list = dataset[\"text\"].tolist()\n",
    "\n",
    "i = 0\n",
    "exceptions = 0\n",
    "while i < len(tweets_list):\n",
    "\n",
    "    try:\n",
    "        tweet = tweets_list[i]\n",
    "        sentiment_value = find_sentiment_llama3(tweet)\n",
    "        all_sentiments.append(sentiment_value)\n",
    "        i = i + 1\n",
    "        print(i, sentiment_value)\n",
    "\n",
    "    except Except as e:\n",
    "        print(\"===================\")\n",
    "        print(\"Exception occured\", e)\n",
    "        exception = exception + 1\n",
    "\n",
    "print(\"Total exception count:\", exceptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.78\n"
     ]
    }
   ],
   "source": [
    "accuracy = accuracy_score(all_sentiments, dataset[\"airline_sentiment\"])\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}