04- RQ2 Run CLIP (seperated topics).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "65c606ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "path =  \"/Bridging_the_digital_divide\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "31411d1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torch version: 1.10.2+cu113\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from pkg_resources import packaging\n",
    "from PIL import Image\n",
    "import tqdm\n",
    "from pathlib import Path\n",
    "print(\"Torch version:\", torch.__version__)\n",
    "\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6b9cf677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>country.name</th>\n",
       "      <th>country.id</th>\n",
       "      <th>region.id</th>\n",
       "      <th>type</th>\n",
       "      <th>imageRelPath</th>\n",
       "      <th>topics</th>\n",
       "      <th>place</th>\n",
       "      <th>income</th>\n",
       "      <th>CLIP score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24655</td>\n",
       "      <td>5fd0457ff2d9500e435933ca</td>\n",
       "      <td>Serbia</td>\n",
       "      <td>rs</td>\n",
       "      <td>eu</td>\n",
       "      <td>video</td>\n",
       "      <td>assets/5fd0457ff2d9500e435933ca/6df313d79f9c0f...</td>\n",
       "      <td>Adding spices to food while cooking,Spices</td>\n",
       "      <td>rajkovic</td>\n",
       "      <td>1193.0</td>\n",
       "      <td>0.239534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16463</td>\n",
       "      <td>5fd04872f2d9500e4359465c</td>\n",
       "      <td>South Africa</td>\n",
       "      <td>za</td>\n",
       "      <td>af</td>\n",
       "      <td>video</td>\n",
       "      <td>assets/5fd04872f2d9500e4359465c/37c209cb7f9adf...</td>\n",
       "      <td>Adding spices to food while cooking,Spices</td>\n",
       "      <td>van-rheede</td>\n",
       "      <td>448.0</td>\n",
       "      <td>0.260990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29818</td>\n",
       "      <td>5fd03a0bf2d9500e4358e9a7</td>\n",
       "      <td>Iran</td>\n",
       "      <td>ir</td>\n",
       "      <td>as</td>\n",
       "      <td>video</td>\n",
       "      <td>assets/5fd03a0bf2d9500e4358e9a7/de9e6e8a759f7e...</td>\n",
       "      <td>Adding spices to food while cooking,Spices</td>\n",
       "      <td>ahmadinejad</td>\n",
       "      <td>2369.0</td>\n",
       "      <td>0.309823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13143</td>\n",
       "      <td>5fd025ad04167a0cab36b3a8</td>\n",
       "      <td>Brazil</td>\n",
       "      <td>br</td>\n",
       "      <td>am</td>\n",
       "      <td>video</td>\n",
       "      <td>assets/5fd025ad04167a0cab36b3a8/647b2947695589...</td>\n",
       "      <td>Adding spices to food while cooking,Spices</td>\n",
       "      <td>cancela-reis</td>\n",
       "      <td>284.0</td>\n",
       "      <td>0.226416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28760</td>\n",
       "      <td>5fd02d9ca8444d0db97b4b22</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>co</td>\n",
       "      <td>am</td>\n",
       "      <td>video</td>\n",
       "      <td>assets/5fd02d9ca8444d0db97b4b22/18f3b1e7cb55e1...</td>\n",
       "      <td>Adding spices to food while cooking,Spices</td>\n",
       "      <td>cruz</td>\n",
       "      <td>1963.0</td>\n",
       "      <td>0.284779</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                        id  country.name country.id region.id  \\\n",
       "0       24655  5fd0457ff2d9500e435933ca        Serbia         rs        eu   \n",
       "1       16463  5fd04872f2d9500e4359465c  South Africa         za        af   \n",
       "2       29818  5fd03a0bf2d9500e4358e9a7          Iran         ir        as   \n",
       "3       13143  5fd025ad04167a0cab36b3a8        Brazil         br        am   \n",
       "4       28760  5fd02d9ca8444d0db97b4b22      Colombia         co        am   \n",
       "\n",
       "    type                                       imageRelPath  \\\n",
       "0  video  assets/5fd0457ff2d9500e435933ca/6df313d79f9c0f...   \n",
       "1  video  assets/5fd04872f2d9500e4359465c/37c209cb7f9adf...   \n",
       "2  video  assets/5fd03a0bf2d9500e4358e9a7/de9e6e8a759f7e...   \n",
       "3  video  assets/5fd025ad04167a0cab36b3a8/647b2947695589...   \n",
       "4  video  assets/5fd02d9ca8444d0db97b4b22/18f3b1e7cb55e1...   \n",
       "\n",
       "                                       topics         place  income  \\\n",
       "0  Adding spices to food while cooking,Spices      rajkovic  1193.0   \n",
       "1  Adding spices to food while cooking,Spices    van-rheede   448.0   \n",
       "2  Adding spices to food while cooking,Spices   ahmadinejad  2369.0   \n",
       "3  Adding spices to food while cooking,Spices  cancela-reis   284.0   \n",
       "4  Adding spices to food while cooking,Spices          cruz  1963.0   \n",
       "\n",
       "   CLIP score  \n",
       "0    0.239534  \n",
       "1    0.260990  \n",
       "2    0.309823  \n",
       "3    0.226416  \n",
       "4    0.284779  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/results_CLIP.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a56766e",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_data = 'data/'\n",
    "path_processed_images = \"/\".join([path_data, \"processed_imgs\"])\n",
    "path_original_images = \"data/original_images/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50f2ce51",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48055\n",
      "Counter({'visit': 3366, 'toothbrush': 552, 'home': 542, 'street view': 532, 'hand washing': 530, 'family': 524, 'washing clothes/cleaning': 507, 'tv': 499, 'toilet': 494, 'hand palm': 492, 'drinking water': 488, 'kitchen sink': 480, 'tooth paste': 468, 'stove/hob': 445, 'salt': 438, 'bed': 427, 'spices': 419, 'toys': 418, 'hair brush/comb': 413, 'everyday shoes': 402, 'bedroom': 400, 'front door': 398, 'plate of food': 398, 'power outlet': 389, 'social drink': 385, 'cooking pots': 384, 'floor': 379, 'place where eating dinner': 378, 'nicest shoes': 375, 'phone': 374, 'lock on front door': 373, 'bathroom privacy': 369, 'wardrobe': 365, 'soap for hands and body': 363, 'ceiling': 362, 'hand back': 360, 'wall inside': 360, 'play area': 357, 'wall decoration': 352, 'dish washing brush/cloth': 351, 'bathroom/toilet': 350, 'refrigerator': 344, 'toilet paper': 343, 'plates': 342, 'light sources': 339, 'dish washing soap': 339, 'shampoo': 339, 'dish racks': 338, 'kitchen': 336, 'cups/mugs/glasses': 333, 'armchair': 332, 'trash/waste': 331, 'drying': 330, 'shower': 329, 'teeth': 329, 'light source in livingroom': 320, 'grains': 319, 'books': 315, 'washing detergent': 315, 'switch on/off': 312, 'light source in kitchen': 309, 'couch': 306, ' sofa': 306, 'water outlet': 306, ' drainage': 306, 'roof': 304, 'cooking utensils': 301, 'source of cool': 301, 'cutlery': 301, 'sitting area': 300, 'vegetables': 293, 'medication': 292, 'street detail': 291, 'pen/pencils': 288, 'most loved item': 285, 'most loved toy': 285, 'living room': 279, 'turning lights on and off': 269, 'music equipment': 258, 'wall': 252, 'tools': 251, 'bed kids': 250, 'backyard': 247, 'cleaning equipment': 240, 'table with food': 235, 'smoke and steam exit': 233, 'get water': 232, 'storage room': 226, 'shaving': 217, 'paper': 212, 'fruits and vegetables': 210, 'radio': 210, 'shoes': 210, 'cooking': 208, 'thing i dream about having': 204, 'children room': 198, 'starting stove': 197, 'meat or fish': 193, 'rug': 193, 'jewelry': 189, 'icons': 188, 'pet': 187, 'freezer': 187, 'computer': 186, 'source of heat': 185, 'wall clock': 184, 'washing hands': 184, 'knifes': 181, 'doing dishes': 180, 'brushing teeth': 179, 'pouring water': 178, 'work area': 169, 'preparing food': 169, 'socializing': 166, 'guest bed': 164, 'mosquito protection': 163, 'next big thing you are planning to buy': 153, 'favorite home decorations': 151, 'pouring drinking water': 150, 'bike': 149, 'place where serving guests': 148, 'make up': 148, 'glasses or lenses': 148, 'frontdoor keys': 147, 'oven': 145, 'playing': 141, 'necklaces': 139, 'source of light': 138, 'earings': 136, 'reading a book': 136, 'cleaning floors': 134, 'worship places': 131, 'waste dumps': 127, 'parking lot': 127, 'menstruation pads / tampax': 125, 'eating': 124, 'cosmetics': 123, 'reading': 122, 'chopping food': 111, 'lightsources by bed': 111, ' reading light': 111, 'family snapshots': 109, 'latest furniture bought': 107, 'family eating': 107, 'drinking social drink': 105, 'arm watch': 101, 'fruit trees': 100, 'plugging into and out of power outlet': 98, 'using toilet': 98, 'car': 96, 'taking a teaspoon of salt': 96, 'instrument': 95, ' looking over the shoulder': 91, 'opening the front door': 88, 'brushing hair': 87, 'sitting and watching tv': 84, 'wedding photos': 83, 'opening and closing the freezer': 83, 'walking towards front door': 83, 'moped/motorcycle': 82, 'writing signature': 81, 'walking to get water': 78, 'diapers (or baby-pants)': 77, 'alcoholic drinks': 76, 'writing': 76, 'photo guide images': 75, 'chopping ingredients': 72, 'kids playing inside': 72, 'vegetable plot': 71, 'chickens': 71, 'cleaning after toilet': 70, 'closing the front door': 70, 'favourite item in kitchen': 69, 'piercings': 68, 'adding spices to food while cooking': 67, 'idols': 65, 'pet foods': 65, 'kids playing outside': 60, 'putting on make up': 60, 'car keys': 58, 'fishes': 58, 'sleeping': 57, 'dishwasher': 55, 'worshipping': 54, 'agriculture land': 53, 'toothpaste on toothbrush': 52, 'answering the phone': 51, 'tattoos': 51, 'opening and closing the refrigerator': 50, 'alarm clock': 49, 'hallway': 49, 'hand open to closed': 49, 'dinner guests': 48, 'using most loved item': 48, 'listening to the radio': 48, 'preparing social drink': 47, 'favourite sports clubs': 46, 'wheel barrow': 45, 'how the most loved item is used': 45, 'playing with most loved toy': 45, ' seeing the back of book': 45, 'turn tv on': 43, 'throwing food trash away': 39, 'music idol': 38, 'house overview': 38, 'presenting dollar street': 36, 'wall ': 34, 'playing an instrument': 34, 'turning heater on': 32, 'arm watches': 30, 'cigarettes': 30, 'writing \"home\"': 28, 'coats and jackets': 26, 'contraceptives': 25, 'goats': 24, 'bowls': 23, 'carrying water': 22, 'transport of heavy things': 21, 'go through mail': 21, 'hanging clothes to dry': 18, 'what i wish i could buy': 17, 'surroundings': 13, 'turning fan/ac on': 13, 'cattle': 12, 'clothes': 12, 'snacks': 11, 'ventilation': 10, 'sources of drinking water': 9, 'portraits': 9, 'rehabilitation technology': 8, 'newspapers': 7, 'playgrounds': 7, 'skies outside': 7, 'smoking': 7, 'laying in bed - pretend to sleep': 7, 'water sources for doing dishes': 7, 'bad outdoor air obstructions': 6, 'daylight ostructions': 6, 'markets': 6, 'milk cows or bulls': 6, 'walking through home': 6, 'boat': 5, 'things i wish i had': 5, 'cooking food': 5, 'water sources': 5, 'air cleaning equipments': 4, 'baby powder': 4, 'bed_hq': 4, 'bread - ready': 4, 'opening mail': 4, 'sheep': 4, 'ingredients': 3, 'drinks': 3, 'elevators': 3, 'foodstores': 3, 'horses': 3, 'nature sceneries': 3, 'other transport': 3, 'soccer balls': 3, 'soccer supporter items': 3, 'disability aid': 3, 'replaced': 3, 'bills of money': 2, 'electric wires': 2, 'meat storages': 2, 'other transport ': 2, 'smog/bad air breathing protection': 2, 'tabloids': 2, 'water purifier solutions': 2, 'air fresheners (scents': 1, ' smells)': 1, 'baking sheets': 1, 'baking tables': 1, 'baking tools': 1, 'bread bowls': 1, 'bread ready': 1, 'car decorations': 1, 'celebrity posters': 1, 'coins': 1, 'electricity wires': 1, 'equipment': 1, 'fields': 1, 'fishing equipment': 1, 'horse stables': 1, 'meat markets': 1, 'most played songs on the radio': 1, 'snack stores': 1, 'tractors': 1, 'vegetable markets': 1, 'picking up the phone': 1, 'youth culture': 1})\n",
      "291\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "list_topics = list(data['topics'])\n",
    "\n",
    "\n",
    "separate_topics = [t.lower() for topic in list_topics for t in topic.split(\",\") ]\n",
    "\n",
    "\n",
    "print(len(separate_topics))\n",
    "print(Counter(separate_topics))\n",
    "set_topics = list(set(separate_topics))\n",
    "print(len(set_topics))\n",
    "# print(set_topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2fe73c3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "291\n"
     ]
    }
   ],
   "source": [
    "# map each topic to list of corresponding images\n",
    "dict_topic2img = {}\n",
    "for list_topics, image_path in zip(data['topics'], data['imageRelPath']):\n",
    "    for topic in list_topics.split(\",\"):\n",
    "        topic = topic.lower()\n",
    "        if topic not in dict_topic2img:\n",
    "            dict_topic2img[topic] = set()\n",
    "        dict_topic2img[topic].add(image_path)\n",
    "\n",
    "print(len(dict_topic2img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "831f7644",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: ftfy in /home/jnwatu/.local/lib/python3.9/site-packages (6.1.1)\n",
      "Requirement already satisfied: regex in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (2021.8.3)\n",
      "Requirement already satisfied: tqdm in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (4.62.3)\n",
      "Requirement already satisfied: wcwidth>=0.2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ftfy) (0.2.5)\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting git+https://github.com/openai/CLIP.git\n",
      "  Cloning https://github.com/openai/CLIP.git to /tmp/pip-req-build-g84oyw2s\n",
      "  Running command git clone -q https://github.com/openai/CLIP.git /tmp/pip-req-build-g84oyw2s\n",
      "  Resolved https://github.com/openai/CLIP.git to commit a1d071733d7111c9c014f024669f959182114e33\n",
      "Requirement already satisfied: ftfy in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (6.1.1)\n",
      "Requirement already satisfied: regex in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from clip==1.0) (2021.8.3)\n",
      "Requirement already satisfied: tqdm in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from clip==1.0) (4.62.3)\n",
      "Requirement already satisfied: torch in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (1.10.2+cu113)\n",
      "Requirement already satisfied: torchvision in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (0.11.3+cu113)\n",
      "Requirement already satisfied: wcwidth>=0.2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ftfy->clip==1.0) (0.2.5)\n",
      "Requirement already satisfied: typing-extensions in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from torch->clip==1.0) (3.10.0.2)\n",
      "Requirement already satisfied: numpy in /home/jnwatu/.local/lib/python3.9/site-packages (from torchvision->clip==1.0) (1.22.4)\n",
      "Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from torchvision->clip==1.0) (8.4.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install ftfy regex tqdm\n",
    "! pip install git+https://github.com/openai/CLIP.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a3fce2a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RN50',\n",
       " 'RN101',\n",
       " 'RN50x4',\n",
       " 'RN50x16',\n",
       " 'RN50x64',\n",
       " 'ViT-B/32',\n",
       " 'ViT-B/16',\n",
       " 'ViT-L/14',\n",
       " 'ViT-L/14@336px']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import clip\n",
    "clip.available_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d647f0d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CLIP(\n",
       "  (visual): VisionTransformer(\n",
       "    (conv1): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)\n",
       "    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "    (transformer): Transformer(\n",
       "      (resblocks): Sequential(\n",
       "        (0): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (6): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (7): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (8): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (9): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (10): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (11): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (transformer): Transformer(\n",
       "    (resblocks): Sequential(\n",
       "      (0): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (1): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (2): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (3): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (4): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (5): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (6): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (7): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (8): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (9): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (10): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (11): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (token_embedding): Embedding(49408, 512)\n",
       "  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, preprocess = clip.load(\"ViT-B/32\")\n",
    "\n",
    "model.cuda().eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f166c72f",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2862f03e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "0\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:24<00:00, 24.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:19<00:00, 25.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:27<00:00, 24.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:29<00:00, 23.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:25<00:00, 24.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:52<00:00, 21.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [03:48<00:00, 21.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "3478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3478/3478 [02:39<00:00, 21.80it/s]\n"
     ]
    }
   ],
   "source": [
    "original_images = [path_original_images + s for s in data['imageRelPath']]\n",
    "processed_images = [path_processed_images + \"/\" + img_id + \".npy\" for img_id in data['id']]\n",
    "\n",
    "count = 5000\n",
    "end = len(processed_images) // count + 1\n",
    "print(end)\n",
    "similarity_CLIP = []\n",
    "\n",
    "list_img_features = []\n",
    "for k in range(0, end):\n",
    "    print(k)                    \n",
    "    imgs_corrupted = []\n",
    "    images_prep = []\n",
    "    k_processed_images = processed_images[count*k:count*(k+1)]\n",
    "    print(len(k_processed_images))\n",
    "    \n",
    "    for image_path in tqdm.tqdm(k_processed_images):\n",
    "        if Path(image_path).is_file():         \n",
    "            images_prep.append(torch.from_numpy(np.load(image_path)).to(device))\n",
    "        else:\n",
    "            imgs_corrupted.append(image_path)\n",
    "            print(f\"{image_path} corrupted\")\n",
    "            \n",
    "    images_prep = torch.stack(images_prep)\n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        image_features = model.encode_image(images_prep)\n",
    "        image_features /= image_features.norm(dim=-1, keepdim=True)\n",
    "    list_img_features.append(image_features)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "aa261ad8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([38478, 512])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_features = torch.cat(list_img_features, dim=0)\n",
    "img_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9aad7e92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([291, 512])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    text_tokens = clip.tokenize([desc for desc in dict_topic2img]).cuda()\n",
    "    text_features = model.encode_text(text_tokens).float()\n",
    "    text_features /= text_features.norm(dim=-1, keepdim=True)\n",
    "\n",
    "text_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cf4cad4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(291, 38478)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity = text_features.cpu().numpy() @ img_features.cpu().numpy().T # dot product of image and text features\n",
    "similarity.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8c5ee1bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_topic2score = {}\n",
    "for i, topic in enumerate(dict_topic2img):\n",
    "    images = dict_topic2img[topic]\n",
    "    count = len(images)\n",
    "#     print(topic, count)\n",
    "    clip_scores = similarity[i] # clip scores between topic and images from data['imageRelPath']\n",
    "        \n",
    "    list_scores_img = list(zip(clip_scores, data['imageRelPath'], data['topics']))\n",
    "    dict_topic2score[topic] = sorted(list_scores_img, key = lambda x: x[0], reverse=True)\n",
    "\n",
    "#dict_topic2score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5e6e0f88",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# load pickle module\n",
    "import pickle\n",
    "\n",
    "# create a binary pickle file \n",
    "f = open(\"data/dict_topic2score.pkl\",\"wb\")\n",
    "\n",
    "# write the python object (dict) to pickle file\n",
    "pickle.dump(dict_topic2score,f)\n",
    "\n",
    "# close file\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7746e902",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ba99c85",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}