Semantic_Search_with_Cohere_and_HuggingFace_Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyP00ega91IHfbQ21kxXZhpu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "de69f2cb4e614f31867ebf9ab6735c6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bc1e646b34274a13971c7669b0316ab6",
              "IPY_MODEL_62ff3a2420744d04b10f35e0287d6e62",
              "IPY_MODEL_57d6dd08ae714182ba77dbc3304d8f0c"
            ],
            "layout": "IPY_MODEL_e2ce9510d30049ff93880264e9a961c2"
          }
        },
        "bc1e646b34274a13971c7669b0316ab6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3edb8f5902df45e68c83874571535009",
            "placeholder": "​",
            "style": "IPY_MODEL_15d4a3de4b194468becbed47f4736790",
            "value": "Downloading data: 100%"
          }
        },
        "62ff3a2420744d04b10f35e0287d6e62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dbc86bcad2794edc8136eb1053ae46d0",
            "max": 59405,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_be41d091fdc542a7b556813e976aa151",
            "value": 59405
          }
        },
        "57d6dd08ae714182ba77dbc3304d8f0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b395b01303c48ddb9754966814c011b",
            "placeholder": "​",
            "style": "IPY_MODEL_6f27438133844d7c9674e68be5f05567",
            "value": " 59.4k/59.4k [00:00&lt;00:00, 124kB/s]"
          }
        },
        "e2ce9510d30049ff93880264e9a961c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3edb8f5902df45e68c83874571535009": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "15d4a3de4b194468becbed47f4736790": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dbc86bcad2794edc8136eb1053ae46d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be41d091fdc542a7b556813e976aa151": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4b395b01303c48ddb9754966814c011b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f27438133844d7c9674e68be5f05567": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e842f29534f40e5929edb87ca145410": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8ad7ba42af0b4b078619642388bd1fd1",
              "IPY_MODEL_dbc9709ac7c4435a8d851885f0737467",
              "IPY_MODEL_3aae7b1d26bf43aba306372643eac2f6"
            ],
            "layout": "IPY_MODEL_8def410291794da6be09595330601d1d"
          }
        },
        "8ad7ba42af0b4b078619642388bd1fd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4923a8bbbfb450f97513ce235918ce2",
            "placeholder": "​",
            "style": "IPY_MODEL_040057e269d54a9aa54f98f607578296",
            "value": "Generating train split: 100%"
          }
        },
        "dbc9709ac7c4435a8d851885f0737467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3160875cc6af4073bae0070ab9439a5d",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_631c64be251645fb981597c5d630a85e",
            "value": 1000
          }
        },
        "3aae7b1d26bf43aba306372643eac2f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfe7a3efe15b430b999744c03c56a6ab",
            "placeholder": "​",
            "style": "IPY_MODEL_b3fccc805003439f98f64bdd2d308ee1",
            "value": " 1000/1000 [00:00&lt;00:00, 6752.86 examples/s]"
          }
        },
        "8def410291794da6be09595330601d1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4923a8bbbfb450f97513ce235918ce2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "040057e269d54a9aa54f98f607578296": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3160875cc6af4073bae0070ab9439a5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "631c64be251645fb981597c5d630a85e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dfe7a3efe15b430b999744c03c56a6ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3fccc805003439f98f64bdd2d308ee1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Eddiebee/AI-Craft/blob/main/Semantic_Search_with_Cohere_and_HuggingFace_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Semantic Search with Cohere and HuggingFace Dataset\n",
        "\n",
        "Text embeddings at various areas of application, but most especially we see them excelling in their application to search tasks. By virtue of the semantic information contained in the embedding we have a search experience that goes beyond the traditional key-word search, but takes into consideration the semantics of the search query and matches this to the best fit in the search documents.\n",
        "\n",
        "We'll be building on my previous notebook on [Text Embedding using Cohere and HuggingFace Dataset.](https://colab.research.google.com/drive/1y69Hy4hM1hGStPq8eeZ7Ey2jQmB7fh8y?usp=sharing)\n",
        "\n",
        "Leggo! 🚀"
      ],
      "metadata": {
        "id": "aRCcP4AStj4u"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup\n",
        "\n",
        "We'll kickoff by installing and importing the required packages."
      ],
      "metadata": {
        "id": "y-JP9rn4vSDC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install cohere datasets -q"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CMNr-YSpvRbp",
        "outputId": "5b7f7127-c23a-4043-84da-b0e94e175852"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m151.2/151.2 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.2/401.2 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "JQiIGSJdtY7z"
      },
      "outputs": [],
      "source": [
        "import cohere\n",
        "from datasets import load_dataset\n",
        "import pandas as pd\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have to initialize Cohere with an API Key.\n",
        "\n",
        "If you don't have an account with Cohere yet, please [sign up for one over here](https://dashboard.cohere.com/welcome/register). NO CREDIT CARD REQUIRED! 🤗"
      ],
      "metadata": {
        "id": "038J2LXjwTcx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "I have mine stored inside my Colab secrets. Loading it becomes a breeze. 💨"
      ],
      "metadata": {
        "id": "qmsFrQ5txHE5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import userdata\n",
        "COHERE_API_KEY = userdata.get('COHERE_TRIAL_KEY')"
      ],
      "metadata": {
        "id": "PUSfxoX3wRQi"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# initialize Cohere using your API key\n",
        "co = cohere.Client(COHERE_API_KEY)"
      ],
      "metadata": {
        "id": "8hf2QS6lxWwr"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll throw in the code needed to turn the [email intent classification dataset's](https://huggingface.co/datasets/aadilsayad/email-intent-classification/) prompt column to text embeddings in the next code cell."
      ],
      "metadata": {
        "id": "jVqzLv0yxep2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load email intent dataset\n",
        "dataset = load_dataset(\"aadilsayad/email-intent-classification\")\n",
        "\n",
        "# define get_embeddings function\n",
        "def get_embeddings(texts,\n",
        "                   model=\"embed-english-v3.0\",\n",
        "                   input_type=\"search_document\"):\n",
        "\n",
        "  response = co.embed(\n",
        "        model=model,\n",
        "        input_type=input_type,\n",
        "        texts=texts)\n",
        "\n",
        "  return response.embeddings\n",
        "\n",
        "# embed dataset\n",
        "dataset[\"train\"] = dataset[\"train\"].add_column(name=\"prompt_embeddings\",\n",
        "                   column=get_embeddings(texts=dataset[\"train\"][\"prompt\"]))\n",
        "\n",
        "# print out our column names\n",
        "dataset.column_names"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 99,
          "referenced_widgets": [
            "de69f2cb4e614f31867ebf9ab6735c6a",
            "bc1e646b34274a13971c7669b0316ab6",
            "62ff3a2420744d04b10f35e0287d6e62",
            "57d6dd08ae714182ba77dbc3304d8f0c",
            "e2ce9510d30049ff93880264e9a961c2",
            "3edb8f5902df45e68c83874571535009",
            "15d4a3de4b194468becbed47f4736790",
            "dbc86bcad2794edc8136eb1053ae46d0",
            "be41d091fdc542a7b556813e976aa151",
            "4b395b01303c48ddb9754966814c011b",
            "6f27438133844d7c9674e68be5f05567",
            "0e842f29534f40e5929edb87ca145410",
            "8ad7ba42af0b4b078619642388bd1fd1",
            "dbc9709ac7c4435a8d851885f0737467",
            "3aae7b1d26bf43aba306372643eac2f6",
            "8def410291794da6be09595330601d1d",
            "e4923a8bbbfb450f97513ce235918ce2",
            "040057e269d54a9aa54f98f607578296",
            "3160875cc6af4073bae0070ab9439a5d",
            "631c64be251645fb981597c5d630a85e",
            "dfe7a3efe15b430b999744c03c56a6ab",
            "b3fccc805003439f98f64bdd2d308ee1"
          ]
        },
        "id": "U-4UfcJwxahN",
        "outputId": "320dc033-9e81-4d1d-d8ae-9387bcf07338"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/59.4k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "de69f2cb4e614f31867ebf9ab6735c6a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/1000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e842f29534f40e5929edb87ca145410"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'train': ['prompt', 'label', 'prompt_embeddings']}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Notice that we set the `input_type` parameter to `search_document` because the texts (documents) we embed will be stored in a vector database.\n",
        "\n",
        "`How about an article on this`? 🤔"
      ],
      "metadata": {
        "id": "BZSoMt2ayu3L"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Applied Text Embeddings - Semantic Search\n",
        "\n",
        "This section, we'll apply this text embeddings in performming the task of semantic search."
      ],
      "metadata": {
        "id": "ufJnOkvrzm9N"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Embed the search prompt\n",
        "\n",
        "We have to first of all embed the search query.\n",
        "\n",
        "To do this, we'll make use of our `get_embeddings` function.\n",
        "\n",
        "Leggo! 🚀"
      ],
      "metadata": {
        "id": "TlNhEwAA0CYB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# define search prompt\n",
        "new_prompt = \"I would love to send this email to the following contacts.\"\n",
        "# new_prompt = \"I would love to share this email to the following contacts.\"\n"
      ],
      "metadata": {
        "id": "xPRJcRZSysZi"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_prompt_embeddings = get_embeddings([new_prompt],\n",
        "                                       input_type=\"search_query\")[0]\n",
        "\n",
        "# we use input_type=\"search_query\" because we're using the text to find\n",
        "# the most relevant documents in your vector database."
      ],
      "metadata": {
        "id": "B8ml2xXi078X"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Compare to Embedded Documents"
      ],
      "metadata": {
        "id": "vE7MYyrD1-o5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity"
      ],
      "metadata": {
        "id": "ooIeeZd1CekF"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_embeddings = dataset[\"train\"][\"prompt_embeddings\"]\n",
        "len(prompt_embeddings)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oZCauDu_Cv73",
        "outputId": "71c6fe65-a021-4bf0-9aea-36f222f9d2c0"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1000"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# set SAMPLE size\n",
        "SAMPLE = 9\n",
        "\n",
        "# calculate cosine similarity between the search query and existing queries\n",
        "def get_similarity(target, candidates):\n",
        "    # turn list into array\n",
        "    candidates = np.array(candidates)\n",
        "    target = np.expand_dims(np.array(target),axis=0)\n",
        "\n",
        "    # calculate cosine similarity\n",
        "    sim = cosine_similarity(target, candidates)\n",
        "    sim = np.squeeze(sim).tolist()\n",
        "    sort_index = np.argsort(sim)[::-1]\n",
        "    sort_score = [sim[i] for i in sort_index]\n",
        "    similarity_scores = zip(sort_index,sort_score)\n",
        "\n",
        "    # return similarity scores\n",
        "    return similarity_scores\n",
        "\n",
        "# get the similarity between the search query and existing queries\n",
        "similarity = get_similarity(new_prompt_embeddings, prompt_embeddings[:SAMPLE])"
      ],
      "metadata": {
        "id": "W93XgkKq1pta"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "similarity = list(similarity)"
      ],
      "metadata": {
        "id": "d4U_RcriFM40"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "similarity"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SMnR-pSzIIN0",
        "outputId": "1b8a7873-f2bf-4ca9-cb0c-2620510bd30b"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(7, 0.669374261590068),\n",
              " (1, 0.6447462950277745),\n",
              " (2, 0.6209104441785771),\n",
              " (5, 0.6050300403484374),\n",
              " (0, 0.5967863521460304),\n",
              " (3, 0.5457825971152368),\n",
              " (4, 0.5264902235388096),\n",
              " (6, 0.5123163466083849),\n",
              " (8, 0.4838913311261836)]"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# view the top 5 articles\n",
        "print(\"Prompt: \")\n",
        "print(new_prompt,\"\\n\")\n",
        "\n",
        "print(\"Most Similar Documents:\")\n",
        "for idx, sim in (similarity):\n",
        "  print(f\"Similarity: {sim:.2f};\", dataset[\"train\"][\"prompt\"][idx])"
      ],
      "metadata": {
        "id": "laSy_27S1shK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "29082886-5db0-4946-857c-3cfe87cbfddb"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: \n",
            "I would love to send this email to the following contacts. \n",
            "\n",
            "Most Similar Documents:\n",
            "Similarity: 0.67; I want to email someone.\n",
            "Similarity: 0.64; I'd like to compose an email.\n",
            "Similarity: 0.62; I need to send an email.\n",
            "Similarity: 0.61; Let's write an email.\n",
            "Similarity: 0.60; Can I send an email, please?\n",
            "Similarity: 0.55; Could you help me write an email?\n",
            "Similarity: 0.53; Is it possible to send an email with you?\n",
            "Similarity: 0.51; Time to send an email.\n",
            "Similarity: 0.48; Open email for writing.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DAFtuIzqHs1l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "PEcLG9itD54E"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}