Face_analysis_in_a_video_call_through_image_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Name: S.Manoj<br>\n",
        "Roll Number: CS20B1018<br>\n",
        "Date: 23-05-2023"
      ],
      "metadata": {
        "id": "fyr-iXqWIytF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import dependencies\n",
        "from IPython.display import display, Javascript, Image\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode, b64encode\n",
        "import cv2\n",
        "import numpy as np\n",
        "import PIL\n",
        "import io\n",
        "import html\n",
        "import time"
      ],
      "metadata": {
        "id": "gR8H2IYFI0cj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# function to convert the JavaScript object into an OpenCV image\n",
        "def js_to_image(js_reply):\n",
        "  \"\"\"\n",
        "  Params:\n",
        "          js_reply: JavaScript object containing image from webcam\n",
        "  Returns:\n",
        "          img: OpenCV BGR image\n",
        "  \"\"\"\n",
        "  # decode base64 image\n",
        "  image_bytes = b64decode(js_reply.split(',')[1])\n",
        "  # convert bytes to numpy array\n",
        "  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)\n",
        "  # decode numpy array into OpenCV BGR image\n",
        "  img = cv2.imdecode(jpg_as_np, flags=1)\n",
        "\n",
        "  return img\n",
        "\n",
        "# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream\n",
        "def bbox_to_bytes(bbox_array):\n",
        "  \"\"\"\n",
        "  Params:\n",
        "          bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.\n",
        "  Returns:\n",
        "        bytes: Base64 image byte string\n",
        "  \"\"\"\n",
        "  # convert array into PIL image\n",
        "  bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')\n",
        "  iobuf = io.BytesIO()\n",
        "  # format bbox into png for return\n",
        "  bbox_PIL.save(iobuf, format='png')\n",
        "  # format return string\n",
        "  bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))\n",
        "\n",
        "  return bbox_bytes"
      ],
      "metadata": {
        "id": "1ZJQVphgpx5D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# initialize the Haar Cascade face detection model\n",
        "face_cascade = cv2.CascadeClassifier(cv2.samples.findFile(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml'))"
      ],
      "metadata": {
        "id": "1F_CGKGGp91W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def take_photo(filename='photo.jpg', quality=0.8):\n",
        "  js = Javascript('''\n",
        "    async function takePhoto(quality) {\n",
        "      const div = document.createElement('div');\n",
        "      const capture = document.createElement('button');\n",
        "      capture.textContent = 'Capture';\n",
        "      div.appendChild(capture);\n",
        "\n",
        "      const video = document.createElement('video');\n",
        "      video.style.display = 'block';\n",
        "      const stream = await navigator.mediaDevices.getUserMedia({video: true});\n",
        "\n",
        "      document.body.appendChild(div);\n",
        "      div.appendChild(video);\n",
        "      video.srcObject = stream;\n",
        "      await video.play();\n",
        "\n",
        "      // Resize the output to fit the video element.\n",
        "      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n",
        "\n",
        "      // Wait for Capture to be clicked.\n",
        "      await new Promise((resolve) => capture.onclick = resolve);\n",
        "\n",
        "      const canvas = document.createElement('canvas');\n",
        "      canvas.width = video.videoWidth;\n",
        "      canvas.height = video.videoHeight;\n",
        "      canvas.getContext('2d').drawImage(video, 0, 0);\n",
        "      stream.getVideoTracks()[0].stop();\n",
        "      div.remove();\n",
        "      return canvas.toDataURL('image/jpeg', quality);\n",
        "    }\n",
        "    ''')\n",
        "  display(js)\n",
        "\n",
        "  # get photo data\n",
        "  data = eval_js('takePhoto({})'.format(quality))\n",
        "  # get OpenCV format image\n",
        "  img = js_to_image(data) \n",
        "  # grayscale img\n",
        "  gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n",
        "  print(gray.shape)\n",
        "  # get face bounding box coordinates using Haar Cascade\n",
        "  faces = face_cascade.detectMultiScale(gray)\n",
        "  # draw face bounding box on image\n",
        "  for (x,y,w,h) in faces:\n",
        "      img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
        "  # save image\n",
        "  cv2.imwrite(filename, img)\n",
        "\n",
        "  return filename"
      ],
      "metadata": {
        "id": "SO7aYxAMqA_J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  filename = take_photo('photo.jpg')\n",
        "  print('Saved to {}'.format(filename))\n",
        "  \n",
        "  # Show the image which was just taken.\n",
        "  display(Image(filename))\n",
        "except Exception as err:\n",
        "  # Errors will be thrown if the user does not have a webcam or if they do not\n",
        "  # grant the page permission to access it.\n",
        "  print(str(err))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Gew9MSEqGu3",
        "outputId": "849a2c0b-44f4-49bf-c96a-c3134c6a1265"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "name 'take_photo' is not defined\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# JavaScript to properly create our live video stream using our webcam as input\n",
        "def video_stream():\n",
        "  js = Javascript('''\n",
        "    var video;\n",
        "    var div = null;\n",
        "    var stream;\n",
        "    var captureCanvas;\n",
        "    var imgElement;\n",
        "    var labelElement;\n",
        "    \n",
        "    var pendingResolve = null;\n",
        "    var shutdown = false;\n",
        "    \n",
        "    function removeDom() {\n",
        "       stream.getVideoTracks()[0].stop();\n",
        "       video.remove();\n",
        "       div.remove();\n",
        "       video = null;\n",
        "       div = null;\n",
        "       stream = null;\n",
        "       imgElement = null;\n",
        "       captureCanvas = null;\n",
        "       labelElement = null;\n",
        "    }\n",
        "    \n",
        "    function onAnimationFrame() {\n",
        "      if (!shutdown) {\n",
        "        window.requestAnimationFrame(onAnimationFrame);\n",
        "      }\n",
        "      if (pendingResolve) {\n",
        "        var result = \"\";\n",
        "        if (!shutdown) {\n",
        "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
        "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
        "        }\n",
        "        var lp = pendingResolve;\n",
        "        pendingResolve = null;\n",
        "        lp(result);\n",
        "      }\n",
        "    }\n",
        "    \n",
        "    async function createDom() {\n",
        "      if (div !== null) {\n",
        "        return stream;\n",
        "      }\n",
        "\n",
        "      div = document.createElement('div');\n",
        "      div.style.border = '2px solid black';\n",
        "      div.style.padding = '3px';\n",
        "      div.style.width = '100%';\n",
        "      div.style.maxWidth = '600px';\n",
        "      document.body.appendChild(div);\n",
        "      \n",
        "      const modelOut = document.createElement('div');\n",
        "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
        "      labelElement = document.createElement('span');\n",
        "      labelElement.innerText = 'No data';\n",
        "      labelElement.style.fontWeight = 'bold';\n",
        "      modelOut.appendChild(labelElement);\n",
        "      div.appendChild(modelOut);\n",
        "           \n",
        "      video = document.createElement('video');\n",
        "      video.style.display = 'block';\n",
        "      video.width = div.clientWidth - 6;\n",
        "      video.setAttribute('playsinline', '');\n",
        "      video.onclick = () => { shutdown = true; };\n",
        "      stream = await navigator.mediaDevices.getUserMedia(\n",
        "          {video: { facingMode: \"environment\"}});\n",
        "      div.appendChild(video);\n",
        "\n",
        "      imgElement = document.createElement('img');\n",
        "      imgElement.style.position = 'absolute';\n",
        "      imgElement.style.zIndex = 1;\n",
        "      imgElement.onclick = () => { shutdown = true; };\n",
        "      div.appendChild(imgElement);\n",
        "      \n",
        "      const instruction = document.createElement('div');\n",
        "      instruction.innerHTML = \n",
        "          '<span style=\"color: red; font-weight: bold;\">' +\n",
        "          'When finished, click here or on the video to stop this demo</span>';\n",
        "      div.appendChild(instruction);\n",
        "      instruction.onclick = () => { shutdown = true; };\n",
        "      \n",
        "      video.srcObject = stream;\n",
        "      await video.play();\n",
        "\n",
        "      captureCanvas = document.createElement('canvas');\n",
        "      captureCanvas.width = 640; //video.videoWidth;\n",
        "      captureCanvas.height = 480; //video.videoHeight;\n",
        "      window.requestAnimationFrame(onAnimationFrame);\n",
        "      \n",
        "      return stream;\n",
        "    }\n",
        "    async function stream_frame(label, imgData) {\n",
        "      if (shutdown) {\n",
        "        removeDom();\n",
        "        shutdown = false;\n",
        "        return '';\n",
        "      }\n",
        "\n",
        "      var preCreate = Date.now();\n",
        "      stream = await createDom();\n",
        "      \n",
        "      var preShow = Date.now();\n",
        "      if (label != \"\") {\n",
        "        labelElement.innerHTML = label;\n",
        "      }\n",
        "            \n",
        "      if (imgData != \"\") {\n",
        "        var videoRect = video.getClientRects()[0];\n",
        "        imgElement.style.top = videoRect.top + \"px\";\n",
        "        imgElement.style.left = videoRect.left + \"px\";\n",
        "        imgElement.style.width = videoRect.width + \"px\";\n",
        "        imgElement.style.height = videoRect.height + \"px\";\n",
        "        imgElement.src = imgData;\n",
        "      }\n",
        "      \n",
        "      var preCapture = Date.now();\n",
        "      var result = await new Promise(function(resolve, reject) {\n",
        "        pendingResolve = resolve;\n",
        "      });\n",
        "      shutdown = false;\n",
        "      \n",
        "      return {'create': preShow - preCreate, \n",
        "              'show': preCapture - preShow, \n",
        "              'capture': Date.now() - preCapture,\n",
        "              'img': result};\n",
        "    }\n",
        "    ''')\n",
        "\n",
        "  display(js)\n",
        "  \n",
        "def video_frame(label, bbox):\n",
        "  data = eval_js('stream_frame(\"{}\", \"{}\")'.format(label, bbox))\n",
        "  return data"
      ],
      "metadata": {
        "id": "5ryKGFCmqJId"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# start streaming video from webcam\n",
        "video_stream()\n",
        "# label for video\n",
        "label_html = 'Capturing...'\n",
        "# initialze bounding box to empty\n",
        "bbox = ''\n",
        "count = 0 \n",
        "while True:\n",
        "    js_reply = video_frame(label_html, bbox)\n",
        "    if not js_reply:\n",
        "        break\n",
        "\n",
        "    # convert JS response to OpenCV Image\n",
        "    img = js_to_image(js_reply[\"img\"])\n",
        "\n",
        "    # create transparent overlay for bounding box\n",
        "    bbox_array = np.zeros([480,640,4], dtype=np.uint8)\n",
        "\n",
        "    # grayscale image for face detection\n",
        "    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n",
        "\n",
        "    # get face region coordinates\n",
        "    faces = face_cascade.detectMultiScale(gray)\n",
        "    # get face bounding box for overlay\n",
        "    for (x,y,w,h) in faces:\n",
        "      bbox_array = cv2.rectangle(bbox_array,(x,y),(x+w,y+h),(255,0,0),2)\n",
        "\n",
        "    bbox_array[:,:,3] = (bbox_array.max(axis = 2) > 0 ).astype(int) * 255\n",
        "    # convert overlay of bbox into bytes\n",
        "    bbox_bytes = bbox_to_bytes(bbox_array)\n",
        "    # update bbox so next frame gets new overlay\n",
        "    bbox = bbox_bytes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "JGCC_7qmqVF4",
        "outputId": "62f958b0-97cb-44ee-a1bd-f165e6e58029"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    var video;\n",
              "    var div = null;\n",
              "    var stream;\n",
              "    var captureCanvas;\n",
              "    var imgElement;\n",
              "    var labelElement;\n",
              "    \n",
              "    var pendingResolve = null;\n",
              "    var shutdown = false;\n",
              "    \n",
              "    function removeDom() {\n",
              "       stream.getVideoTracks()[0].stop();\n",
              "       video.remove();\n",
              "       div.remove();\n",
              "       video = null;\n",
              "       div = null;\n",
              "       stream = null;\n",
              "       imgElement = null;\n",
              "       captureCanvas = null;\n",
              "       labelElement = null;\n",
              "    }\n",
              "    \n",
              "    function onAnimationFrame() {\n",
              "      if (!shutdown) {\n",
              "        window.requestAnimationFrame(onAnimationFrame);\n",
              "      }\n",
              "      if (pendingResolve) {\n",
              "        var result = \"\";\n",
              "        if (!shutdown) {\n",
              "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
              "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
              "        }\n",
              "        var lp = pendingResolve;\n",
              "        pendingResolve = null;\n",
              "        lp(result);\n",
              "      }\n",
              "    }\n",
              "    \n",
              "    async function createDom() {\n",
              "      if (div !== null) {\n",
              "        return stream;\n",
              "      }\n",
              "\n",
              "      div = document.createElement('div');\n",
              "      div.style.border = '2px solid black';\n",
              "      div.style.padding = '3px';\n",
              "      div.style.width = '100%';\n",
              "      div.style.maxWidth = '600px';\n",
              "      document.body.appendChild(div);\n",
              "      \n",
              "      const modelOut = document.createElement('div');\n",
              "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
              "      labelElement = document.createElement('span');\n",
              "      labelElement.innerText = 'No data';\n",
              "      labelElement.style.fontWeight = 'bold';\n",
              "      modelOut.appendChild(labelElement);\n",
              "      div.appendChild(modelOut);\n",
              "           \n",
              "      video = document.createElement('video');\n",
              "      video.style.display = 'block';\n",
              "      video.width = div.clientWidth - 6;\n",
              "      video.setAttribute('playsinline', '');\n",
              "      video.onclick = () => { shutdown = true; };\n",
              "      stream = await navigator.mediaDevices.getUserMedia(\n",
              "          {video: { facingMode: \"environment\"}});\n",
              "      div.appendChild(video);\n",
              "\n",
              "      imgElement = document.createElement('img');\n",
              "      imgElement.style.position = 'absolute';\n",
              "      imgElement.style.zIndex = 1;\n",
              "      imgElement.onclick = () => { shutdown = true; };\n",
              "      div.appendChild(imgElement);\n",
              "      \n",
              "      const instruction = document.createElement('div');\n",
              "      instruction.innerHTML = \n",
              "          '<span style=\"color: red; font-weight: bold;\">' +\n",
              "          'When finished, click here or on the video to stop this demo</span>';\n",
              "      div.appendChild(instruction);\n",
              "      instruction.onclick = () => { shutdown = true; };\n",
              "      \n",
              "      video.srcObject = stream;\n",
              "      await video.play();\n",
              "\n",
              "      captureCanvas = document.createElement('canvas');\n",
              "      captureCanvas.width = 640; //video.videoWidth;\n",
              "      captureCanvas.height = 480; //video.videoHeight;\n",
              "      window.requestAnimationFrame(onAnimationFrame);\n",
              "      \n",
              "      return stream;\n",
              "    }\n",
              "    async function stream_frame(label, imgData) {\n",
              "      if (shutdown) {\n",
              "        removeDom();\n",
              "        shutdown = false;\n",
              "        return '';\n",
              "      }\n",
              "\n",
              "      var preCreate = Date.now();\n",
              "      stream = await createDom();\n",
              "      \n",
              "      var preShow = Date.now();\n",
              "      if (label != \"\") {\n",
              "        labelElement.innerHTML = label;\n",
              "      }\n",
              "            \n",
              "      if (imgData != \"\") {\n",
              "        var videoRect = video.getClientRects()[0];\n",
              "        imgElement.style.top = videoRect.top + \"px\";\n",
              "        imgElement.style.left = videoRect.left + \"px\";\n",
              "        imgElement.style.width = videoRect.width + \"px\";\n",
              "        imgElement.style.height = videoRect.height + \"px\";\n",
              "        imgElement.src = imgData;\n",
              "      }\n",
              "      \n",
              "      var preCapture = Date.now();\n",
              "      var result = await new Promise(function(resolve, reject) {\n",
              "        pendingResolve = resolve;\n",
              "      });\n",
              "      shutdown = false;\n",
              "      \n",
              "      return {'create': preShow - preCreate, \n",
              "              'show': preCapture - preShow, \n",
              "              'capture': Date.now() - preCapture,\n",
              "              'img': result};\n",
              "    }\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "VAJYl52kuNtP"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}