colab_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t29woLVqstOf"
      },
      "source": [
        "# EyeFormer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6qkOH_v4stOg"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -qq ruamel.yaml==0.17.21"
      ],
      "metadata": {
        "id": "3Sdf35adtNi2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad686c20-9e99-4ce5-f607-bd236bf1fd04"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.5/109.5 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.7/526.7 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "btSItrlhstOg"
      },
      "outputs": [],
      "source": [
        "import argparse\n",
        "import os\n",
        "import ruamel.yaml as yaml\n",
        "import numpy as np\n",
        "import random\n",
        "import time\n",
        "import datetime\n",
        "import json\n",
        "from pathlib import Path\n",
        "import csv\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.backends.cudnn as cudnn\n",
        "import torch.distributed as dist"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E4_aTEuwstOg"
      },
      "source": [
        "Clone Eyeformer & Create the environment"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pytorch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 cudatoolkit==11.0\n",
        "!pip install opencv-python==4.5.3.56 Pillow einops multimatch-gaze\n",
        "!pip install transformers==4.8.1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dkZHo55qvCaZ",
        "outputId": "1219a727-2a5f-46e9-a417-58499838a594"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: Could not find a version that satisfies the requirement pytorch==1.7.1 (from versions: 0.1.2, 1.0.2)\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[31mERROR: No matching distribution found for pytorch==1.7.1\u001b[0m\u001b[31m\n",
            "\u001b[0mCollecting opencv-python==4.5.3.56\n",
            "  Downloading opencv-python-4.5.3.56.tar.gz (89.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpip subprocess to install build dependencies\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25herror\n",
            "\u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "\n",
            "\u001b[31m×\u001b[0m \u001b[32mpip subprocess to install build dependencies\u001b[0m did not run successfully.\n",
            "\u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "\u001b[31m╰─>\u001b[0m See above for output.\n",
            "\n",
            "\u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "Collecting transformers==4.8.1\n",
            "  Downloading transformers-4.8.1-py3-none-any.whl.metadata (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.8/48.8 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (3.15.4)\n",
            "Collecting huggingface-hub==0.0.12 (from transformers==4.8.1)\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl.metadata (5.6 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (1.26.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (24.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (2024.5.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (2.31.0)\n",
            "Collecting sacremoses (from transformers==4.8.1)\n",
            "  Downloading sacremoses-0.1.1-py3-none-any.whl.metadata (8.3 kB)\n",
            "Collecting tokenizers<0.11,>=0.10.1 (from transformers==4.8.1)\n",
            "  Downloading tokenizers-0.10.3.tar.gz (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.7/212.7 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.8.1) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from huggingface-hub==0.0.12->transformers==4.8.1) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.8.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.8.1) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.8.1) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.8.1) (2024.7.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from sacremoses->transformers==4.8.1) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from sacremoses->transformers==4.8.1) (1.4.2)\n",
            "Downloading transformers-4.8.1-py3-none-any.whl (2.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m43.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Downloading sacremoses-0.1.1-py3-none-any.whl (897 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m897.5/897.5 kB\u001b[0m \u001b[31m38.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: tokenizers\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mBuilding wheel for tokenizers \u001b[0m\u001b[1;32m(\u001b[0m\u001b[32mpyproject.toml\u001b[0m\u001b[1;32m)\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for tokenizers (pyproject.toml) ... \u001b[?25l\u001b[?25herror\n",
            "\u001b[31m  ERROR: Failed building wheel for tokenizers\u001b[0m\u001b[31m\n",
            "\u001b[0mFailed to build tokenizers\n",
            "\u001b[31mERROR: ERROR: Failed to build installable wheels for some pyproject.toml based projects (tokenizers)\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install timm==0.4.9"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zY4wii6lv0yJ",
        "outputId": "5add99fc-47e4-4d58-afa6-23df3522a49a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting timm==0.4.9\n",
            "  Downloading timm-0.4.9-py3-none-any.whl.metadata (27 kB)\n",
            "Requirement already satisfied: torch>=1.4 in /usr/local/lib/python3.10/dist-packages (from timm==0.4.9) (2.3.1+cu121)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from timm==0.4.9) (0.18.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (1.13.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (2024.6.1)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.4->timm==0.4.9)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4->timm==0.4.9) (2.3.1)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.4->timm==0.4.9)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision->timm==0.4.9) (1.26.4)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->timm==0.4.9) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.4->timm==0.4.9) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.4->timm==0.4.9) (1.3.0)\n",
            "Downloading timm-0.4.9-py3-none-any.whl (346 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m346.1/346.1 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, timm\n",
            "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.82 nvidia-nvtx-cu12-12.1.105 timm-0.4.9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2SxZn6IMstOg",
        "outputId": "9ad81b66-fdc9-460f-a604-47e4e3e6df9b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'EyeFormer-UIST2024'...\n",
            "remote: Enumerating objects: 98, done.\u001b[K\n",
            "remote: Counting objects: 100% (98/98), done.\u001b[K\n",
            "remote: Compressing objects: 100% (77/77), done.\u001b[K\n",
            "remote: Total 98 (delta 24), reused 90 (delta 18), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (98/98), 335.72 KiB | 9.87 MiB/s, done.\n",
            "Resolving deltas: 100% (24/24), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/YueJiang-nj/EyeFormer-UIST2024.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd EyeFormer-UIST2024"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QF3YyyUUuftA",
        "outputId": "e49d3446-f227-4035-e8f1-10a4ecc20790"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/EyeFormer-UIST2024\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **NOTE**\n",
        "\n",
        "Please change the `eval_image_root` string in `configs/Tracking.yaml` to your eval image dir"
      ],
      "metadata": {
        "id": "mYjiRznmxLtj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "Nh40dex3stOh"
      },
      "outputs": [],
      "source": [
        "import gdown\n",
        "url = 'https://drive.google.com/drive/folders/1eae63VRBfQ9XAYj3k2q2N9xDtLmSqQg_'\n",
        "gdown.download_folder(url, quiet=True)\n",
        "!mv scanpath_prediction_population_rl weights"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "dJY2ladlstOh"
      },
      "outputs": [],
      "source": [
        "from models.model_tracking import TrackingTransformer\n",
        "from models.vit import interpolate_pos_embed\n",
        "\n",
        "import utils\n",
        "from dataset import create_dataset, create_sampler, create_loader\n",
        "import csv"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vFrOM84ustOh"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "KXu75aZPstOh"
      },
      "outputs": [],
      "source": [
        "def test(model, data_loader, tokenizer, device, output_dir, config):\n",
        "    # train\n",
        "    model.eval()\n",
        "\n",
        "    metric_logger = utils.MetricLogger(delimiter=\"  \")\n",
        "    header = 'Testing:'\n",
        "\n",
        "    image_names = []\n",
        "    results = []\n",
        "    widths = []\n",
        "    heights = []\n",
        "    # user_ids = []\n",
        "    for i, (image, image_name, width, height) in enumerate(metric_logger.log_every(data_loader, 1, header)):\n",
        "        image = image.to(device, non_blocking=True)\n",
        "        # user_id = user_id.to(device, non_blocking=True)\n",
        "\n",
        "        coord = model.inference(image)\n",
        "        coord = coord.cpu().numpy().tolist()\n",
        "\n",
        "        width = width.numpy().tolist()\n",
        "        height = height.numpy().tolist()\n",
        "\n",
        "        # user_id = user_id.cpu().numpy().tolist()\n",
        "\n",
        "        image_names.extend(image_name)\n",
        "        results.extend(coord)\n",
        "        widths.extend(width)\n",
        "        heights.extend(height)\n",
        "        # user_ids.extend(user_id)\n",
        "\n",
        "    with open(os.path.join(output_dir, 'predicted_result.csv'), 'w') as wfile:\n",
        "        writer = csv.writer(wfile)\n",
        "        writer.writerow([\"image\", \"width\", \"height\", \"x\", \"y\", \"timestamp\"])\n",
        "\n",
        "        for image, width, height, coord in zip(image_names, widths, heights, results):\n",
        "\n",
        "            for row in coord:\n",
        "                x = row[0] * width\n",
        "                y = row[1] * height\n",
        "                t = row[2]\n",
        "                # username = data_loader.dataset.id2user[user_id]\n",
        "                writer.writerow([image, width, height,\n",
        "                                x, y, t])\n",
        "\n",
        "    return"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "aGtJPjU8stOh"
      },
      "outputs": [],
      "source": [
        "def main(args, config):\n",
        "    # utils.init_distributed_mode(args)\n",
        "\n",
        "    device = torch.device(args.device)\n",
        "\n",
        "    # fix the seed for reproducibility\n",
        "    seed = args.seed + utils.get_rank()\n",
        "    torch.manual_seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    random.seed(seed)\n",
        "    cudnn.benchmark = True\n",
        "\n",
        "    #### Dataset ####\n",
        "    print(\"Creating dataset\")\n",
        "    datasets = [create_dataset('inference', config)]\n",
        "\n",
        "    if False:\n",
        "        num_tasks = utils.get_world_size()\n",
        "        global_rank = utils.get_rank()\n",
        "        samplers = create_sampler(datasets, [True], num_tasks, global_rank)\n",
        "    else:\n",
        "        samplers = [None]\n",
        "\n",
        "    data_loader = create_loader(datasets,\n",
        "                                samplers,\n",
        "                                batch_size=[config['batch_size_test']],\n",
        "                                num_workers=[32],\n",
        "                                is_trains=[False],\n",
        "                                collate_fns=[None])[0]\n",
        "\n",
        "    # tokenizer = BertTokenizer.from_pretrained(args.text_encoder)\n",
        "    tokenizer = None\n",
        "\n",
        "    #### Model ####\n",
        "    print(\"Creating model\")\n",
        "    model = TrackingTransformer(config=config, init_deit=False)\n",
        "\n",
        "    model = model.to(device)\n",
        "\n",
        "    if args.checkpoint:\n",
        "        checkpoint = torch.load(args.checkpoint, map_location='cpu')\n",
        "        state_dict = checkpoint['model']\n",
        "\n",
        "        msg = model.load_state_dict(state_dict)\n",
        "        print('load checkpoint from %s' % args.checkpoint)\n",
        "        print(msg)\n",
        "\n",
        "    model_without_ddp = model\n",
        "\n",
        "    print(\"Start testing\")\n",
        "    start_time = time.time()\n",
        "\n",
        "    test(model, data_loader, tokenizer, device, args.output_dir, config)\n",
        "\n",
        "    # dist.barrier()\n",
        "\n",
        "    total_time = time.time() - start_time\n",
        "    total_time_str = str(datetime.timedelta(seconds=int(total_time)))\n",
        "    print('Testing time {}'.format(total_time_str))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "EXfTMtBDstOh"
      },
      "outputs": [],
      "source": [
        "class ARGS:\n",
        "    config = './configs/Tracking.yaml'\n",
        "    checkpoint = './weights/checkpoint_19.pth'\n",
        "    resume = False\n",
        "    output_dir = 'output/tracking_eval'\n",
        "    text_encoder = 'bert-base-uncased'\n",
        "    device ='cuda'\n",
        "    seed = 42\n",
        "    world_size = 1\n",
        "    dist_url = 'env://'\n",
        "    distributed = True"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from ruamel.yaml import YAML"
      ],
      "metadata": {
        "id": "cxCNgH4F09in"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "id": "HQTLur4YstOh",
        "outputId": "0ce16496-7f04-4500-a550-ff7c40587719"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Not using distributed mode\n",
            "Creating dataset\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "[Errno 2] No such file or directory: '/home/researcher/Documents/saliency_modeling_issues/UEyes_dataset/images'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-5b868cfce248>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmkdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0myaml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'config.yaml'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-18-6c612877015b>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(args, config)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;31m#### Dataset ####\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Creating dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mdatasets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcreate_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'inference'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistributed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/EyeFormer-UIST2024/dataset/__init__.py\u001b[0m in \u001b[0;36mcreate_dataset\u001b[0;34m(dataset, config)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'inference'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtracking_dataset_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eval_image_root'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracking_transform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"max_words\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/EyeFormer-UIST2024/dataset/coord_dataset.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, image_root, transform, max_words)\u001b[0m\n\u001b[1;32m    319\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mtracking_dataset_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimage_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 321\u001b[0;31m         self.image_list = [os.path.join(image_root, r) for r in os.listdir(image_root)\\\n\u001b[0m\u001b[1;32m    322\u001b[0m                            if (os.path.isfile(os.path.join(image_root, r)) and (r.endswith(\".png\") or r.endswith(\".jpg\")))]\n\u001b[1;32m    323\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimage_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimage_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/researcher/Documents/saliency_modeling_issues/UEyes_dataset/images'"
          ]
        }
      ],
      "source": [
        "args = ARGS()\n",
        "\n",
        "yaml = YAML(typ='rt')\n",
        "config = yaml.load(open(args.config, 'rt'))\n",
        "\n",
        "Path(args.output_dir).mkdir(parents=True, exist_ok=True)\n",
        "yaml.dump(config, open(os.path.join(args.output_dir, 'config.yaml'), 'w'))\n",
        "main(args, config)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1s73yrVz0_8g"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "language_info": {
      "name": "python"
    },
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}