nuscenes_main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import time\n",
    "\n",
    "from Nuscenes_dataset_test import NuscenesDatasetTest\n",
    "from Model import EnDeWithPooling, EnDeConvLSTM_ws, SkipLSTMEnDe\n",
    "from torchvision import transforms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveTransformedImages(imageTensor):\n",
    "    to_pil = torchvision.transforms.ToPILImage()\n",
    "    im = to_pil(imageTensor)\n",
    "    mn, mx = np.min(im), np.max(im)\n",
    "    im = (im - mn) / (mx - mn)\n",
    "    print(im)\n",
    "    plt.imshow(im, cmap='gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti, seqLen, im_path, numFrames=None):\n",
    "    fig = plt.figure(figsize=(8, 8))\n",
    "    plt.plot(yValsGT, xValsGT, c='r', label='Ground Truth')\n",
    "    plt.plot(yValsPred, xValsPred, c='g', label='Prediction')\n",
    "    plt.plot(yValsPredMulti, xValsPredMulti, c='b', label='Multimodal Prediction', alpha=0.8)\n",
    "    axes = plt.gca()\n",
    "    axes.set_xlim([1, 256])\n",
    "    axes.set_ylim([1, 256])\n",
    "    plt.xlabel('X-Axis')\n",
    "    plt.ylabel('Y-Axis')\n",
    "    plt.legend(loc='upper right')\n",
    "    if numFrames == None:\n",
    "        plt.title('Trajectory')\n",
    "    else:\n",
    "        plot_title = 'Trajectory (' + str(numFrames // 10 - 2) + \"s)\"\n",
    "        plt.title(plot_title)\n",
    "    plt.savefig(im_path)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heatmapAccuracy(outputMap, labelMap, thr=1.5):\n",
    "    pred = np.unravel_index(outputMap.argmax(), outputMap.shape)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "\n",
    "    dist = math.sqrt((pred[0] - gt[0]) ** 2 + (pred[1] - gt[1]) ** 2)\n",
    "    if dist <= thr:\n",
    "        return 1, dist, (pred[0], pred[1]), (gt[0], gt[1])\n",
    "    return 0, dist, (pred[0], pred[1]), (gt[0], gt[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def largest_indices(ary, n):\n",
    "    \"\"\"Returns the n largest indices from a numpy array.\"\"\"\n",
    "    flat = ary.flatten()\n",
    "    indices = np.argpartition(flat, -n)[-n:]\n",
    "    indices = indices[np.argsort(-flat[indices])]\n",
    "    return np.unravel_index(indices, ary.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiAccuracy(outputMap, labelMap, topK=5):\n",
    "    pred = largest_indices(outputMap, topK)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "    dist_arr = []\n",
    "    for i in range(len(pred[0])):\n",
    "        dist = math.sqrt((pred[0][i] - gt[0]) ** 2 + (pred[1][i] - gt[1]) ** 2)\n",
    "        dist_arr.append(dist)\n",
    "    \n",
    "    min_val = np.min(dist_arr)\n",
    "    min_idx = np.argmin(dist_arr)\n",
    "    within_radius = 0\n",
    "    if min_val <= 4:\n",
    "        within_radius = 1\n",
    "    return 0, min_val, (pred[0][min_idx], pred[1][min_idx]), (gt[0], gt[1]), within_radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_default_tensor_type(torch.cuda.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Cross Validation No"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path = \"/home/sas115/trajectory_prediction_INFER-master/ablation_cache_nuscenes/skipLSTM/split-0/checkpoint_future_best.tar\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose the corresponding model for each split\n",
    "# checkpoint_path = os.path.join(repo_dir, \"models\", \"kitti-main\", \"cv-0\", \"checkpoint_future.tar\")\n",
    "# checkpoint_path = os.path.join(repo_dir, \"models\", \"kitti-main\", \"cv-1\", \"checkpoint_future.tar\")\n",
    "# checkpoint_path = os.path.join(repo_dir, \"models\", \"kitti-main\", \"cv-2\", \"checkpoint_future.tar\")\n",
    "# checkpoint_path = os.path.join(repo_dir, \"models\", \"kitti-main\", \"cv-3\", \"checkpoint_future.tar\")\n",
    "# checkpoint_path = os.path.join(repo_dir, \"models\", \"kitti-main\", \"cv-4\", \"checkpoint_future.tar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load(checkpoint_path)\n",
    "model = SkipLSTMEnDe(activation=\"relu\", initType=\"default\", numChannels=5, imageHeight=256, imageWidth=256, batchnorm=False, softmax=False)\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "model = model.cuda()\n",
    "model.convlstm = model.convlstm.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_dir = \"/home/fbd/rrc/submission/INFER-datasets/kitti\"\n",
    "# val_dir = os.path.join(data_dir, \"final-validation\", \"test\" + str(cv_num) + \".csv\")\n",
    "# val_dataset = KittiDataset(data_dir, height=256, width=256, train=False, infoPath=val_dir, augmentation=False, groundTruth=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/sas115/trajectory_prediction_INFER-master/nuScenes_project_dataset_test\"\n",
    "val_dataset = NuscenesDatasetTest(data_dir, height=256, width=256,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Future Prediction (Final, Validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "upsample_512 = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n",
    "labelTransform = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "\n",
    "valLoss1, valLoss2, valLoss3, valLoss4, valLoss = [], [], [], [], []\n",
    "topK = 5\n",
    "totalPreds = 0\n",
    "hitPreds = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scene: 44.0, Sequence: 1.0, Frame: 11, Seq Loss: 18.863141051654853\n",
      "Scene: 44.0, Sequence: 2.0, Frame: 11, Seq Loss: 17.051666480941222\n",
      "Scene: 44.0, Sequence: 3.0, Frame: 11, Seq Loss: 13.065673497156007\n",
      "Scene: 44.0, Sequence: 4.0, Frame: 11, Seq Loss: 18.870043407738603\n",
      "Scene: 44.0, Sequence: 5.0, Frame: 11, Seq Loss: 17.10087016907076\n",
      "Scene: 44.0, Sequence: 6.0, Frame: 11, Seq Loss: 24.861222851055068\n",
      "Scene: 44.0, Sequence: 7.0, Frame: 11, Seq Loss: 28.187009557687553\n",
      "Scene: 44.0, Sequence: 8.0, Frame: 11, Seq Loss: 23.88066000181337\n",
      "Scene: 44.0, Sequence: 9.0, Frame: 11, Seq Loss: 20.33354319797276\n",
      "Scene: 44.0, Sequence: 10.0, Frame: 11, Seq Loss: 28.88684620209215\n",
      "Scene: 44.0, Sequence: 11.0, Frame: 11, Seq Loss: 24.038254719523742\n",
      "Scene: 44.0, Sequence: 12.0, Frame: 11, Seq Loss: 29.07378766858003\n",
      "Scene: 44.0, Sequence: 13.0, Frame: 11, Seq Loss: 29.242405934600594\n",
      "Scene: 44.0, Sequence: 14.0, Frame: 11, Seq Loss: 24.46201954806481\n",
      "Scene: 44.0, Sequence: 15.0, Frame: 11, Seq Loss: 37.87362584501312\n",
      "Scene: 44.0, Sequence: 16.0, Frame: 11, Seq Loss: 53.78121725989321\n",
      "Scene: 44.0, Sequence: 17.0, Frame: 11, Seq Loss: 31.440836913552744\n",
      "Scene: 44.0, Sequence: 18.0, Frame: 11, Seq Loss: 31.031973932710443\n",
      "Scene: 44.0, Sequence: 19.0, Frame: 11, Seq Loss: 62.953935831626644\n",
      "Scene: 44.0, Sequence: 20.0, Frame: 11, Seq Loss: 53.870627272082956\n",
      "Scene: 44.0, Sequence: 21.0, Frame: 11, Seq Loss: 20.024416691379283\n",
      "Scene: 44.0, Sequence: 22.0, Frame: 11, Seq Loss: 23.261560671964958\n",
      "Scene: 44.0, Sequence: 23.0, Frame: 11, Seq Loss: 19.908314407423156\n",
      "Scene: 44.0, Sequence: 24.0, Frame: 11, Seq Loss: 88.0408292523184\n",
      "Scene: 44.0, Sequence: 25.0, Frame: 11, Seq Loss: 44.94869782210963\n",
      "Scene: 44.0, Sequence: 26.0, Frame: 11, Seq Loss: 52.97454779355523\n",
      "Scene: 45.0, Sequence: 1.0, Frame: 11, Seq Loss: 18.099858524035458\n",
      "Scene: 45.0, Sequence: 2.0, Frame: 11, Seq Loss: 14.532706172706567\n",
      "Scene: 45.0, Sequence: 3.0, Frame: 11, Seq Loss: 8.647508927531367\n",
      "Scene: 45.0, Sequence: 4.0, Frame: 11, Seq Loss: 8.745663541882909\n",
      "Scene: 45.0, Sequence: 5.0, Frame: 11, Seq Loss: 14.907492432706318\n",
      "Scene: 45.0, Sequence: 6.0, Frame: 11, Seq Loss: 16.18185361454108\n",
      "Scene: 45.0, Sequence: 7.0, Frame: 11, Seq Loss: 23.313872345494516\n",
      "Scene: 45.0, Sequence: 8.0, Frame: 11, Seq Loss: 19.75326804207341\n",
      "Scene: 45.0, Sequence: 9.0, Frame: 11, Seq Loss: 20.868244567254273\n",
      "Scene: 45.0, Sequence: 10.0, Frame: 11, Seq Loss: 26.615363625129618\n",
      "Scene: 45.0, Sequence: 11.0, Frame: 11, Seq Loss: 25.550042834544843\n",
      "Scene: 45.0, Sequence: 12.0, Frame: 11, Seq Loss: 19.44654038856357\n",
      "Scene: 45.0, Sequence: 13.0, Frame: 11, Seq Loss: 19.90806956972748\n",
      "Scene: 45.0, Sequence: 14.0, Frame: 11, Seq Loss: 18.35068452644302\n",
      "Scene: 45.0, Sequence: 15.0, Frame: 11, Seq Loss: 21.21079752354649\n",
      "Scene: 45.0, Sequence: 16.0, Frame: 11, Seq Loss: 19.530267187053497\n",
      "Scene: 45.0, Sequence: 17.0, Frame: 11, Seq Loss: 20.274868936003344\n",
      "Scene: 45.0, Sequence: 18.0, Frame: 11, Seq Loss: 9.060623231929767\n",
      "Scene: 45.0, Sequence: 19.0, Frame: 11, Seq Loss: 27.307209387542677\n",
      "Scene: 45.0, Sequence: 20.0, Frame: 11, Seq Loss: 14.588339707777568\n",
      "Scene: 45.0, Sequence: 21.0, Frame: 11, Seq Loss: 14.95622165046415\n",
      "Scene: 45.0, Sequence: 22.0, Frame: 11, Seq Loss: 7.97308725226365\n",
      "Scene: 45.0, Sequence: 23.0, Frame: 11, Seq Loss: 6.9039548356136855\n",
      "Scene: 45.0, Sequence: 24.0, Frame: 11, Seq Loss: 2.732804352645269\n",
      "Scene: 45.0, Sequence: 25.0, Frame: 11, Seq Loss: 7.318496562628854\n",
      "Scene: 45.0, Sequence: 26.0, Frame: 11, Seq Loss: 10.422277069178655\n",
      "Scene: 46.0, Sequence: 1.0, Frame: 11, Seq Loss: 14.08912740271094\n",
      "Scene: 46.0, Sequence: 2.0, Frame: 11, Seq Loss: 21.94167148060447\n",
      "Scene: 46.0, Sequence: 3.0, Frame: 11, Seq Loss: 6.897530695110768\n",
      "Scene: 46.0, Sequence: 4.0, Frame: 11, Seq Loss: 6.218081542984821\n",
      "Scene: 46.0, Sequence: 5.0, Frame: 11, Seq Loss: 5.487802363405741\n",
      "Scene: 46.0, Sequence: 6.0, Frame: 11, Seq Loss: 6.433801212557199\n",
      "Scene: 46.0, Sequence: 7.0, Frame: 11, Seq Loss: 3.4057764064044154\n",
      "Scene: 46.0, Sequence: 8.0, Frame: 11, Seq Loss: 7.020159657736417\n",
      "Scene: 46.0, Sequence: 9.0, Frame: 11, Seq Loss: 7.549429299735285\n",
      "Scene: 46.0, Sequence: 10.0, Frame: 11, Seq Loss: 2.235476532860451\n",
      "Scene: 46.0, Sequence: 11.0, Frame: 11, Seq Loss: 1.9542197524582479\n",
      "Scene: 46.0, Sequence: 12.0, Frame: 11, Seq Loss: 8.003366622655717\n",
      "Scene: 46.0, Sequence: 13.0, Frame: 11, Seq Loss: 7.924393039069608\n",
      "Scene: 46.0, Sequence: 14.0, Frame: 11, Seq Loss: 7.548000316840781\n",
      "Scene: 46.0, Sequence: 15.0, Frame: 11, Seq Loss: 22.80952727636162\n",
      "Scene: 46.0, Sequence: 16.0, Frame: 11, Seq Loss: 44.62001319039755\n",
      "Scene: 46.0, Sequence: 17.0, Frame: 11, Seq Loss: 29.621188936119136\n",
      "Scene: 46.0, Sequence: 18.0, Frame: 11, Seq Loss: 22.727962895100635\n",
      "Scene: 46.0, Sequence: 19.0, Frame: 11, Seq Loss: 29.024293082285897\n",
      "Scene: 46.0, Sequence: 20.0, Frame: 11, Seq Loss: 26.038421758736646\n",
      "Scene: 46.0, Sequence: 21.0, Frame: 11, Seq Loss: 54.745689096341835\n",
      "Scene: 46.0, Sequence: 22.0, Frame: 11, Seq Loss: 22.802215126945693\n",
      "Scene: 46.0, Sequence: 23.0, Frame: 11, Seq Loss: 55.01654470534554\n",
      "Scene: 46.0, Sequence: 24.0, Frame: 11, Seq Loss: 70.09717487205552\n",
      "Scene: 46.0, Sequence: 25.0, Frame: 11, Seq Loss: 76.92258887710251\n",
      "Scene: 46.0, Sequence: 26.0, Frame: 11, Seq Loss: 23.306470370215823\n",
      "Scene: 47.0, Sequence: 1.0, Frame: 11, Seq Loss: 9.524643333589902\n",
      "Scene: 47.0, Sequence: 2.0, Frame: 11, Seq Loss: 10.410689155085944\n",
      "Scene: 47.0, Sequence: 3.0, Frame: 11, Seq Loss: 4.0329780443524665\n",
      "Scene: 47.0, Sequence: 4.0, Frame: 11, Seq Loss: 5.929260968447564\n",
      "Scene: 47.0, Sequence: 5.0, Frame: 11, Seq Loss: 3.000176046435593\n",
      "Scene: 47.0, Sequence: 6.0, Frame: 11, Seq Loss: 2.182773143051522\n",
      "Scene: 47.0, Sequence: 7.0, Frame: 11, Seq Loss: 1.4571067811865475\n",
      "Scene: 47.0, Sequence: 8.0, Frame: 11, Seq Loss: 1.7140674110214293\n",
      "Scene: 47.0, Sequence: 9.0, Frame: 11, Seq Loss: 1.8952847075210475\n",
      "Scene: 47.0, Sequence: 10.0, Frame: 11, Seq Loss: 2.316799212912266\n",
      "Scene: 47.0, Sequence: 11.0, Frame: 11, Seq Loss: 5.65711543589281\n",
      "Scene: 47.0, Sequence: 12.0, Frame: 11, Seq Loss: 1.375\n",
      "Scene: 47.0, Sequence: 13.0, Frame: 11, Seq Loss: 4.6767766952966365\n",
      "Scene: 47.0, Sequence: 14.0, Frame: 11, Seq Loss: 8.460925765359242\n",
      "Scene: 47.0, Sequence: 15.0, Frame: 11, Seq Loss: 18.11253259804188\n",
      "Scene: 47.0, Sequence: 16.0, Frame: 11, Seq Loss: 10.365310440602695\n",
      "Scene: 47.0, Sequence: 17.0, Frame: 11, Seq Loss: 19.173711680143843\n",
      "Scene: 47.0, Sequence: 18.0, Frame: 11, Seq Loss: 18.162879242718187\n",
      "Scene: 47.0, Sequence: 19.0, Frame: 11, Seq Loss: 22.743109392306994\n",
      "Scene: 47.0, Sequence: 20.0, Frame: 11, Seq Loss: 20.953119349128677\n",
      "Scene: 47.0, Sequence: 21.0, Frame: 11, Seq Loss: 37.835879740979976\n",
      "Scene: 47.0, Sequence: 22.0, Frame: 11, Seq Loss: 37.15824031352369\n",
      "Scene: 47.0, Sequence: 23.0, Frame: 11, Seq Loss: 40.40142630775388\n",
      "Scene: 47.0, Sequence: 24.0, Frame: 11, Seq Loss: 43.791624312438024\n",
      "Scene: 47.0, Sequence: 25.0, Frame: 11, Seq Loss: 27.700094339704513\n",
      "Scene: 47.0, Sequence: 26.0, Frame: 11, Seq Loss: 16.491368413867235\n",
      "Scene: 48.0, Sequence: 1.0, Frame: 11, Seq Loss: 10.71998711790132\n",
      "Scene: 48.0, Sequence: 2.0, Frame: 11, Seq Loss: 9.011205902900182\n",
      "Scene: 48.0, Sequence: 3.0, Frame: 11, Seq Loss: 8.558525464156435\n",
      "Scene: 48.0, Sequence: 4.0, Frame: 11, Seq Loss: 5.078627066676104\n",
      "Scene: 48.0, Sequence: 5.0, Frame: 11, Seq Loss: 2.9562851924841107\n",
      "Scene: 48.0, Sequence: 6.0, Frame: 11, Seq Loss: 4.478881767405279\n",
      "Scene: 48.0, Sequence: 7.0, Frame: 11, Seq Loss: 6.625\n",
      "Scene: 48.0, Sequence: 8.0, Frame: 11, Seq Loss: 2.2795084971874737\n",
      "Scene: 48.0, Sequence: 9.0, Frame: 11, Seq Loss: 5.2682633965639925\n",
      "Scene: 48.0, Sequence: 10.0, Frame: 11, Seq Loss: 1.926776695296637\n",
      "Scene: 48.0, Sequence: 11.0, Frame: 11, Seq Loss: 3.7216733956863184\n",
      "Scene: 48.0, Sequence: 12.0, Frame: 11, Seq Loss: 3.9828409671268683\n",
      "Scene: 48.0, Sequence: 13.0, Frame: 11, Seq Loss: 4.13251059871976\n",
      "Scene: 48.0, Sequence: 14.0, Frame: 11, Seq Loss: 6.130670127148408\n",
      "Scene: 48.0, Sequence: 15.0, Frame: 11, Seq Loss: 9.921900086740688\n",
      "Scene: 48.0, Sequence: 16.0, Frame: 11, Seq Loss: 13.245035114235357\n",
      "Scene: 48.0, Sequence: 17.0, Frame: 11, Seq Loss: 8.538351265604597\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scene: 48.0, Sequence: 18.0, Frame: 11, Seq Loss: 11.713130071595804\n",
      "Scene: 48.0, Sequence: 19.0, Frame: 11, Seq Loss: 9.943978831174121\n",
      "Scene: 48.0, Sequence: 20.0, Frame: 11, Seq Loss: 18.48655106339094\n",
      "Scene: 48.0, Sequence: 21.0, Frame: 11, Seq Loss: 8.485792129122839\n",
      "Scene: 48.0, Sequence: 22.0, Frame: 11, Seq Loss: 8.330818966704374\n",
      "Scene: 48.0, Sequence: 23.0, Frame: 11, Seq Loss: 11.953490677190317\n",
      "Scene: 48.0, Sequence: 24.0, Frame: 11, Seq Loss: 17.022283788318944\n",
      "Scene: 48.0, Sequence: 25.0, Frame: 11, Seq Loss: 25.718979425183264\n",
      "Scene: 48.0, Sequence: 26.0, Frame: 11, Seq Loss: 14.394732339738983\n",
      "Scene: 49.0, Sequence: 1.0, Frame: 11, Seq Loss: 14.987533170640413\n",
      "Scene: 49.0, Sequence: 2.0, Frame: 11, Seq Loss: 10.804497871008717\n",
      "Scene: 49.0, Sequence: 3.0, Frame: 11, Seq Loss: 6.5573584813073476\n",
      "Scene: 49.0, Sequence: 4.0, Frame: 11, Seq Loss: 4.700535882715187\n",
      "Scene: 49.0, Sequence: 5.0, Frame: 11, Seq Loss: 3.686186517852847\n",
      "Scene: 49.0, Sequence: 6.0, Frame: 11, Seq Loss: 3.11554772290775\n",
      "Scene: 49.0, Sequence: 7.0, Frame: 11, Seq Loss: 5.103553390593273\n",
      "Scene: 49.0, Sequence: 8.0, Frame: 11, Seq Loss: 16.46665878243399\n",
      "Scene: 49.0, Sequence: 9.0, Frame: 11, Seq Loss: 10.708909556245457\n",
      "Scene: 49.0, Sequence: 10.0, Frame: 11, Seq Loss: 11.923966119094182\n",
      "Scene: 49.0, Sequence: 11.0, Frame: 11, Seq Loss: 11.444186524991261\n",
      "Scene: 49.0, Sequence: 12.0, Frame: 11, Seq Loss: 20.958181662549777\n",
      "Scene: 49.0, Sequence: 13.0, Frame: 11, Seq Loss: 31.087782278913913\n",
      "Scene: 49.0, Sequence: 14.0, Frame: 11, Seq Loss: 21.91686122572347\n",
      "Scene: 49.0, Sequence: 15.0, Frame: 11, Seq Loss: 18.2354508007122\n",
      "Scene: 49.0, Sequence: 16.0, Frame: 11, Seq Loss: 18.691320261872516\n",
      "Scene: 49.0, Sequence: 17.0, Frame: 11, Seq Loss: 16.593311302743487\n",
      "Scene: 49.0, Sequence: 18.0, Frame: 11, Seq Loss: 15.280301782863354\n",
      "Scene: 49.0, Sequence: 19.0, Frame: 11, Seq Loss: 6.754770132907032\n",
      "Scene: 49.0, Sequence: 20.0, Frame: 11, Seq Loss: 9.037754026248678\n",
      "Scene: 49.0, Sequence: 21.0, Frame: 11, Seq Loss: 6.242220545972735\n",
      "Scene: 49.0, Sequence: 22.0, Frame: 11, Seq Loss: 8.988627968328501\n",
      "Scene: 49.0, Sequence: 23.0, Frame: 11, Seq Loss: 16.099432622124073\n",
      "Scene: 49.0, Sequence: 24.0, Frame: 11, Seq Loss: 11.01473816607653\n",
      "Scene: 49.0, Sequence: 25.0, Frame: 11, Seq Loss: 17.187482848570436\n",
      "Scene: 49.0, Sequence: 26.0, Frame: 11, Seq Loss: 12.205049406691275\n",
      "Scene: 50.0, Sequence: 1.0, Frame: 11, Seq Loss: 25.118128134800827\n",
      "Scene: 50.0, Sequence: 2.0, Frame: 11, Seq Loss: 30.645897664070503\n",
      "Scene: 50.0, Sequence: 3.0, Frame: 11, Seq Loss: 32.36187273469493\n",
      "Scene: 50.0, Sequence: 4.0, Frame: 11, Seq Loss: 56.715463750525494\n",
      "Scene: 50.0, Sequence: 5.0, Frame: 11, Seq Loss: 21.069268271673884\n",
      "Scene: 50.0, Sequence: 6.0, Frame: 11, Seq Loss: 20.228752088265637\n",
      "Scene: 50.0, Sequence: 7.0, Frame: 11, Seq Loss: 31.729190084197874\n",
      "Scene: 50.0, Sequence: 8.0, Frame: 11, Seq Loss: 63.07670396728071\n",
      "Scene: 50.0, Sequence: 9.0, Frame: 11, Seq Loss: 52.182157505276116\n",
      "Scene: 50.0, Sequence: 10.0, Frame: 11, Seq Loss: 66.5284382738903\n",
      "Scene: 50.0, Sequence: 11.0, Frame: 11, Seq Loss: 135.40276947467606\n",
      "Scene: 50.0, Sequence: 12.0, Frame: 11, Seq Loss: 115.63696973217384\n",
      "Scene: 50.0, Sequence: 13.0, Frame: 11, Seq Loss: 39.99853898671574\n",
      "Scene: 50.0, Sequence: 14.0, Frame: 11, Seq Loss: 35.767594996983775\n",
      "Scene: 50.0, Sequence: 15.0, Frame: 11, Seq Loss: 20.51100287442678\n",
      "Scene: 50.0, Sequence: 16.0, Frame: 11, Seq Loss: 50.49942755341343\n",
      "Scene: 50.0, Sequence: 17.0, Frame: 11, Seq Loss: 20.385818731860702\n",
      "Scene: 50.0, Sequence: 18.0, Frame: 11, Seq Loss: 18.531807075235488\n",
      "Scene: 50.0, Sequence: 19.0, Frame: 11, Seq Loss: 44.188930201655786\n",
      "Scene: 50.0, Sequence: 20.0, Frame: 11, Seq Loss: 23.72395898886759\n",
      "Scene: 50.0, Sequence: 21.0, Frame: 11, Seq Loss: 16.390896887396934\n",
      "Scene: 50.0, Sequence: 22.0, Frame: 11, Seq Loss: 41.31066330085147\n",
      "Scene: 50.0, Sequence: 23.0, Frame: 11, Seq Loss: 20.595600360920347\n",
      "Scene: 50.0, Sequence: 24.0, Frame: 11, Seq Loss: 22.127419569773814\n",
      "Scene: 50.0, Sequence: 25.0, Frame: 11, Seq Loss: 38.872962507573746\n",
      "Scene: 50.0, Sequence: 26.0, Frame: 11, Seq Loss: 35.72360400167641\n",
      "Scene: 51.0, Sequence: 1.0, Frame: 11, Seq Loss: 28.83181732753247\n",
      "Scene: 51.0, Sequence: 2.0, Frame: 11, Seq Loss: 20.19051530412028\n",
      "Scene: 51.0, Sequence: 3.0, Frame: 11, Seq Loss: 33.136306860983474\n",
      "Scene: 51.0, Sequence: 4.0, Frame: 11, Seq Loss: 26.823095563223475\n",
      "Scene: 51.0, Sequence: 5.0, Frame: 11, Seq Loss: 20.996333508062445\n",
      "Scene: 51.0, Sequence: 6.0, Frame: 11, Seq Loss: 12.667375759098686\n",
      "Scene: 51.0, Sequence: 7.0, Frame: 11, Seq Loss: 23.17909606709871\n",
      "Scene: 51.0, Sequence: 8.0, Frame: 11, Seq Loss: 55.56779859980046\n",
      "Scene: 51.0, Sequence: 9.0, Frame: 11, Seq Loss: 15.66942446011326\n",
      "Scene: 51.0, Sequence: 10.0, Frame: 11, Seq Loss: 16.391571458636733\n",
      "Scene: 51.0, Sequence: 11.0, Frame: 11, Seq Loss: 31.921323255270345\n",
      "Scene: 51.0, Sequence: 12.0, Frame: 11, Seq Loss: 17.265241560219685\n",
      "Scene: 51.0, Sequence: 13.0, Frame: 11, Seq Loss: 30.583226731140286\n",
      "Scene: 51.0, Sequence: 14.0, Frame: 11, Seq Loss: 18.454948393681573\n",
      "Scene: 51.0, Sequence: 15.0, Frame: 11, Seq Loss: 39.21833835527246\n",
      "Scene: 51.0, Sequence: 16.0, Frame: 11, Seq Loss: 55.10619972241264\n",
      "Scene: 51.0, Sequence: 17.0, Frame: 11, Seq Loss: 67.19715841628414\n",
      "Scene: 51.0, Sequence: 18.0, Frame: 11, Seq Loss: 17.646106521251404\n",
      "Scene: 51.0, Sequence: 19.0, Frame: 11, Seq Loss: 20.18439327486513\n",
      "Scene: 51.0, Sequence: 20.0, Frame: 11, Seq Loss: 32.848009944973974\n",
      "Scene: 51.0, Sequence: 21.0, Frame: 11, Seq Loss: 23.013110744406323\n",
      "Scene: 51.0, Sequence: 22.0, Frame: 11, Seq Loss: 26.064513111692605\n",
      "Scene: 51.0, Sequence: 23.0, Frame: 11, Seq Loss: 97.35931194471291\n",
      "Scene: 51.0, Sequence: 24.0, Frame: 11, Seq Loss: 120.79862817140976\n",
      "Scene: 51.0, Sequence: 25.0, Frame: 11, Seq Loss: 103.164328798474\n",
      "Scene: 51.0, Sequence: 26.0, Frame: 11, Seq Loss: 90.83271425149574\n",
      "Scene: 52.0, Sequence: 1.0, Frame: 11, Seq Loss: 6.253333359668634\n",
      "Scene: 52.0, Sequence: 2.0, Frame: 11, Seq Loss: 7.772747778318438\n",
      "Scene: 52.0, Sequence: 3.0, Frame: 11, Seq Loss: 8.74831491562563\n",
      "Scene: 52.0, Sequence: 4.0, Frame: 11, Seq Loss: 10.952916376186906\n",
      "Scene: 52.0, Sequence: 5.0, Frame: 11, Seq Loss: 18.757062204644598\n",
      "Scene: 52.0, Sequence: 6.0, Frame: 11, Seq Loss: 14.419265430985726\n",
      "Scene: 52.0, Sequence: 7.0, Frame: 11, Seq Loss: 12.862709878087234\n",
      "Scene: 52.0, Sequence: 8.0, Frame: 11, Seq Loss: 28.954441856193597\n",
      "Scene: 52.0, Sequence: 9.0, Frame: 11, Seq Loss: 14.252433090873133\n",
      "Scene: 52.0, Sequence: 10.0, Frame: 11, Seq Loss: 5.096012740111778\n",
      "Scene: 52.0, Sequence: 11.0, Frame: 11, Seq Loss: 13.612288757756126\n",
      "Scene: 52.0, Sequence: 12.0, Frame: 11, Seq Loss: 7.608971728808651\n",
      "Scene: 52.0, Sequence: 13.0, Frame: 11, Seq Loss: 8.284865956177292\n",
      "Scene: 52.0, Sequence: 14.0, Frame: 11, Seq Loss: 6.423384047985186\n",
      "Scene: 52.0, Sequence: 15.0, Frame: 11, Seq Loss: 18.645418108501886\n",
      "Scene: 52.0, Sequence: 16.0, Frame: 11, Seq Loss: 5.253610163454119\n",
      "Scene: 52.0, Sequence: 17.0, Frame: 11, Seq Loss: 9.09555015534057\n",
      "Scene: 52.0, Sequence: 18.0, Frame: 11, Seq Loss: 10.90361166015256\n",
      "Scene: 52.0, Sequence: 19.0, Frame: 11, Seq Loss: 8.94396978734159\n",
      "Scene: 52.0, Sequence: 20.0, Frame: 11, Seq Loss: 4.430561853346276\n",
      "Scene: 52.0, Sequence: 21.0, Frame: 11, Seq Loss: 21.702626946445893\n",
      "Scene: 52.0, Sequence: 22.0, Frame: 11, Seq Loss: 21.483702830048387\n",
      "Scene: 52.0, Sequence: 23.0, Frame: 11, Seq Loss: 8.526114266961917\n",
      "Scene: 52.0, Sequence: 24.0, Frame: 11, Seq Loss: 10.328563881006373\n",
      "Scene: 52.0, Sequence: 25.0, Frame: 11, Seq Loss: 8.749087818792455\n",
      "Scene: 52.0, Sequence: 26.0, Frame: 11, Seq Loss: 7.29111150879365\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "debug, prevOut, state = True, None, None\n",
    "prevChannels = None\n",
    "xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti = [], [], [], [], [], []\n",
    "seqLoss, seqVals = [], []\n",
    "seqNum, seqLen = 0, 0\n",
    "\n",
    "start_time = time.time()\n",
    "model.eval()\n",
    "new_seq_loss = []\n",
    "\n",
    "for i in range(len(val_dataset)):\n",
    "    grid, sceneNum, seqNum, frame_num, endOfSequence = val_dataset[i]\n",
    "    grid = grid.type(torch.cuda.FloatTensor)\n",
    "    \n",
    "    # The Last Channel is the target frame and first n - 1 are source frames\n",
    "    inp = grid[:-1, :].unsqueeze(0).to(device)\n",
    "    currLabel = grid[-1:, :].unsqueeze(0).to(device)\n",
    "    \n",
    "    if frame_num < 4:\n",
    "        prevChannels = inp\n",
    "\n",
    "    if frame_num >= 4:\n",
    "        new_inp = inp.clone().squeeze(0)\n",
    "        mn, mx = torch.min(prevOut), torch.max(prevOut)\n",
    "        prevOut = (prevOut - mn) / (mx - mn)\n",
    "        new_inp[0] = prevOut\n",
    "        new_inp[4] = prevChannels[0, 4, :, :]        \n",
    "        inp = new_inp.unsqueeze(0).cuda()\n",
    "\n",
    "    # Forward the input and obtain the result\n",
    "    out = model.forward(inp, state)\n",
    "    state = (model.h, model.c, model.h1, model.c1, model.h2, model.c2)\n",
    "    currOutputMap = out.clone()\n",
    "    newOutputMap = upsample_512(currOutputMap)\n",
    "    \n",
    "    nextTargetTensor = grid[5,:,:]\n",
    "    \n",
    "    prevOut = currOutputMap.detach().cpu().squeeze(0).squeeze(0)\n",
    "    currOutputMap = currOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel = currLabel.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    # Upsampled outputs and inputs\n",
    "#     currOutputMap1 = newOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "#     currLabel1 = nextTargetTensor_upsampled.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    _, dist1, predCoordinates1, gtCoordinates1 = heatmapAccuracy(currOutputMap, currLabel)\n",
    "    _, dist2, predCoordinates2, gtCoordinates2, within_radius = multiAccuracy(currOutputMap, currLabel, topK=topK)\n",
    "    \n",
    "    seqLen += 1\n",
    "    xValsGT.append(gtCoordinates1[0])\n",
    "    yValsGT.append(gtCoordinates1[1])\n",
    "    xValsPred.append(predCoordinates1[0])\n",
    "    yValsPred.append(predCoordinates1[1])\n",
    "    xValsPredMulti.append(predCoordinates2[0])\n",
    "    yValsPredMulti.append(predCoordinates2[1])\n",
    "    \n",
    "    if frame_num >= 4:\n",
    "        seqLoss.append(dist2)\n",
    "        new_seq_loss.append([dist2])        \n",
    "        totalPreds += 1\n",
    "        if within_radius == 1:\n",
    "            hitPreds += 1\n",
    "        #import pdb; pdb.set_trace()\n",
    "            \n",
    "        #visualize \n",
    "        if sceneNum == 49.0 and seqNum == 19.0 and frame_num == 6:\n",
    "            for j in range(len(grid)):\n",
    "                plt.imshow(grid[j,:,:].detach().cpu().numpy())\n",
    "                plt.savefig(\"grid_\" + str(j) + \"_\" + str(sceneNum) + \".png\")\n",
    "                plt.clf()\n",
    "            plt.imshow(currOutputMap)\n",
    "            plt.savefig(\"pred\" + \"_\" + str(sceneNum) + \".png\")\n",
    "            plt.clf()\n",
    "            im_path = \"traj_49\"\n",
    "            plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti, seqLen, im_path, numFrames=None)\n",
    "\n",
    "        if sceneNum == 47.0 and seqNum == 15.0 and frame_num == 7:\n",
    "            for j in range(len(grid)):\n",
    "                plt.imshow(grid[j,:,:].detach().cpu().numpy())\n",
    "                plt.savefig(\"grid_\" + str(j) + \"_\" + str(sceneNum) + \".png\")\n",
    "                plt.clf()\n",
    "            plt.imshow(currOutputMap)\n",
    "            plt.savefig(\"pred\" + \"_\" + str(sceneNum) + \".png\")\n",
    "            plt.clf()\n",
    "            im_path = \"traj_47\"\n",
    "            plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti, seqLen, im_path, numFrames=None)\n",
    "            \n",
    "        if sceneNum == 46.0 and seqNum == 12.0 and frame_num == 9:\n",
    "            for j in range(len(grid)):\n",
    "                plt.imshow(grid[j,:,:].detach().cpu().numpy())\n",
    "                plt.savefig(\"grid_\" + str(j) + \"_\" + str(sceneNum) + \".png\")\n",
    "                plt.clf()\n",
    "            plt.imshow(currOutputMap)\n",
    "            plt.savefig(\"pred\" + \"_\" + str(sceneNum) + \".png\")\n",
    "            plt.clf()\n",
    "            im_path = \"traj_46\"\n",
    "            plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti, seqLen, im_path, numFrames=None)\n",
    "\n",
    "   \n",
    "    \n",
    "    #import pdb; pdb.set_trace()\n",
    "    if endOfSequence:\n",
    "        seqVals.append(seqLen)\n",
    "        xValsGT, yValsGT, xValsPred, yValsPred, xValsPredMulti, yValsPredMulti = [], [], [], [], [], []\n",
    "        seqNum +=1\n",
    "        state = None\n",
    "        valLoss.append(np.mean(seqLoss))\n",
    "        valLoss1.append(np.mean(seqLoss[:2]))\n",
    "        valLoss2.append(np.mean(seqLoss[:4]))\n",
    "        valLoss3.append(np.mean(seqLoss[:6]))\n",
    "        valLoss4.append(np.mean(seqLoss[:8]))\n",
    "        print(\"Scene: {}, Sequence: {}, Frame: {}, Seq Loss: {}\".format(sceneNum, seqNum, frame_num, np.mean(seqLoss)))\n",
    "        #import pdb; pdb.set_trace()\n",
    "        seqLoss = []\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1s: 11.415061582638508, 2s: 15.13690567416456, 3s: 17.956807309559537, 4s: 22.2281926754762\n"
     ]
    }
   ],
   "source": [
    "print(\"1s: {}, 2s: {}, 3s: {}, 4s: {}\".format(np.mean(valLoss1), np.mean(valLoss2), np.mean(valLoss3), np.mean(valLoss4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HitPreds: 4536, TotalPreds: 13104, Hit Rate: 0.34615384615384615\n"
     ]
    }
   ],
   "source": [
    "print(\"HitPreds: {}, TotalPreds: {}, Hit Rate: {}\".format(hitPreds, totalPreds, hitPreds / totalPreds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}