Dimensionality_reduction_with_PCA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "verBJk4CIwf6"
   },
   "source": [
    "\n",
    "*   Data: Sentinel-2 satellite images of shape: (740, 740, 12)\n",
    "*   Dimensionality reduction PCA\n",
    "\n",
    "### PCA Code from: \n",
    "Mimi Kuan Ting Kuo - [Code link](https://github.com/MITCriticalData-Colombia/Dengue-Prediction-with-Satellite-Images/blob/main/Dimensionality_reduction_with_statistical_model.ipynb): [mimikuo365](https://github.com/mimikuo365)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kpVHuEf5hUTU"
   },
   "source": [
    "# Setup Environment and Load Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install plotly\n",
    "#!pip install epiweeks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "Nf6xQA_1nLAv"
   },
   "outputs": [],
   "source": [
    "# sklearn library\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import LocallyLinearEmbedding\n",
    "from sklearn.metrics import mean_absolute_percentage_error\n",
    "from sklearn import linear_model  \n",
    "from sklearn.svm import SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "cTsy3_Iq21K-"
   },
   "outputs": [],
   "source": [
    "from random import randint, randrange\n",
    "from skimage import io\n",
    "from skimage.transform import rescale, resize, downscale_local_mean\n",
    "\n",
    "import skimage\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import plotly.express as px\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "gyQb5j_zCjoy",
    "outputId": "72962baf-3ebe-4cde-a938-6219c0a40552"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.19.2'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skimage.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KlpIqYTQDeKu",
    "outputId": "e1a5b951-89d2-46f2-b25b-c8564d423d7f"
   },
   "outputs": [],
   "source": [
    "from epiweeks import Week, Year\n",
    "from datetime import date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get list of images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total number of images is 165\n"
     ]
    }
   ],
   "source": [
    "# Main Directory\n",
    "main_dir = 'Dataset/5001'\n",
    "print(f'The total number of images is {len(os.listdir(main_dir))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image list top 5 examples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Dataset/5001/image_2015-11-01.tiff',\n",
       " 'Dataset/5001/image_2015-11-08.tiff',\n",
       " 'Dataset/5001/image_2015-11-15.tiff',\n",
       " 'Dataset/5001/image_2015-11-22.tiff',\n",
       " 'Dataset/5001/image_2015-11-29.tiff']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# specify the img directory path\n",
    "path = main_dir\n",
    "\n",
    "# list files in img directory\n",
    "files = os.listdir(path)\n",
    "\n",
    "image_list = []\n",
    "\n",
    "for file in files:\n",
    "    # make sure file is an image\n",
    "    if file.endswith(('.jpg', '.png', 'jpeg', 'tiff')):\n",
    "        #img_path = path + file\n",
    "        image_list.append(os.path.join(path, file))\n",
    "        \n",
    "print(f'Image list top 5 examples:')\n",
    "image_list.sort(reverse=False)\n",
    "image_list[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_image(path, target_size = (740, 740, 12), verbose = True):\n",
    "    image = io.imread(path)\n",
    "    \n",
    "    if target_size:\n",
    "        image_test = resize(image, (target_size[0], target_size[1]),\n",
    "                               anti_aliasing=True)\n",
    "        \n",
    "    if verbose:\n",
    "        print(os.path.basename(path), '(origin shape:', image.shape, '-> rescale:', str(image_test.shape) + ')')\n",
    "    return image_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_image_name(path):\n",
    "    image_name = path[path.index('/image')+7:path.index('.tiff')]\n",
    "    return image_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IkaBRXTRhMdI",
    "outputId": "cee920b5-b996-440f-83b7-96f6f1514870"
   },
   "outputs": [],
   "source": [
    "def get_images(image_list, target_size = (740, 740, 12)):\n",
    "    \n",
    "    dates = list()\n",
    "    images = list()\n",
    "    \n",
    "    for image_path in image_list:\n",
    "        image = read_image(image_path, target_size, verbose=False)\n",
    "        date = get_image_name(image_path)\n",
    "        images.append(image)\n",
    "        dates.append(date)\n",
    "        \n",
    "    images = np.array(images)\n",
    "    dates = np.array(dates)\n",
    "    return images, dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_size = (740, 740, 12)\n",
    "images, dates = get_images(image_list, target_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(165, 740, 740, 12)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(165,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dates.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iLc9X_o2vDww"
   },
   "source": [
    "## Dimensionality reduction - PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "aJg4PwES7L1T"
   },
   "outputs": [],
   "source": [
    "def dimension_reduct_with_PCA(images, components):\n",
    "    print(' PRINCIPAL COMPONENT ANALYSIS  '.center(100, '='))\n",
    "\n",
    "    reshape_images = images.reshape(images.shape[0], -1)\n",
    "\n",
    "    #pca = PCA(n_components=0.95) \n",
    "    pca = PCA(n_components=components)\n",
    "    pca_images = pca.fit_transform(reshape_images)\n",
    "\n",
    "    print('Origin shape'.ljust(15), reshape_images.shape)\n",
    "    print('Resize shape'.ljust(15), pca_images.shape)  \n",
    "\n",
    "    return pca_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "Aioj_XnOSpaw"
   },
   "outputs": [],
   "source": [
    "def pca_per_band(images, components_per_band = 10):\n",
    "    pca_images = None\n",
    "\n",
    "    for band in range(images.shape[3]):\n",
    "        pca_band = dimension_reduct_with_PCA(images[:, :, band], components = components_per_band)\n",
    "        \n",
    "        if pca_images is None:\n",
    "            pca_images = pca_band\n",
    "        else:\n",
    "            pca_images = np.append(pca_images, pca_band, axis = 1)\n",
    "\n",
    "    print(pca_images.shape)\n",
    "    return pca_images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA per band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 8880)\n",
      "Resize shape    (165, 10)\n",
      "(165, 120)\n"
     ]
    }
   ],
   "source": [
    "pca_images_per_band = pca_per_band(images, components_per_band = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_columns(df):\n",
    "    df_aux = pd.DataFrame(df['PCA'].tolist())\n",
    "    df_aux = pd.concat( [df['Date'], df_aux], axis=1)\n",
    "    return df_aux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_pca_dataframe(image_list, dates):\n",
    "    \n",
    "    pca_df = pd.DataFrame(columns=['Date', 'PCA'])\n",
    "    \n",
    "    for i in range(len(image_list)):\n",
    "        image = image_list[i]\n",
    "        date = dates[i]\n",
    "        pca_df = pca_df.append({'Date': date, 'PCA': image}, ignore_index=True )\n",
    "\n",
    "    # new df from the column of lists\n",
    "    pca_df = split_columns(pca_df)\n",
    "    \n",
    "    return pca_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>110</th>\n",
       "      <th>111</th>\n",
       "      <th>112</th>\n",
       "      <th>113</th>\n",
       "      <th>114</th>\n",
       "      <th>115</th>\n",
       "      <th>116</th>\n",
       "      <th>117</th>\n",
       "      <th>118</th>\n",
       "      <th>119</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-11-01</td>\n",
       "      <td>2.294995</td>\n",
       "      <td>0.188790</td>\n",
       "      <td>-2.814999</td>\n",
       "      <td>-4.106509</td>\n",
       "      <td>-4.826127</td>\n",
       "      <td>-1.931590</td>\n",
       "      <td>2.110046</td>\n",
       "      <td>2.169669</td>\n",
       "      <td>-3.337853</td>\n",
       "      <td>...</td>\n",
       "      <td>4.991791</td>\n",
       "      <td>-1.130083</td>\n",
       "      <td>-4.237726</td>\n",
       "      <td>-8.811673</td>\n",
       "      <td>2.755142</td>\n",
       "      <td>-2.230883</td>\n",
       "      <td>1.070571</td>\n",
       "      <td>3.062169</td>\n",
       "      <td>-2.531766</td>\n",
       "      <td>-0.283356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-11-08</td>\n",
       "      <td>2.294995</td>\n",
       "      <td>0.188790</td>\n",
       "      <td>-2.814999</td>\n",
       "      <td>-4.106509</td>\n",
       "      <td>-4.826127</td>\n",
       "      <td>-1.931590</td>\n",
       "      <td>2.110046</td>\n",
       "      <td>2.169669</td>\n",
       "      <td>-3.337853</td>\n",
       "      <td>...</td>\n",
       "      <td>4.991791</td>\n",
       "      <td>-1.130083</td>\n",
       "      <td>-4.237726</td>\n",
       "      <td>-8.811673</td>\n",
       "      <td>2.755142</td>\n",
       "      <td>-2.230883</td>\n",
       "      <td>1.070571</td>\n",
       "      <td>3.062169</td>\n",
       "      <td>-2.531766</td>\n",
       "      <td>-0.283356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-11-15</td>\n",
       "      <td>-2.992776</td>\n",
       "      <td>-4.160585</td>\n",
       "      <td>-1.680303</td>\n",
       "      <td>0.850790</td>\n",
       "      <td>0.399466</td>\n",
       "      <td>-0.382515</td>\n",
       "      <td>1.082947</td>\n",
       "      <td>0.885324</td>\n",
       "      <td>0.939764</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.651444</td>\n",
       "      <td>-4.150722</td>\n",
       "      <td>-1.211129</td>\n",
       "      <td>0.537388</td>\n",
       "      <td>1.388784</td>\n",
       "      <td>-0.833611</td>\n",
       "      <td>1.429209</td>\n",
       "      <td>-0.413522</td>\n",
       "      <td>0.802893</td>\n",
       "      <td>2.269096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-11-22</td>\n",
       "      <td>-2.992776</td>\n",
       "      <td>-4.160585</td>\n",
       "      <td>-1.680303</td>\n",
       "      <td>0.850790</td>\n",
       "      <td>0.399466</td>\n",
       "      <td>-0.382515</td>\n",
       "      <td>1.082947</td>\n",
       "      <td>0.885324</td>\n",
       "      <td>0.939764</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.651444</td>\n",
       "      <td>-4.150722</td>\n",
       "      <td>-1.211129</td>\n",
       "      <td>0.537388</td>\n",
       "      <td>1.388784</td>\n",
       "      <td>-0.833611</td>\n",
       "      <td>1.429209</td>\n",
       "      <td>-0.413522</td>\n",
       "      <td>0.802893</td>\n",
       "      <td>2.269096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-11-29</td>\n",
       "      <td>-2.958983</td>\n",
       "      <td>-0.799012</td>\n",
       "      <td>0.408298</td>\n",
       "      <td>-0.275674</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>-0.400654</td>\n",
       "      <td>0.018210</td>\n",
       "      <td>-0.093618</td>\n",
       "      <td>0.140624</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.709884</td>\n",
       "      <td>-0.841614</td>\n",
       "      <td>0.306946</td>\n",
       "      <td>0.147025</td>\n",
       "      <td>-0.587185</td>\n",
       "      <td>-0.286853</td>\n",
       "      <td>-0.149727</td>\n",
       "      <td>0.027457</td>\n",
       "      <td>0.067555</td>\n",
       "      <td>0.085697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>2018-11-25</td>\n",
       "      <td>-5.522933</td>\n",
       "      <td>0.242564</td>\n",
       "      <td>-0.107411</td>\n",
       "      <td>-0.137835</td>\n",
       "      <td>0.082961</td>\n",
       "      <td>-0.152577</td>\n",
       "      <td>0.088666</td>\n",
       "      <td>-0.176468</td>\n",
       "      <td>0.371281</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.691023</td>\n",
       "      <td>0.108531</td>\n",
       "      <td>-0.000158</td>\n",
       "      <td>-0.193409</td>\n",
       "      <td>-0.068230</td>\n",
       "      <td>-0.285634</td>\n",
       "      <td>0.164417</td>\n",
       "      <td>-0.391486</td>\n",
       "      <td>-0.267216</td>\n",
       "      <td>-0.075663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>2018-12-02</td>\n",
       "      <td>9.346026</td>\n",
       "      <td>2.523671</td>\n",
       "      <td>-4.925394</td>\n",
       "      <td>3.703968</td>\n",
       "      <td>-5.341458</td>\n",
       "      <td>-1.085077</td>\n",
       "      <td>1.720805</td>\n",
       "      <td>2.278464</td>\n",
       "      <td>-0.258981</td>\n",
       "      <td>...</td>\n",
       "      <td>7.905108</td>\n",
       "      <td>4.050743</td>\n",
       "      <td>-2.794811</td>\n",
       "      <td>-0.623507</td>\n",
       "      <td>6.047697</td>\n",
       "      <td>-1.914679</td>\n",
       "      <td>-0.644306</td>\n",
       "      <td>-0.847323</td>\n",
       "      <td>0.325168</td>\n",
       "      <td>2.051028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>2018-12-09</td>\n",
       "      <td>-2.652909</td>\n",
       "      <td>-0.563147</td>\n",
       "      <td>0.440027</td>\n",
       "      <td>-0.110709</td>\n",
       "      <td>-0.128444</td>\n",
       "      <td>-0.195813</td>\n",
       "      <td>0.060844</td>\n",
       "      <td>-0.127574</td>\n",
       "      <td>0.285074</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.629335</td>\n",
       "      <td>-0.678518</td>\n",
       "      <td>0.398805</td>\n",
       "      <td>-0.135323</td>\n",
       "      <td>0.174343</td>\n",
       "      <td>-0.425002</td>\n",
       "      <td>0.079115</td>\n",
       "      <td>-0.410194</td>\n",
       "      <td>-0.122415</td>\n",
       "      <td>-0.016983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>2018-12-16</td>\n",
       "      <td>0.993320</td>\n",
       "      <td>0.777557</td>\n",
       "      <td>14.432274</td>\n",
       "      <td>2.801776</td>\n",
       "      <td>1.415641</td>\n",
       "      <td>0.281821</td>\n",
       "      <td>0.892250</td>\n",
       "      <td>-1.261930</td>\n",
       "      <td>-2.372967</td>\n",
       "      <td>...</td>\n",
       "      <td>2.240127</td>\n",
       "      <td>2.318474</td>\n",
       "      <td>14.520888</td>\n",
       "      <td>1.314388</td>\n",
       "      <td>0.167148</td>\n",
       "      <td>-0.415064</td>\n",
       "      <td>0.581475</td>\n",
       "      <td>0.089534</td>\n",
       "      <td>-1.565607</td>\n",
       "      <td>-1.747052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>2018-12-23</td>\n",
       "      <td>-2.017882</td>\n",
       "      <td>1.194812</td>\n",
       "      <td>9.614885</td>\n",
       "      <td>5.268044</td>\n",
       "      <td>5.292767</td>\n",
       "      <td>1.333542</td>\n",
       "      <td>3.294566</td>\n",
       "      <td>1.864643</td>\n",
       "      <td>-2.056928</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.715591</td>\n",
       "      <td>3.223827</td>\n",
       "      <td>10.172059</td>\n",
       "      <td>5.945499</td>\n",
       "      <td>-0.615524</td>\n",
       "      <td>-0.222450</td>\n",
       "      <td>4.947362</td>\n",
       "      <td>1.967058</td>\n",
       "      <td>-0.392646</td>\n",
       "      <td>-0.777560</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 121 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date         0         1          2         3         4         5  \\\n",
       "0    2015-11-01  2.294995  0.188790  -2.814999 -4.106509 -4.826127 -1.931590   \n",
       "1    2015-11-08  2.294995  0.188790  -2.814999 -4.106509 -4.826127 -1.931590   \n",
       "2    2015-11-15 -2.992776 -4.160585  -1.680303  0.850790  0.399466 -0.382515   \n",
       "3    2015-11-22 -2.992776 -4.160585  -1.680303  0.850790  0.399466 -0.382515   \n",
       "4    2015-11-29 -2.958983 -0.799012   0.408298 -0.275674  0.540541 -0.400654   \n",
       "..          ...       ...       ...        ...       ...       ...       ...   \n",
       "160  2018-11-25 -5.522933  0.242564  -0.107411 -0.137835  0.082961 -0.152577   \n",
       "161  2018-12-02  9.346026  2.523671  -4.925394  3.703968 -5.341458 -1.085077   \n",
       "162  2018-12-09 -2.652909 -0.563147   0.440027 -0.110709 -0.128444 -0.195813   \n",
       "163  2018-12-16  0.993320  0.777557  14.432274  2.801776  1.415641  0.281821   \n",
       "164  2018-12-23 -2.017882  1.194812   9.614885  5.268044  5.292767  1.333542   \n",
       "\n",
       "            6         7         8  ...       110       111        112  \\\n",
       "0    2.110046  2.169669 -3.337853  ...  4.991791 -1.130083  -4.237726   \n",
       "1    2.110046  2.169669 -3.337853  ...  4.991791 -1.130083  -4.237726   \n",
       "2    1.082947  0.885324  0.939764  ... -2.651444 -4.150722  -1.211129   \n",
       "3    1.082947  0.885324  0.939764  ... -2.651444 -4.150722  -1.211129   \n",
       "4    0.018210 -0.093618  0.140624  ... -2.709884 -0.841614   0.306946   \n",
       "..        ...       ...       ...  ...       ...       ...        ...   \n",
       "160  0.088666 -0.176468  0.371281  ... -5.691023  0.108531  -0.000158   \n",
       "161  1.720805  2.278464 -0.258981  ...  7.905108  4.050743  -2.794811   \n",
       "162  0.060844 -0.127574  0.285074  ... -2.629335 -0.678518   0.398805   \n",
       "163  0.892250 -1.261930 -2.372967  ...  2.240127  2.318474  14.520888   \n",
       "164  3.294566  1.864643 -2.056928  ... -0.715591  3.223827  10.172059   \n",
       "\n",
       "          113       114       115       116       117       118       119  \n",
       "0   -8.811673  2.755142 -2.230883  1.070571  3.062169 -2.531766 -0.283356  \n",
       "1   -8.811673  2.755142 -2.230883  1.070571  3.062169 -2.531766 -0.283356  \n",
       "2    0.537388  1.388784 -0.833611  1.429209 -0.413522  0.802893  2.269096  \n",
       "3    0.537388  1.388784 -0.833611  1.429209 -0.413522  0.802893  2.269096  \n",
       "4    0.147025 -0.587185 -0.286853 -0.149727  0.027457  0.067555  0.085697  \n",
       "..        ...       ...       ...       ...       ...       ...       ...  \n",
       "160 -0.193409 -0.068230 -0.285634  0.164417 -0.391486 -0.267216 -0.075663  \n",
       "161 -0.623507  6.047697 -1.914679 -0.644306 -0.847323  0.325168  2.051028  \n",
       "162 -0.135323  0.174343 -0.425002  0.079115 -0.410194 -0.122415 -0.016983  \n",
       "163  1.314388  0.167148 -0.415064  0.581475  0.089534 -1.565607 -1.747052  \n",
       "164  5.945499 -0.615524 -0.222450  4.947362  1.967058 -0.392646 -0.777560  \n",
       "\n",
       "[165 rows x 121 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new df from the column of lists\n",
    "pca_df = generate_pca_dataframe(pca_images_per_band, dates)\n",
    "\n",
    "pca_df.to_csv('Embeddings/pca_medellin_120features(10_per_band).csv',index=False)\n",
    "# display the resulting df\n",
    "pca_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA in the overall image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================== PRINCIPAL COMPONENT ANALYSIS  ===================================\n",
      "Origin shape    (165, 6571200)\n",
      "Resize shape    (165, 100)\n"
     ]
    }
   ],
   "source": [
    "pca_images = dimension_reduct_with_PCA(images, components=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-11-01</td>\n",
       "      <td>98.435754</td>\n",
       "      <td>108.962228</td>\n",
       "      <td>-40.915729</td>\n",
       "      <td>39.468939</td>\n",
       "      <td>47.325778</td>\n",
       "      <td>70.472168</td>\n",
       "      <td>187.431726</td>\n",
       "      <td>-229.801740</td>\n",
       "      <td>43.888213</td>\n",
       "      <td>...</td>\n",
       "      <td>0.587965</td>\n",
       "      <td>-0.365837</td>\n",
       "      <td>-2.923909</td>\n",
       "      <td>0.096117</td>\n",
       "      <td>-0.608161</td>\n",
       "      <td>0.119047</td>\n",
       "      <td>1.095555</td>\n",
       "      <td>-1.305447</td>\n",
       "      <td>-1.034465</td>\n",
       "      <td>-0.486601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-11-08</td>\n",
       "      <td>98.435754</td>\n",
       "      <td>108.962228</td>\n",
       "      <td>-40.915729</td>\n",
       "      <td>39.468939</td>\n",
       "      <td>47.325778</td>\n",
       "      <td>70.472168</td>\n",
       "      <td>187.431726</td>\n",
       "      <td>-229.801740</td>\n",
       "      <td>43.888213</td>\n",
       "      <td>...</td>\n",
       "      <td>0.587965</td>\n",
       "      <td>-0.365837</td>\n",
       "      <td>-2.923909</td>\n",
       "      <td>0.096117</td>\n",
       "      <td>-0.608161</td>\n",
       "      <td>0.119047</td>\n",
       "      <td>1.095555</td>\n",
       "      <td>-1.305447</td>\n",
       "      <td>-1.034465</td>\n",
       "      <td>-0.486601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-11-15</td>\n",
       "      <td>-149.578506</td>\n",
       "      <td>118.675591</td>\n",
       "      <td>-28.032221</td>\n",
       "      <td>-37.096690</td>\n",
       "      <td>-3.305299</td>\n",
       "      <td>-67.746083</td>\n",
       "      <td>37.584909</td>\n",
       "      <td>-11.838416</td>\n",
       "      <td>-15.613838</td>\n",
       "      <td>...</td>\n",
       "      <td>-13.746142</td>\n",
       "      <td>13.683615</td>\n",
       "      <td>-1.131025</td>\n",
       "      <td>-3.349845</td>\n",
       "      <td>-10.681103</td>\n",
       "      <td>-6.437450</td>\n",
       "      <td>12.024817</td>\n",
       "      <td>12.829025</td>\n",
       "      <td>12.699458</td>\n",
       "      <td>0.475610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-11-22</td>\n",
       "      <td>-149.578506</td>\n",
       "      <td>118.675591</td>\n",
       "      <td>-28.032221</td>\n",
       "      <td>-37.096690</td>\n",
       "      <td>-3.305299</td>\n",
       "      <td>-67.746083</td>\n",
       "      <td>37.584909</td>\n",
       "      <td>-11.838416</td>\n",
       "      <td>-15.613838</td>\n",
       "      <td>...</td>\n",
       "      <td>-13.746142</td>\n",
       "      <td>13.683615</td>\n",
       "      <td>-1.131025</td>\n",
       "      <td>-3.349845</td>\n",
       "      <td>-10.681103</td>\n",
       "      <td>-6.437450</td>\n",
       "      <td>12.024817</td>\n",
       "      <td>12.829025</td>\n",
       "      <td>12.699458</td>\n",
       "      <td>0.475610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-11-29</td>\n",
       "      <td>-28.099436</td>\n",
       "      <td>34.715737</td>\n",
       "      <td>47.153387</td>\n",
       "      <td>-50.761985</td>\n",
       "      <td>-63.137078</td>\n",
       "      <td>-58.088638</td>\n",
       "      <td>-6.508121</td>\n",
       "      <td>18.149721</td>\n",
       "      <td>-29.781310</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.670381</td>\n",
       "      <td>-5.520467</td>\n",
       "      <td>-4.101447</td>\n",
       "      <td>1.232668</td>\n",
       "      <td>-2.903102</td>\n",
       "      <td>2.551955</td>\n",
       "      <td>-3.338012</td>\n",
       "      <td>-5.490846</td>\n",
       "      <td>-2.383766</td>\n",
       "      <td>-1.712792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>2018-11-25</td>\n",
       "      <td>-154.158021</td>\n",
       "      <td>-40.385477</td>\n",
       "      <td>14.959814</td>\n",
       "      <td>-16.314940</td>\n",
       "      <td>32.494744</td>\n",
       "      <td>32.049769</td>\n",
       "      <td>-110.123837</td>\n",
       "      <td>-32.925602</td>\n",
       "      <td>70.966415</td>\n",
       "      <td>...</td>\n",
       "      <td>-24.752246</td>\n",
       "      <td>-0.418285</td>\n",
       "      <td>-13.387984</td>\n",
       "      <td>11.733950</td>\n",
       "      <td>22.422833</td>\n",
       "      <td>-23.083992</td>\n",
       "      <td>-1.774827</td>\n",
       "      <td>-12.985468</td>\n",
       "      <td>-0.344016</td>\n",
       "      <td>-11.627382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>2018-12-02</td>\n",
       "      <td>71.530670</td>\n",
       "      <td>-194.253970</td>\n",
       "      <td>-60.854103</td>\n",
       "      <td>59.359774</td>\n",
       "      <td>61.782572</td>\n",
       "      <td>136.246484</td>\n",
       "      <td>-189.902994</td>\n",
       "      <td>-27.714046</td>\n",
       "      <td>46.346134</td>\n",
       "      <td>...</td>\n",
       "      <td>2.033835</td>\n",
       "      <td>-3.499518</td>\n",
       "      <td>1.421727</td>\n",
       "      <td>0.473346</td>\n",
       "      <td>-3.668544</td>\n",
       "      <td>3.775593</td>\n",
       "      <td>0.595169</td>\n",
       "      <td>9.517483</td>\n",
       "      <td>1.630138</td>\n",
       "      <td>-4.106867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>2018-12-09</td>\n",
       "      <td>-83.791680</td>\n",
       "      <td>26.428490</td>\n",
       "      <td>19.535233</td>\n",
       "      <td>-9.675937</td>\n",
       "      <td>22.466795</td>\n",
       "      <td>24.155634</td>\n",
       "      <td>-16.545517</td>\n",
       "      <td>8.703227</td>\n",
       "      <td>-4.421293</td>\n",
       "      <td>...</td>\n",
       "      <td>5.490728</td>\n",
       "      <td>13.986629</td>\n",
       "      <td>-1.157662</td>\n",
       "      <td>9.442495</td>\n",
       "      <td>17.961227</td>\n",
       "      <td>5.249652</td>\n",
       "      <td>-13.657355</td>\n",
       "      <td>17.829749</td>\n",
       "      <td>10.827447</td>\n",
       "      <td>-14.438771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>2018-12-16</td>\n",
       "      <td>101.759249</td>\n",
       "      <td>-100.737490</td>\n",
       "      <td>-118.204529</td>\n",
       "      <td>-52.278016</td>\n",
       "      <td>58.832020</td>\n",
       "      <td>212.931482</td>\n",
       "      <td>27.047104</td>\n",
       "      <td>-18.487913</td>\n",
       "      <td>-129.743708</td>\n",
       "      <td>...</td>\n",
       "      <td>2.554147</td>\n",
       "      <td>4.894150</td>\n",
       "      <td>-3.007075</td>\n",
       "      <td>0.121323</td>\n",
       "      <td>-0.454753</td>\n",
       "      <td>7.455948</td>\n",
       "      <td>2.480901</td>\n",
       "      <td>4.855074</td>\n",
       "      <td>4.578702</td>\n",
       "      <td>-5.699132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>2018-12-23</td>\n",
       "      <td>-127.483830</td>\n",
       "      <td>74.647305</td>\n",
       "      <td>54.392430</td>\n",
       "      <td>-4.815894</td>\n",
       "      <td>64.802156</td>\n",
       "      <td>84.031067</td>\n",
       "      <td>23.167054</td>\n",
       "      <td>14.882480</td>\n",
       "      <td>-104.283492</td>\n",
       "      <td>...</td>\n",
       "      <td>0.157574</td>\n",
       "      <td>1.284880</td>\n",
       "      <td>-2.542231</td>\n",
       "      <td>3.877846</td>\n",
       "      <td>-3.967451</td>\n",
       "      <td>-8.966984</td>\n",
       "      <td>-7.433821</td>\n",
       "      <td>-14.207809</td>\n",
       "      <td>-8.780055</td>\n",
       "      <td>4.064164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date           0           1           2          3          4  \\\n",
       "0    2015-11-01   98.435754  108.962228  -40.915729  39.468939  47.325778   \n",
       "1    2015-11-08   98.435754  108.962228  -40.915729  39.468939  47.325778   \n",
       "2    2015-11-15 -149.578506  118.675591  -28.032221 -37.096690  -3.305299   \n",
       "3    2015-11-22 -149.578506  118.675591  -28.032221 -37.096690  -3.305299   \n",
       "4    2015-11-29  -28.099436   34.715737   47.153387 -50.761985 -63.137078   \n",
       "..          ...         ...         ...         ...        ...        ...   \n",
       "160  2018-11-25 -154.158021  -40.385477   14.959814 -16.314940  32.494744   \n",
       "161  2018-12-02   71.530670 -194.253970  -60.854103  59.359774  61.782572   \n",
       "162  2018-12-09  -83.791680   26.428490   19.535233  -9.675937  22.466795   \n",
       "163  2018-12-16  101.759249 -100.737490 -118.204529 -52.278016  58.832020   \n",
       "164  2018-12-23 -127.483830   74.647305   54.392430  -4.815894  64.802156   \n",
       "\n",
       "              5           6           7           8  ...         90  \\\n",
       "0     70.472168  187.431726 -229.801740   43.888213  ...   0.587965   \n",
       "1     70.472168  187.431726 -229.801740   43.888213  ...   0.587965   \n",
       "2    -67.746083   37.584909  -11.838416  -15.613838  ... -13.746142   \n",
       "3    -67.746083   37.584909  -11.838416  -15.613838  ... -13.746142   \n",
       "4    -58.088638   -6.508121   18.149721  -29.781310  ...  -6.670381   \n",
       "..          ...         ...         ...         ...  ...        ...   \n",
       "160   32.049769 -110.123837  -32.925602   70.966415  ... -24.752246   \n",
       "161  136.246484 -189.902994  -27.714046   46.346134  ...   2.033835   \n",
       "162   24.155634  -16.545517    8.703227   -4.421293  ...   5.490728   \n",
       "163  212.931482   27.047104  -18.487913 -129.743708  ...   2.554147   \n",
       "164   84.031067   23.167054   14.882480 -104.283492  ...   0.157574   \n",
       "\n",
       "            91         92         93         94         95         96  \\\n",
       "0    -0.365837  -2.923909   0.096117  -0.608161   0.119047   1.095555   \n",
       "1    -0.365837  -2.923909   0.096117  -0.608161   0.119047   1.095555   \n",
       "2    13.683615  -1.131025  -3.349845 -10.681103  -6.437450  12.024817   \n",
       "3    13.683615  -1.131025  -3.349845 -10.681103  -6.437450  12.024817   \n",
       "4    -5.520467  -4.101447   1.232668  -2.903102   2.551955  -3.338012   \n",
       "..         ...        ...        ...        ...        ...        ...   \n",
       "160  -0.418285 -13.387984  11.733950  22.422833 -23.083992  -1.774827   \n",
       "161  -3.499518   1.421727   0.473346  -3.668544   3.775593   0.595169   \n",
       "162  13.986629  -1.157662   9.442495  17.961227   5.249652 -13.657355   \n",
       "163   4.894150  -3.007075   0.121323  -0.454753   7.455948   2.480901   \n",
       "164   1.284880  -2.542231   3.877846  -3.967451  -8.966984  -7.433821   \n",
       "\n",
       "            97         98         99  \n",
       "0    -1.305447  -1.034465  -0.486601  \n",
       "1    -1.305447  -1.034465  -0.486601  \n",
       "2    12.829025  12.699458   0.475610  \n",
       "3    12.829025  12.699458   0.475610  \n",
       "4    -5.490846  -2.383766  -1.712792  \n",
       "..         ...        ...        ...  \n",
       "160 -12.985468  -0.344016 -11.627382  \n",
       "161   9.517483   1.630138  -4.106867  \n",
       "162  17.829749  10.827447 -14.438771  \n",
       "163   4.855074   4.578702  -5.699132  \n",
       "164 -14.207809  -8.780055   4.064164  \n",
       "\n",
       "[165 rows x 101 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new df from the column of lists\n",
    "pca_df = generate_pca_dataframe(pca_images, dates)\n",
    "\n",
    "pca_df.to_csv('Embeddings/pca_medellin_100features.csv',index=False)\n",
    "# display the resulting df\n",
    "pca_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "rK9kW_7xHORf",
    "kpVHuEf5hUTU",
    "4Km6do3bvRKw",
    "cWEMb1ZjvUTq",
    "Pph6i9sHvurx",
    "x_5iiCDpvBag",
    "yQTM0ZTP-fNQ",
    "G9wEr6hlxUC8",
    "P_QU-17t4Opz",
    "r3H2ftvS3m1e",
    "DPeC32Ai38Ck"
   ],
   "name": "Dimensionality reduction with statistical model.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}