join_streams.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Attributes\" data-toc-modified-id=\"Attributes-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Attributes</a></span><ul class=\"toc-item\"><li><span><a href=\"#Test:-STALN\" data-toc-modified-id=\"Test:-STALN-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Test: STALN</a></span></li><li><span><a href=\"#Test-POTEK\" data-toc-modified-id=\"Test-POTEK-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Test POTEK</a></span></li><li><span><a href=\"#Test-VRSTA\" data-toc-modified-id=\"Test-VRSTA-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Test VRSTA</a></span></li><li><span><a href=\"#Test-TIPTV\" data-toc-modified-id=\"Test-TIPTV-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Test TIPTV</a></span></li><li><span><a href=\"#Test-STANJE_IM\" data-toc-modified-id=\"Test-STANJE_IM-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Test STANJE_IM</a></span></li><li><span><a href=\"#Conclusion\" data-toc-modified-id=\"Conclusion-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>Conclusion</a></span></li></ul></li><li><span><a href=\"#Filter\" data-toc-modified-id=\"Filter-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Filter</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Streams (join features by attribute)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "shp_src = \"d:\\\\nejc\\\\hand_index\\\\DRSV_HIDRO5_LIN_PV\\\\DRSV_HIDRO5_LIN_PV.shp\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf = gpd.GeoDataFrame.from_file(shp_src)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of features: 480314\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total number of features: {gdf.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['OBJECTID_1', 'ID_REG_HLI', 'HLINPV_ID', 'HLINPV_IM', 'VERZIJA',\n",
      "       'DATP_ZAC', 'DATP_KON', 'RAZVOJ', 'JEZIK', 'DAT_ZAC', 'DAT_KON',\n",
      "       'VODE_ID', 'HMZ_ID', 'IME_ID', 'IME', 'VRSTA_ID', 'VRSTA_IME', 'OS_ID',\n",
      "       'OS_IME', 'VTOK_ID', 'VTOK_IME', 'IZVOR_ID', 'IZVOR_IM', 'STALN_ID',\n",
      "       'STALN_IME', 'STANJE_ID', 'STANJE_IM', 'POTEK_ID', 'POTEK_IM',\n",
      "       'TIPTV_ID', 'TIPTV_IM', 'TIPPREH_ID', 'TIPPREH_IM', 'SIRINA_ID',\n",
      "       'SIRINA_IM', 'VIRP_ID', 'VIRP_IM', 'DVIRP', 'PAS_VZ', 'PREOB_ID',\n",
      "       'PREOBLIK', 'geometry'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(gdf.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test: STALN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ni podatka      362655\n",
       "funkcionalen    117616\n",
       "občasen             28\n",
       "stalen              15\n",
       "Name: STALN_IME, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.STALN_IME.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9999    362655\n",
       "2       117616\n",
       "3           28\n",
       "4           15\n",
       "Name: STALN_ID, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.STALN_ID.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision: include all features, because there is too many attributes with \"unknown\" label `\"9999\": \"ni podatka\"`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test POTEK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "da - potek je znan    461652\n",
       "ne - potek ni znan     18662\n",
       "Name: POTEK_IM, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.POTEK_IM.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    461652\n",
       "2     18662\n",
       "Name: POTEK_ID, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.POTEK_ID.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision: include only ID `\"1\": \"da - potek je znan\"`.\n",
    "\n",
    "Revised: can as well include `\"2\"` if it also has VRSTA_ID `\"2\": \"prehod\"`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test VRSTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tekoča voda              384159\n",
       "prehod                    92391\n",
       "stoječa voda povezava      3396\n",
       "akumulacijsko jezero        368\n",
       "Name: VRSTA_IME, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.VRSTA_IME.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    384159\n",
       "2     92391\n",
       "4      3396\n",
       "3       368\n",
       "Name: VRSTA_ID, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.VRSTA_ID.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    17681\n",
       "Name: VRSTA_ID, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf[(gdf.VRSTA_ID == \"2\") & (gdf.POTEK_ID == \"2\")].VRSTA_ID.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prehod                   17681\n",
       "tekoča voda                930\n",
       "stoječa voda povezava       51\n",
       "Name: VRSTA_IME, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf[gdf.POTEK_ID == \"2\"].VRSTA_IME.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision: Since most of the `\"potek ni znan\"` are calssified as `\"prehod\"`, we can just keep all of these features. The `\"prehod\"` features are important for breaching step in DEM pre-processing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test TIPTV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vodotok                                362697\n",
       "melioracijski jarek                     75754\n",
       "obcestni jarek                          33316\n",
       "razbremenilni jarek                      8329\n",
       "objekt (kanal) za posebno rabo vode       174\n",
       "padavinski jarek                           44\n",
       "Name: TIPTV_IM, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.TIPTV_IM.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    362697\n",
       "5     75754\n",
       "4     33316\n",
       "6      8329\n",
       "7       174\n",
       "8        44\n",
       "Name: TIPTV_ID, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.TIPTV_ID.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tekoča voda              294373\n",
       "prehod                    65277\n",
       "stoječa voda povezava      2679\n",
       "akumulacijsko jezero        368\n",
       "Name: VRSTA_IME, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf[gdf.TIPTV_ID == \"1\"].VRSTA_IME.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save only \"`\"objekt (kanal) za posebno rabo vode\"\n",
    "tiptv = gdf[gdf.TIPTV_ID == \"7\"]\n",
    "pth_tiptv = \"c:\\\\Users\\\\ncoz\\\\ARRS_susa\\\\TIPTV\"\n",
    "os.makedirs(pth_tiptv, exist_ok=True)\n",
    "tiptv.to_file(os.path.join(pth_tiptv, \"tiptv_kanal-objekt.shp\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision: include only IDs `\"1\": \"vodotok\"` and `\"5\": \"melioracijski jarek\"`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test STANJE_IM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "občasen         272665\n",
       "stalen          207605\n",
       "ni podatka          42\n",
       "funkcionalen         1\n",
       "suh                  1\n",
       "Name: STANJE_IM, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.STANJE_IM.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3       272665\n",
       "4       207605\n",
       "9999        42\n",
       "2            1\n",
       "1            1\n",
       "Name: STANJE_ID, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.STANJE_ID.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision: iclude only ID `\"4\": \"stalen\"`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conclusion\n",
    "\n",
    "In conclusion, we only filter by two criteria:\n",
    "\n",
    "    - TIPTV == (\"1\" - vodotok) and (\"5\" - melioracije)\n",
    "    - STANJE == (\"4\" - stalen)\n",
    "    \n",
    "EDIT 16/3/2021\n",
    "Also add (\"6\" - razbremenilni jarek) - Gubarjev prekop in drugi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID_1</th>\n",
       "      <th>TIPTV_ID</th>\n",
       "      <th>TIPTV_IM</th>\n",
       "      <th>STANJE_ID</th>\n",
       "      <th>STANJE_IM</th>\n",
       "      <th>IME_ID</th>\n",
       "      <th>IME</th>\n",
       "      <th>VRSTA_ID</th>\n",
       "      <th>VRSTA_IME</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>480312</th>\n",
       "      <td>15970</td>\n",
       "      <td>1</td>\n",
       "      <td>vodotok</td>\n",
       "      <td>3</td>\n",
       "      <td>občasen</td>\n",
       "      <td>0</td>\n",
       "      <td>0000 (zemljepisno ime v viru ni določeno)</td>\n",
       "      <td>2</td>\n",
       "      <td>prehod</td>\n",
       "      <td>LINESTRING Z (547845.940 116738.440 219.140, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480313</th>\n",
       "      <td>15971</td>\n",
       "      <td>1</td>\n",
       "      <td>vodotok</td>\n",
       "      <td>3</td>\n",
       "      <td>občasen</td>\n",
       "      <td>0</td>\n",
       "      <td>0000 (zemljepisno ime v viru ni določeno)</td>\n",
       "      <td>2</td>\n",
       "      <td>prehod</td>\n",
       "      <td>LINESTRING Z (546318.440 127203.610 692.370, 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        OBJECTID_1 TIPTV_ID TIPTV_IM STANJE_ID STANJE_IM IME_ID  \\\n",
       "480312       15970        1  vodotok         3   občasen      0   \n",
       "480313       15971        1  vodotok         3   občasen      0   \n",
       "\n",
       "                                              IME VRSTA_ID VRSTA_IME  \\\n",
       "480312  0000 (zemljepisno ime v viru ni določeno)        2    prehod   \n",
       "480313  0000 (zemljepisno ime v viru ni določeno)        2    prehod   \n",
       "\n",
       "                                                 geometry  \n",
       "480312  LINESTRING Z (547845.940 116738.440 219.140, 5...  \n",
       "480313  LINESTRING Z (546318.440 127203.610 692.370, 5...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keep_cols = [\n",
    "    \"OBJECTID_1\",\n",
    "    \"TIPTV_ID\", \"TIPTV_IM\",\n",
    "    \"STANJE_ID\", \"STANJE_IM\",\n",
    "    \"IME_ID\", \"IME\",\n",
    "    \"VRSTA_ID\", \"VRSTA_IME\",\n",
    "    \"geometry\",\n",
    "]\n",
    "gdf = gdf[keep_cols]\n",
    "gdf.tail(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter\n",
    "Only select streams with:\n",
    "\n",
    "    - TIPTV == (\"1\" - vodotok) and (\"5\" - melioracije) and (\"6\" - razbremenilni jarek)\n",
    "    - STANJE == (\"4\" - stalen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID_1</th>\n",
       "      <th>TIPTV_ID</th>\n",
       "      <th>TIPTV_IM</th>\n",
       "      <th>STANJE_ID</th>\n",
       "      <th>STANJE_IM</th>\n",
       "      <th>IME_ID</th>\n",
       "      <th>IME</th>\n",
       "      <th>VRSTA_ID</th>\n",
       "      <th>VRSTA_IME</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>476307</th>\n",
       "      <td>21931</td>\n",
       "      <td>1</td>\n",
       "      <td>vodotok</td>\n",
       "      <td>4</td>\n",
       "      <td>stalen</td>\n",
       "      <td>0</td>\n",
       "      <td>0000 (zemljepisno ime v viru ni določeno)</td>\n",
       "      <td>2</td>\n",
       "      <td>prehod</td>\n",
       "      <td>LINESTRING Z (534309.280 134630.090 319.370, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476308</th>\n",
       "      <td>22962</td>\n",
       "      <td>1</td>\n",
       "      <td>vodotok</td>\n",
       "      <td>4</td>\n",
       "      <td>stalen</td>\n",
       "      <td>0</td>\n",
       "      <td>0000 (zemljepisno ime v viru ni določeno)</td>\n",
       "      <td>2</td>\n",
       "      <td>prehod</td>\n",
       "      <td>LINESTRING Z (588983.590 162427.830 182.070, 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        OBJECTID_1 TIPTV_ID TIPTV_IM STANJE_ID STANJE_IM IME_ID  \\\n",
       "476307       21931        1  vodotok         4    stalen      0   \n",
       "476308       22962        1  vodotok         4    stalen      0   \n",
       "\n",
       "                                              IME VRSTA_ID VRSTA_IME  \\\n",
       "476307  0000 (zemljepisno ime v viru ni določeno)        2    prehod   \n",
       "476308  0000 (zemljepisno ime v viru ni določeno)        2    prehod   \n",
       "\n",
       "                                                 geometry  \n",
       "476307  LINESTRING Z (534309.280 134630.090 319.370, 5...  \n",
       "476308  LINESTRING Z (588983.590 162427.830 182.070, 5...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered = gdf[(gdf.TIPTV_ID.isin([\"1\", \"5\", \"6\"])) & (gdf.STANJE_ID == \"4\")]\n",
    "filtered.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "205523\n",
      "Removed 274791 features (of 480314 total).\n"
     ]
    }
   ],
   "source": [
    "print(filtered.shape[0])\n",
    "diff = gdf.shape[0] - filtered.shape[0]\n",
    "print(f\"Removed {diff} features (of {gdf.shape[0]} total).\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save filtered\n",
    "pth_save = shp_src[:-4] + \"_filter_tiptv_1_5_6_stanje_4.shp\"\n",
    "filtered.to_file(pth_save)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "279.273px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}