parse_load_gdf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse and Load .gdf\n",
    "Parse and the gdf file into 2 csv files, 1 for metadata and the other for relations data. This is for further processing in `R`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-14T13:49:12.570457Z",
     "start_time": "2019-05-14T13:49:12.068829Z"
    }
   },
   "outputs": [],
   "source": [
    "from langdetect import detect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-14T13:49:19.557227Z",
     "start_time": "2019-05-14T13:49:12.576827Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split `.gdf` file into metadata and relation files for processing in `R`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-14T13:49:19.585538Z",
     "start_time": "2019-05-14T13:49:19.562240Z"
    }
   },
   "outputs": [],
   "source": [
    "def parse_gdf_channel(file_name):\n",
    "    with open(file_name) as f:\n",
    "        rows = [line.strip().split(',') for line in f.readlines()]\n",
    "        # print(rows)\n",
    "        metadata = [row for row in rows if len(row) > 3]\n",
    "        graph_data = [row for row in rows if len(row) == 3]\n",
    "        return pd.DataFrame(metadata[1:], columns=metadata[0]), pd.DataFrame(\n",
    "            graph_data[1:], columns=graph_data[0])\n",
    "\n",
    "def parse_gdf_video(file_name):\n",
    "    with open(file_name, encoding='utf-8') as f:\n",
    "        rows = [line.strip().split(',') for line in f.readlines()]\n",
    "        # print(rows)\n",
    "        metadata = [row[:14] for row in rows if len(row) > 3]\n",
    "        graph_data = [row for row in rows if len(row) == 3]\n",
    "        \n",
    "        return pd.DataFrame(metadata[1:], columns=metadata[0]), pd.DataFrame(\n",
    "            graph_data[1:], columns=graph_data[0])\n",
    "    \n",
    "def all_caps_to_proper(string):\n",
    "    out_string = ''\n",
    "    for token in string.split(' '):\n",
    "        if token.isupper():\n",
    "            out_string += ' ' + token.capitalize()\n",
    "        else:\n",
    "            out_string += ' ' + token\n",
    "    return out_string.strip()\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-14T14:29:38.796249Z",
     "start_time": "2019-05-14T14:29:35.059913Z"
    }
   },
   "outputs": [],
   "source": [
    "def join_update_clean_dfs(df_graph, df_metadata, title, replace_node_ids=True):\n",
    "    df_metadata.rename(columns={\n",
    "        'nodedef>name VARCHAR': 'nodeid',\n",
    "        'label VARCHAR': 'label',\n",
    "        'subscriberCount INT': 'subscriberCount'\n",
    "    },\n",
    "        inplace=True)\n",
    "    df_graph.rename(columns={\n",
    "        'edgedef>node1 VARCHAR': 'nodeid_1',\n",
    "        'node2 VARCHAR': 'nodeid_2',\n",
    "        'directed BOOLEAN': 'directed'\n",
    "    },\n",
    "        inplace=True)\n",
    "    if replace_node_ids is True:\n",
    "        df_graph = pd.merge(df_graph,\n",
    "                            df_metadata[['label', 'nodeid']],\n",
    "                            left_on='nodeid_1',\n",
    "                            right_on='nodeid',\n",
    "                            how='right')[['label', 'nodeid_2', 'directed'\n",
    "                                          ]].rename(columns={'label': 'node1'})\n",
    "\n",
    "        df_graph = pd.merge(df_graph,\n",
    "                            df_metadata[['label', 'nodeid']],\n",
    "                            left_on='nodeid_2',\n",
    "                            right_on='nodeid',\n",
    "                            how='right')[['node1', 'label', 'directed'\n",
    "                                          ]].rename(columns={'label': 'node2'})\n",
    "    else:\n",
    "        df_graph = pd.merge(df_graph,\n",
    "                            df_metadata[['label', 'nodeid']],\n",
    "                            left_on='nodeid_1',\n",
    "                            right_on='label',\n",
    "                            how='right')[['label', 'nodeid_2', 'directed'\n",
    "                                          ]].rename(columns={'label': 'node1'})\n",
    "\n",
    "        df_graph = pd.merge(df_graph,\n",
    "                            df_metadata[['label', 'nodeid']],\n",
    "                            left_on='nodeid_2',\n",
    "                            right_on='label',\n",
    "                            how='right')[['node1', 'label', 'directed'\n",
    "                                          ]].rename(columns={'label': 'node2'})\n",
    "    return df_metadata, df_graph\n",
    "\n",
    "\n",
    "def load_video_data(filename):\n",
    "    df_video_metadata, df_video_rel = parse_gdf_video(filename + '.gdf')\n",
    "    # write un-cleaned relations to file for use in gephi later\n",
    "    df_video_rel.to_csv(filename + '_relations_orig.csv')\n",
    "    # clean up titles for language classification\n",
    "    df_video_metadata['label_clean'] = df_video_metadata['label VARCHAR'].apply(all_caps_to_proper)\n",
    "    # detect video title language \n",
    "    df_video_metadata['title_language'] = df_video_metadata['label_clean'].apply(detect)\n",
    "    # clean column names and update node ids\n",
    "    df_video_metadata, df_video_rel = join_update_clean_dfs(df_video_rel, df_video_metadata, filename)\n",
    "    df_video_metadata['url'] = 'https://www.youtube.com/watch?v=' + df_video_metadata['nodeid']\n",
    "    # get channel ids for seed videos\n",
    "    seed_channel_ids = list(df_video_metadata[df_video_metadata['isSeed VARCHAR'] == 'yes']['channelId VARCHAR'])\n",
    "    # label whether or not all videos are in the seed channel\n",
    "    df_video_metadata['seed_channel'] = df_video_metadata['channelId VARCHAR'].apply(lambda x : True if x in seed_channel_ids else False)\n",
    "    df_video_metadata.to_csv(filename + '_metadata.csv')\n",
    "    df_video_rel.to_csv(filename + '_relations.csv')\n",
    "    return df_video_metadata, df_video_rel\n",
    "    \n",
    "\n",
    "def load_channel_data(filename, replace_node_ids=True):\n",
    "    df_metadata, df_rel = parse_gdf_channel(filename + '.gdf')        \n",
    "    # clean column names and update node ids\n",
    "    df_metadata, df_graph = join_update_clean_dfs(df_rel, df_metadata, filename, replace_node_ids=replace_node_ids)\n",
    "    df_metadata['url'] = 'https://www.youtube.com/channel/' + df_metadata['nodeid']\n",
    "    df_graph = df_graph.dropna()\n",
    "    df_metadata.to_csv(filename + '_metadata.csv')\n",
    "    df_graph.to_csv(filename + '_relations.csv')\n",
    "    return df_metadata, df_graph\n",
    "    \n",
    "\n",
    "    \n",
    "df_video_metadata, df_video_graph = load_video_data('masha_and_shark_2019_05_14')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}