analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import calendar\n",
    "import datetime\n",
    "import math\n",
    "import os\n",
    "from pprint import pprint\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "# %matplotlib widget\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO:\n",
    "* error bars for the \"by month\" plots\n",
    "* top_x authors over time - all months not just top months\n",
    "* top_x_all authors over time\n",
    "* make by-month and per-month graphs more smooth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_NAME = 'all'\n",
    "# FILE_NAME = 'test'\n",
    "OUTPUT_DATA_FILE = os.path.join('data', f'{FILE_NAME}_data.parquet')\n",
    "OUTPUT_METADATA_FILE = os.path.join('data', f'{FILE_NAME}_meta.parquet')\n",
    "\n",
    "AGGREGATION_MIN = 5\n",
    "\n",
    "os.makedirs('pngs', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_dataframe(dfs_to_plot, xvalues=None, title=None, xlabel=None, ylabel=None, ylim_bottom=0, yscale=None, lables=None, override_font=False, show_plot=True, output_file_name=None):\n",
    "    with plt.xkcd(scale=0.5):\n",
    "        # Make it into a list if it isn't\n",
    "        dfs_to_plot = [dfs_to_plot] if type(dfs_to_plot) is not list else dfs_to_plot\n",
    "\n",
    "        # The styalized font XKCD uses doesn't have very much unicode coverage, override font if you need to use unicode text\n",
    "        if override_font:\n",
    "            matplotlib.rc('font', family='Arial')\n",
    "        \n",
    "        # Set fig size\n",
    "        plt.figure(figsize=(1920/80, 1080/80))\n",
    "        \n",
    "        for df_to_plot in dfs_to_plot:\n",
    "            # Plot with labels if provided, else without\n",
    "            if xvalues is not None:\n",
    "                plt.plot(xvalues, df_to_plot)\n",
    "            else:\n",
    "                plt.plot(df_to_plot)\n",
    "\n",
    "        if ylim_bottom is not None:\n",
    "            plt.ylim(bottom=ylim_bottom)\n",
    "        \n",
    "        # Style plot and add text\n",
    "        plt.grid(True, lw=0.5, zorder=0)\n",
    "        plt.title(title)\n",
    "        plt.xlabel(xlabel)\n",
    "        plt.ylabel(ylabel)\n",
    "        \n",
    "        # Semilog-y plots\n",
    "        # TODO: is this working?\n",
    "        if yscale is not None:\n",
    "            plt.yscale(yscale)\n",
    "\n",
    "        # Add lables if provided\n",
    "        if lables is not None:\n",
    "            plt.legend(lables)\n",
    "\n",
    "        # Save and plot!\n",
    "        if output_file_name is not None:\n",
    "            plt.savefig(output_file_name)\n",
    "        if show_plot:\n",
    "            plt.show()\n",
    "\n",
    "def plot_windowed_msg_per_min(msg_per_min, window_length=10, show_plot=True, output_file_name='test.png'):\n",
    "    # Pad with zeros before the start of the server for filtering\n",
    "    pad_length = int(math.ceil(window_length/2.0))\n",
    "    pre_pad = msg_per_min[:pad_length]\n",
    "    pre_pad = pre_pad.tshift(-pad_length)\n",
    "    pre_pad[:] = 0\n",
    "\n",
    "    post_pad = msg_per_min[-pad_length:]\n",
    "    post_pad = post_pad.tshift(pad_length)\n",
    "    post_pad[:] = msg_per_min[-pad_length:].mean()\n",
    "\n",
    "    # Filter with centered blackman-harris window function and slice off the pad data\n",
    "    filtered_msg_per_min = pd.concat([pre_pad, msg_per_min, post_pad])\n",
    "    filtered_msg_per_min = filtered_msg_per_min.rolling(window_length, center=True, win_type='blackmanharris').mean()[pad_length:-pad_length]\n",
    "    \n",
    "    # Plot filtered data\n",
    "    plot_dataframe(\n",
    "        filtered_msg_per_min,\n",
    "        title=f'Averaged Smoothed Message Rate Over History ({window_length*AGGREGATION_MIN/60/24} day window)',\n",
    "        xlabel='Datetime (ref:UTC)',\n",
    "        ylabel='msg/min (avg)',\n",
    "        show_plot=show_plot,\n",
    "        output_file_name=output_file_name\n",
    "    )\n",
    "\n",
    "def plot_windowed_msg_per_min2(msg_per_min, window_days=7, show_plot=True, output_file_name='test.png', lables=None):\n",
    "    # Pad with zeros before the start of the server, and mean of half the window after the last data point, for filtering\n",
    "    window_lengths = np.atleast_1d(window_days)*np.timedelta64(1, 'D').astype('timedelta64[m]')/np.timedelta64(AGGREGATION_MIN, 'm')\n",
    "    pad_lengths = list(np.round(np.ceil(np.array(window_lengths)/2.0)).astype('int'))\n",
    "\n",
    "    filtered_msg_per_min_list = []\n",
    "    for i in range(len(window_lengths)):\n",
    "        window_length = int(window_lengths[i])\n",
    "        pad_length = pad_lengths[i]\n",
    "\n",
    "        pre_pad = msg_per_min[:pad_length]\n",
    "        pre_pad = pre_pad.tshift(-pad_length)\n",
    "        pre_pad[:] = 0\n",
    "\n",
    "        post_pad = msg_per_min[-pad_length:]\n",
    "        post_pad = post_pad.tshift(pad_length)\n",
    "        post_pad[:] = msg_per_min[-pad_length:].mean()\n",
    "\n",
    "        # Filter with centered blackman-harris window function and slice off the pad data\n",
    "        filtered_msg_per_min = pd.concat([pre_pad, msg_per_min, post_pad])\n",
    "        filtered_msg_per_min = filtered_msg_per_min.rolling(window_length, center=True, win_type='blackmanharris').mean()[pad_length:-pad_length]\n",
    "        filtered_msg_per_min_list.append(filtered_msg_per_min)\n",
    "    \n",
    "    # Plot filtered data\n",
    "    plot_dataframe(\n",
    "        filtered_msg_per_min_list,\n",
    "#         title=f'Averaged Smoothed Message Rate Over History ({[str(i*AGGREGATION_MIN/60/24) + \" \" for i in window_lengths]} day window)',\n",
    "        title=f'Averaged Smoothed Message Rate Over History ({window_lengths*AGGREGATION_MIN/60/24} day window)',\n",
    "        xlabel='Datetime (ref:UTC)',\n",
    "        ylabel='msg/min (avg)',\n",
    "        show_plot=show_plot,\n",
    "        output_file_name=output_file_name,\n",
    "        lables=lables,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_parquet(f'{OUTPUT_DATA_FILE}')\n",
    "metadata = pd.read_parquet(f'{OUTPUT_METADATA_FILE}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Number of messages per channel out of {len(data)} total messages:')\n",
    "msgs_by_user = data['channel_name'].value_counts()\n",
    "msgs_by_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "msg_per_min = pd.Series(1, index=data['creation_datetime']).resample(f'{AGGREGATION_MIN}min').count()/AGGREGATION_MIN\n",
    "# msg_per_min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# window_length_7 = int(round(datetime.timedelta(days=7).total_seconds()/60/AGGREGATION_MIN))\n",
    "# plot_windowed_msg_per_min(msg_per_min, window_length_7, output_file_name='pngs/msg_rate_7day_window.png')\n",
    "plot_windowed_msg_per_min2(msg_per_min, window_days=7, output_file_name='pngs/msg_rate_7day_window.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# window_length_30 = int(round(datetime.timedelta(days=30).total_seconds()/60/AGGREGATION_MIN))\n",
    "# plot_windowed_msg_per_min(msg_per_min, window_length_30, output_file_name='pngs/msg_rate_30day_window.png')\n",
    "plot_windowed_msg_per_min2(msg_per_min, window_days=30, output_file_name='pngs/msg_rate_30day_window.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# window_length_365 = int(round(datetime.timedelta(days=365).total_seconds()/60/AGGREGATION_MIN))\n",
    "# plot_windowed_msg_per_min(msg_per_min, window_length_365, output_file_name='pngs/msg_rate_365day_window.png')\n",
    "plot_windowed_msg_per_min2(msg_per_min, window_days=365, output_file_name='pngs/msg_rate_365day_window.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_windowed_msg_per_min2(msg_per_min, [7, 30, 365], lables=['7 Day Window', '30 Day Window', '365 Day Window'], output_file_name='pngs/msg_rate_7_30_365day_window.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Message rate over hours of the day\n",
    "plot_dataframe(\n",
    "    msg_per_min.groupby(msg_per_min.index.hour).mean(),\n",
    "    title='Average message rate over hour of the day',\n",
    "    xlabel='Hour of the day (ref:UTC)',\n",
    "    ylabel='msg/min (avg)',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/hour_of_day.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Message rate over day of the week\n",
    "plot_dataframe(\n",
    "    msg_per_min.groupby(msg_per_min.index.dayofweek).mean(),\n",
    "    xvalues=list(calendar.day_name),\n",
    "    title='Average message rate over day of the week',\n",
    "    xlabel='Day of the week (ref:UTC)',\n",
    "    ylabel='msg/min (avg)',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/day_of_week.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Message rate over day of the week\n",
    "plot_dataframe(\n",
    "    msg_per_min.groupby(msg_per_min.index.weekofyear).mean(),\n",
    "    title='Message rate over week of the year',\n",
    "    xlabel='Week of the Year (ref:UTC)',\n",
    "    ylabel='msg/min (avg)',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/week_of_year.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# comulitive sum of messages across users\n",
    "plot_dataframe(\n",
    "    data['author'].value_counts().to_numpy()/len(data),\n",
    "    title='Fraction of total messages by user',\n",
    "    xlabel='Users',\n",
    "    ylabel='Fraction of total messages',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/msg_by_user_fraction.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# comulitive sum of messages across users, reverse-sorted\n",
    "plot_dataframe(\n",
    "    data['author'].value_counts()[::-1].cumsum().to_numpy()/len(data),\n",
    "    title='Cumsum of fraction of total messages by user',\n",
    "    xlabel='Users',\n",
    "    ylabel='Fraction of total messages',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/msg_by_user_cumsum.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_counts_by_month = [(n, g['author'].value_counts()) for n, g in data.groupby(pd.Grouper(key='creation_datetime', freq='M'))]\n",
    "mean_msgs_per_author_counts_by_month = pd.Series([i[1].mean() for i in author_counts_by_month], [i[0] for i in author_counts_by_month])\n",
    "active_users_by_month = pd.Series([i[1].count() for i in author_counts_by_month], [i[0] for i in author_counts_by_month])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('% of total messages for the top 10 most prolific authors:')\n",
    "msgs_by_user = data['author'].value_counts()\n",
    "print(msgs_by_user[:10]/len(data)*100)\n",
    "\n",
    "# print('')\n",
    "# # Replace \"author\" with the author string of your choice, the format is \"name#1234\"\n",
    "# print(f'msgs by \"author\": {msgs_by_user[\"author\"]/len(data)*100}%    #{msgs_by_user.index.get_loc(\"author\")+1} on the server')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average msgs per active user per month\n",
    "plot_dataframe(\n",
    "    active_users_by_month,\n",
    "    title='Active users per month',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='Active Users',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/active_users_per_month.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    active_users_by_month.groupby(active_users_by_month.index.month).mean(),\n",
    "    title='Active users',\n",
    "    xlabel='Month (ref:UTC)',\n",
    "    ylabel='Active users/month',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/active_users_by_month.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average msgs per active user per month\n",
    "plot_dataframe(\n",
    "    mean_msgs_per_author_counts_by_month,\n",
    "    title='Average messages per active user per month',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='Average messages/user',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/msg_per_user_per_month.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    mean_msgs_per_author_counts_by_month.groupby(mean_msgs_per_author_counts_by_month.index.month).mean(),\n",
    "    title='Average msgs per active user by month',\n",
    "    xlabel='Month (ref:UTC)',\n",
    "    ylabel='Average messages/user',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/msg_per_user_by_month.png',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOP_N_PER_MONTH = 1\n",
    "\n",
    "top_author_counts_by_month = [(i, j[0:TOP_N_PER_MONTH]) for i, j in author_counts_by_month]\n",
    "all_top_authors = set()\n",
    "for i, j in top_author_counts_by_month:\n",
    "    [all_top_authors.add(i) for i in j.index.to_list()]\n",
    "\n",
    "print(f'All users that have been in the top {TOP_N_PER_MONTH} authors in any given month in the history of the server:')\n",
    "pprint(sorted(list(all_top_authors)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init the dataframe\n",
    "top_authors_across_months_count = pd.DataFrame(index=[i for i, j in top_author_counts_by_month])\n",
    "for i in all_top_authors:\n",
    "    top_authors_across_months_count[i] = 0.0\n",
    "\n",
    "# over all months and the top authors of all time, calculate the number of messages send, zero if they had no messages that month\n",
    "for i, j in author_counts_by_month:\n",
    "    for k in all_top_authors:\n",
    "        top_authors_across_months_count.at[i, k] = j.get(k, 0)\n",
    "\n",
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    top_authors_across_months_count/(30*24),\n",
    "    title=f'Average msgs per hour for each of the top {TOP_N_PER_MONTH} users in any month',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='messages/hour',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/rate_top_bymonth_users.png',\n",
    "    lables=all_top_authors,\n",
    "    override_font=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init the dataframe\n",
    "top_authors_across_months_perc = pd.DataFrame(index=[i for i, j in top_author_counts_by_month])\n",
    "for i in all_top_authors:\n",
    "    top_authors_across_months_perc[i] = 0.0\n",
    "\n",
    "# over all months and the top authors of all time, calculate the percentage of messages sent by a particular user, zero if they had no messages that month\n",
    "for i, j in author_counts_by_month:\n",
    "    for k in all_top_authors:\n",
    "        top_authors_across_months_perc.at[i, k] = 100*j.get(k, 0)/j.sum()\n",
    "\n",
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    top_authors_across_months_perc,\n",
    "    title=f'Percentage of total msgs per month for each of the top {TOP_N_PER_MONTH} users in any month',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='Percent of total messages/month',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/perc_top_bymonth_users.png',\n",
    "    lables=all_top_authors,\n",
    "    override_font=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOP_N_EVER = 5\n",
    "top_n_users = msgs_by_user[:TOP_N_EVER].index.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init the dataframe\n",
    "top_authors_count = pd.DataFrame(index=[i for i, j in top_author_counts_by_month])\n",
    "for i in top_n_users:\n",
    "    top_authors_count[i] = 0\n",
    "\n",
    "# over all months and the top authors of all time, calculate the number of messages send, zero if they had no messages that month\n",
    "for i, j in author_counts_by_month:\n",
    "    for k in top_n_users:\n",
    "        top_authors_count.at[i, k] = j.get(k, 0)\n",
    "\n",
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    top_authors_count/(30*24),\n",
    "    title=f'Average msgs per hour for each of the top {TOP_N_EVER} authors ever',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='messages/hour',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/perc_top_users.png',\n",
    "    lables=msgs_by_user[:TOP_N_EVER].index.to_list(),\n",
    "    override_font=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init the dataframe\n",
    "top_authors_perc = pd.DataFrame(index=[i for i, j in top_author_counts_by_month])\n",
    "for i in top_n_users:\n",
    "    top_authors_perc[i] = 0.0\n",
    "\n",
    "# over all months and the top authors of all time, calculate the percentage of messages sent by a particular user, zero if they had no messages that month\n",
    "for i, j in author_counts_by_month:\n",
    "    for k in top_n_users:\n",
    "        top_authors_perc.at[i, k] = 100.0*j.get(k, 0)/float(j.sum())\n",
    "\n",
    "# Average msgs per active user by month\n",
    "plot_dataframe(\n",
    "    top_authors_perc,\n",
    "    title=f'Percent of total msgs per month for each of the top {TOP_N_EVER} users of all time',\n",
    "    xlabel='Datetime (ref:UTC)',\n",
    "    ylabel='Percentage of messages/month',\n",
    "#     yscale='log'\n",
    "    output_file_name='pngs/rate_top_users.png',\n",
    "    lables=msgs_by_user[:TOP_N_EVER].index.to_list(),\n",
    "    override_font=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (discord)",
   "language": "python",
   "name": "discord"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}